: Fix SMP oops in pppol2tp driver
Subsystems (4)
PATCH 1/1 : Fix SMP oops in pppol2tp driver
Fix locking issues in the pppol2tp driver which can cause a kernel crash on SMP boxes when hundreds of L2TP sessions are created/deleted simultaneously (ISP environment). The driver was violating read_lock() and write_lock() scheduling rules so we now consistently use the _irq variants of the lock functions.
--
This patch has been verified by the ISP that discovered the problem.
If the patch is accepted, it should be pushed to the stable 2.6.23 and 2.6.24 trees.
drivers/net/pppol2tp.c +32 -22
@@ -301,15 +301,16 @@ pppol2tp_session_find(struct pppol2tp_tupppol2tp_session_id_hash (tunnel, session_id);struct pppol2tp_session *session;struct hlist_node *walk; +unsigned long flags; -read_lock (&tunnel->hlist_lock); +read_lock_irqsave (&tunnel->hlist_lock, flags);hlist_for_each_entry (session, walk, session_list, hlist) {if (session->tunnel_addr.s_session == session_id) { -read_unlock (&tunnel->hlist_lock); +read_unlock_irqrestore (&tunnel->hlist_lock, flags);return session; } } -read_unlock (&tunnel->hlist_lock); +read_unlock_irqrestore (&tunnel->hlist_lock, flags);return NULL ; }
@@ -319,15 +320,16 @@ pppol2tp_session_find(struct pppol2tp_tustatic struct pppol2tp_tunnel *pppol2tp_tunnel_find (u16 tunnel_id) {struct pppol2tp_tunnel *tunnel =NULL ; +unsigned long flags; -read_lock (&pppol2tp_tunnel_list_lock); +read_lock_irqsave (&pppol2tp_tunnel_list_lock, flags);list_for_each_entry (tunnel, &pppol2tp_tunnel_list, list) {if (tunnel->stats.tunnel_id == tunnel_id) { -read_unlock (&pppol2tp_tunnel_list_lock); +read_unlock_irqrestore (&pppol2tp_tunnel_list_lock, flags);return tunnel; } } -read_unlock (&pppol2tp_tunnel_list_lock); +read_unlock_irqrestore (&pppol2tp_tunnel_list_lock, flags);return NULL ; }
@@ -1099,6 +1101,7 @@ static void pppol2tp_tunnel_closeall(strstruct hlist_node *tmp;struct pppol2tp_session *session;struct sock *sk; +unsigned long flags;if (tunnel ==NULL )BUG ();
@@ -1106,7 +1109,7 @@ static void pppol2tp_tunnel_closeall(strPRINTK (tunnel->debug,PPPOL2TP_MSG_CONTROL ,KERN_INFO ,"%s: closing all sessions...\n" , tunnel->name); -write_lock (&tunnel->hlist_lock); +write_lock_irqsave (&tunnel->hlist_lock, flags);for (hash =0 ; hash <PPPOL2TP_HASH_SIZE ; hash++) { again:hlist_for_each_safe (walk, tmp, &tunnel->session_hlist[hash]) {
@@ -1126,7 +1129,7 @@ again:* disappear as we're jumping between locks. */ sock_hold (sk); -write_unlock (&tunnel->hlist_lock); +write_unlock_irqrestore (&tunnel->hlist_lock, flags);lock_sock (sk);if (sk->sk_state & (PPPOX_CONNECTED |PPPOX_BOUND )) {
@@ -1148,11 +1151,11 @@ again:* list so we are guaranteed to make forward * progress. */ -write_lock (&tunnel->hlist_lock); +write_lock_irqsave (&tunnel->hlist_lock, flags); goto again; } } -write_unlock (&tunnel->hlist_lock); +write_unlock_irqrestore (&tunnel->hlist_lock, flags); }/* Really kill the tunnel.
@@ -1160,10 +1163,12 @@ again:*/ static void pppol2tp_tunnel_free (struct pppol2tp_tunnel *tunnel) { +unsigned long flags; +/* Remove from socket list */ -write_lock (&pppol2tp_tunnel_list_lock); +write_lock_irqsave (&pppol2tp_tunnel_list_lock, flags);list_del_init (&tunnel->list); -write_unlock (&pppol2tp_tunnel_list_lock); +write_unlock_irqrestore (&pppol2tp_tunnel_list_lock, flags);atomic_dec (&pppol2tp_tunnel_count);kfree (tunnel);
@@ -1212,6 +1217,7 @@ end:static void pppol2tp_session_destruct (struct sock *sk) {struct pppol2tp_session *session =NULL ; +unsigned long flags;if (sk->sk_user_data !=NULL ) {struct pppol2tp_tunnel *tunnel;
@@ -1239,9 +1245,9 @@ static void pppol2tp_session_destruct(st/* Delete the session socket from the * hash */ -write_lock (&tunnel->hlist_lock); +write_lock_irqsave (&tunnel->hlist_lock, flags);hlist_del_init (&session->hlist); -write_unlock (&tunnel->hlist_lock); +write_unlock_irqrestore (&tunnel->hlist_lock, flags);atomic_dec (&pppol2tp_session_count); }
@@ -1312,6 +1318,7 @@ static struct sock *pppol2tp_prepare_tunstruct sock *sk;struct pppol2tp_tunnel *tunnel;struct sock *ret =NULL ; +unsigned long flags;/* Get the tunnel UDP socket from the fd, which was opened by * the userspace L2TP daemon.
@@ -1386,9 +1393,9 @@ static struct sock *pppol2tp_prepare_tun/* Add tunnel to our list */ INIT_LIST_HEAD (&tunnel->list); -write_lock (&pppol2tp_tunnel_list_lock); +write_lock_irqsave (&pppol2tp_tunnel_list_lock, flags);list_add (&tunnel->list, &pppol2tp_tunnel_list); -write_unlock (&pppol2tp_tunnel_list_lock); +write_unlock_irqrestore (&pppol2tp_tunnel_list_lock, flags);atomic_inc (&pppol2tp_tunnel_count);/* Bump the reference count. The tunnel context is deleted
@@ -1462,6 +1469,7 @@ static int pppol2tp_connect(struct sockestruct pppol2tp_tunnel *tunnel;struct dst_entry *dst;int error =0 ; +unsigned long irqflags;lock_sock (sk);
@@ -1593,11 +1601,11 @@ static int pppol2tp_connect(struct socke sk->sk_user_data = session;/* Add session to the tunnel's hash list */ -write_lock (&tunnel->hlist_lock); +write_lock_irqsave (&tunnel->hlist_lock, irqflags);hlist_add_head (&session->hlist,pppol2tp_session_id_hash (tunnel, session->tunnel_addr.s_session)); -write_unlock (&tunnel->hlist_lock); +write_unlock_irqrestore (&tunnel->hlist_lock, irqflags);atomic_inc (&pppol2tp_session_count);
@@ -2198,8 +2206,9 @@ static struct pppol2tp_session *next_sesint found =0 ;int next =0 ;int i; +unsigned long flags; -read_lock (&tunnel->hlist_lock); +read_lock_irqsave (&tunnel->hlist_lock, flags);for (i =0 ; i <PPPOL2TP_HASH_SIZE ; i++) {hlist_for_each_entry (session, walk, &tunnel->session_hlist[i], hlist) {if (curr ==NULL ) {
@@ -2217,7 +2226,7 @@ static struct pppol2tp_session *next_ses } } out: -read_unlock (&tunnel->hlist_lock); +read_unlock_irqrestore (&tunnel->hlist_lock, flags);if (!found) session =NULL ;
@@ -2227,14 +2236,15 @@ out:static struct pppol2tp_tunnel *next_tunnel (struct pppol2tp_tunnel *curr) {struct pppol2tp_tunnel *tunnel =NULL ; +unsigned long flags; -read_lock (&pppol2tp_tunnel_list_lock); +read_lock_irqsave (&pppol2tp_tunnel_list_lock, flags);if (list_is_last (&curr->list, &pppol2tp_tunnel_list)) { goto out; } tunnel = list_entry(curr->list.next, struct pppol2tp_tunnel, list); out: -read_unlock (&pppol2tp_tunnel_list_lock); +read_unlock_irqrestore (&pppol2tp_tunnel_list_lock, flags);return tunnel; }
James Chapman wrote, On 02/11/2008 10:22 AM:
Quoted 5 lines
Fix locking issues in the pppol2tp driver which can cause a kernel crash on SMP boxes when hundreds of L2TP sessions are created/deleted simultaneously (ISP environment). The driver was violating
read_lock()andwrite_lock()scheduling rules so we now consistently use the_irqvariants of the lock functions.
...
Hi,
Could you explain what exactly scheduling rules do you mean here, and why disabling interrupts is the best solution for this?
Thanks,
Jarek P.
Quoted 7 lines · Jarek Poplawski
James Chapman wrote, On 02/11/2008 10:22 AM:
Quoted 5 lines
Fix locking issues in the pppol2tp driver which can cause a kernel crash on SMP boxes when hundreds of L2TP sessions are created/deleted simultaneously (ISP environment). The driver was violating
read_lock()andwrite_lock()scheduling rules so we now consistently use the_irqvariants of the lock functions.
Quoted 6 lines
...
Hi,
Could you explain what exactly scheduling rules do you mean here, and why disabling interrupts is the best solution for this?
Below is example output from lockdep. The oops is reproducible when creating/deleting lots of sessions while passing data. The lock is being acquired for read and write in softirq contexts.
Is there a better way to fix this?
=================================
[ INFO: inconsistent lock state ] 2.6.24-core2 #1
---------------------------------
inconsistent {in-softirq-R} -> {softirq-on-W} usage. openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes:
(&tunnel->hlist_lock){---?}, at: [<f8eea157>]pppol2tp_connect+0x517/0x6d0 [pppol2tp] {in-softirq-R} state was registered at:
[<c014edaf>] __lock_acquire+0x6bf/0x10a0 [<c03ee75b>] fn_hash_lookup+0x1b/0xe0 [<c014f804>] lock_acquire+0x74/0xa0 [<f8ee859f>] pppol2tp_session_find+0x1f/0x80 [pppol2tp] [<c040427a>] _read_lock+0x2a/0x40 [<f8ee859f>] pppol2tp_session_find+0x1f/0x80 [pppol2tp] [<f8ee859f>] pppol2tp_session_find+0x1f/0x80 [pppol2tp] [<f8ee8dc8>] pppol2tp_recv_core+0xd8/0x960 [pppol2tp] [<f8d3f72a>] ipt_do_table+0x23a/0x500 [ip_tables] [<f8ee967e>] pppol2tp_udp_encap_recv+0x2e/0x70 [pppol2tp] [<c0403fb4>] _read_unlock+0x14/0x20 [<c03dd696>] udp_queue_rcv_skb+0x106/0x2a0 [<c03ddc5a>] __udp4_lib_rcv+0x42a/0x7e0 [<f8d57090>] ipt_hook+0x0/0x20 [iptable_filter] [<c03bc2da>] ip_local_deliver_finish+0xca/0x1c0 [<c03bc23e>] ip_local_deliver_finish+0x2e/0x1c0 [<c03bbfaf>] ip_rcv_finish+0xff/0x360 [<c03bc6dc>] ip_rcv+0x20c/0x2a0 [<c03bbeb0>] ip_rcv_finish+0x0/0x360 [<c039ad87>] netif_receive_skb+0x317/0x4b0 [<c039ab70>] netif_receive_skb+0x100/0x4b0 [<f8d9627a>] e1000_clean_rx_irq_ps+0x28a/0x560 [e1000] [<f8d95ff0>] e1000_clean_rx_irq_ps+0x0/0x560 [e1000] [<f8d9384d>] e1000_clean+0x5d/0x290 [e1000] [<c039d580>] net_rx_action+0x1a0/0x2a0 [<c039d43f>] net_rx_action+0x5f/0x2a0 [<c0131e72>] __do_softirq+0x92/0x120 [<c0131f78>] do_softirq+0x78/0x80 [<c010b15a>] do_IRQ+0x4a/0xa0 [<c0108dcc>] common_interrupt+0x24/0x34 [<c0108dd6>] common_interrupt+0x2e/0x34 [<c01062d6>] mwait_idle_with_hints+0x46/0x60 [<c0106550>] mwait_idle+0x0/0x20 [<c0106694>] cpu_idle+0x74/0xe0 [<c0536a9a>] start_kernel+0x30a/0x3a0 [<c0536150>] unknown_bootoption+0x0/0x1f0 [<ffffffff>] 0xffffffff irq event stamp: 275 hardirqs last enabled at (275): [<c0132317>] local_bh_enable_ip+0xa7/0x120 hardirqs last disabled at (273): [<c01322a6>] local_bh_enable_ip+0x36/0x120 softirqs last enabled at (274): [<f8eab8bc>] ppp_register_channel+0xdc/0xf0 [ppp_generic] softirqs last disabled at (272): [<c040410b>] _spin_lock_bh+0xb/0x40
Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote:
...
Quoted 6 lines
Below is example output from lockdep. The oops is reproducible when creating/deleting lots of sessions while passing data. The lock is being acquired for read and write in softirq contexts.
Is there a better way to fix this?
Quoted 9 lines
=================================
[ INFO: inconsistent lock state ] 2.6.24-core2 #1
---------------------------------
inconsistent {in-softirq-R} -> {softirq-on-W} usage. openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes:
(&tunnel->hlist_lock){---?}, at: [<f8eea157>]pppol2tp_connect+0x517/0x6d0[pppol2tp] {in-softirq-R} state was registered at:
IMHO, according to this, disabling bh should be enough. And if it's like in this report: only read_lock is taken from softirqs, then this should be necessary to change only all write_locks to write_lock_bh (of course unless somewhere bhs are disabled already). Unless I miss something?!
Cheers,
Jarek P.
Quoted 5 lines · Jarek Poplawski
On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote:
...Quoted 3 lines
Below is example output from lockdep. The oops is reproducible when creating/deleting lots of sessions while passing data. The lock is being acquired for read and write in softirq contexts.
...Hmmm... And according to this, changing read_locks should be necessary too.
Jarek P.
Quoted 7 lines · Jarek Poplawski
Quoted 3 lines · Jarek Poplawski
On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote:
...Quoted 1 line
Below is example output from lockdep. The oops is reproducible when creating/deleting lots of sessions while passing data. The lock is being acquired for read and write in softirq contexts.
...Hmmm... And according to this, changing
read_locksshould be necessary too.
The patch changes both read and write locks.
Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
Quoted 4 lines · James Chapman
Quoted 3 lines · Jarek Poplawski
Quoted 2 lines · Jarek Poplawski
On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote:
...
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
Quoted 3 lines
Below is example output from lockdep. The oops is reproducible when
creating/deleting lots of sessions while passing data. The lock is
being acquired for read and write in softirq contexts....Hmmm... And according to this, changing
read_locksshould be necessary too.The patch changes both read and write locks.
Right! This was only "errata" to my earlier comment where I considered only lockdep info and forgot about yours...
Sorry,
Jarek P.
Quoted 16 lines · Jarek Poplawski
On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote:
...Quoted 4 lines
Below is example output from lockdep. The oops is reproducible when creating/deleting lots of sessions while passing data. The lock is being acquired for read and write in softirq contexts.
Is there a better way to fix this?
Quoted 9 lines
=================================
[ INFO: inconsistent lock state ] 2.6.24-core2 #1
---------------------------------
inconsistent {in-softirq-R} -> {softirq-on-W} usage. openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes:
(&tunnel->hlist_lock){---?}, at: [<f8eea157>]pppol2tp_connect+0x517/0x6d0[pppol2tp] {in-softirq-R} state was registered at:
Quoted 5 lines
IMHO, according to this, disabling bh should be enough. And if it's like in this report: only
read_lockis taken from softirqs, then this should be necessary to change only allwrite_lockstowrite_lock_bh(of course unless somewhere bhs are disabled already). Unless I miss something?!
I thought so too. I tried _bh locks first and the problem still occurred. Maybe I'll try it again in case I messed something up.
Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
David Miller
maintainerFrom: James Chapman <[email protected]>
Date: Mon, 11 Feb 2008 23:41:18 +0000
Quoted 19 lines
Quoted 18 lines · Jarek Poplawski
On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote:
...Quoted 15 lines
Below is example output from lockdep. The oops is reproducible when creating/deleting lots of sessions while passing data. The lock is being acquired for read and write in softirq contexts.
Is there a better way to fix this?
=================================
[ INFO: inconsistent lock state ] 2.6.24-core2 #1
---------------------------------
inconsistent {in-softirq-R} -> {softirq-on-W} usage. openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes:
(&tunnel->hlist_lock){---?}, at: [<f8eea157>]pppol2tp_connect+0x517/0x6d0[pppol2tp] {in-softirq-R} state was registered at:
Quoted 8 lines
Quoted 5 lines
IMHO, according to this, disabling bh should be enough. And if it's like in this report: only
read_lockis taken from softirqs, then this should be necessary to change only allwrite_lockstowrite_lock_bh(of course unless somewhere bhs are disabled already). Unless I miss something?!I thought so too. I tried
_bhlocks first and the problem still occurred. Maybe I'll try it again in case I messed something up.
I agree with Jarek here, I look at all the code paths that take ->hlist_lock and all of them are in user context or software interrupts.
Please get a lockdep trace with the change to _bh intead of hw interrupt protection so we can find out what that doesn't work.
Thanks!
Quoted 30 lines · David Miller
From: James Chapman <[email protected]>
Date: Mon, 11 Feb 2008 23:41:18 +0000Quoted 16 lines
Quoted 15 lines · Jarek Poplawski
On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote:
...Quoted 13 lines
Below is example output from lockdep. The oops is reproducible when creating/deleting lots of sessions while passing data. The lock is being acquired for read and write in softirq contexts.
Is there a better way to fix this?
=================================
[ INFO: inconsistent lock state ] 2.6.24-core2 #1
---------------------------------
inconsistent {in-softirq-R} -> {softirq-on-W} usage. openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes:
(&tunnel->hlist_lock){---?}, at: [<f8eea157>]pppol2tp_connect+0x517/0x6d0[pppol2tp] {in-softirq-R} state was registered at:Quoted 6 lines
Quoted 5 lines
IMHO, according to this, disabling bh should be enough. And if it's like in this report: only
read_lockis taken from softirqs, then this should be necessary to change only allwrite_lockstowrite_lock_bh(of course unless somewhere bhs are disabled already). Unless I miss something?!I thought so too. I tried
_bhlocks first and the problem still occurred. Maybe I'll try it again in case I messed something up.I agree with Jarek here, I look at all the code paths that take ->hlist_lock and all of them are in user context or software interrupts.
Quoted 5 lines
Please get a lockdep trace with the change to
_bhintead of hw interrupt protection so we can find out what that doesn't work.Thanks!
Here is a trace from when we had _bh locks.
Feb 5 16:26:32 ======================================================
Feb 5 16:26:32 [ INFO: soft-safe -> soft-unsafe lock order detected ]
Feb 5 16:26:32 2.6.24-core2 #1
Feb 5 16:26:32 ------------------------------------------------------
Feb 5 16:26:32 pppd/3224 [HC0[0]:SC0[2]:HE1:SE0] is trying to acquire:
Feb 5 16:26:32 (&sk->sk_dst_lock){----}, at: [<f8efacac>] pppol2tp_xmit+0x23c/0x460 [pppol2tp]
Feb 5 16:26:32
Feb 5 16:26:32 and this task is already holding:
Feb 5 16:26:32 (&pch->downl){-...}, at: [<f8eb828e>] ppp_push+0x44e/0x620 [ppp_generic]
Feb 5 16:26:32 which would create a new lock dependency:
Feb 5 16:26:32 (&pch->downl){-...} -> (&sk->sk_dst_lock){----}
Feb 5 16:26:32
Feb 5 16:26:32 but this new dependency connects a soft-irq-safe lock:
Feb 5 16:26:32 (&pch->upl){-.-+}
Feb 5 16:26:32 ... which became soft-irq-safe at:
Feb 5 16:26:32 [<c014d87f>] check_usage_backwards+0x1f/0x50
Feb 5 16:26:32 [<c014c479>] save_trace+0x39/0xa0
Feb 5 16:26:32 [<c014edaf>] __lock_acquire+0x6bf/0x10a0
Feb 5 16:26:32 [<c014ee8e>] __lock_acquire+0x79e/0x10a0
Feb 5 16:26:32 [<c014ee8e>] __lock_acquire+0x79e/0x10a0
Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0
Feb 5 16:26:32 [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic]
Feb 5 16:26:32 [<c040425f>] _read_lock_bh+0x2f/0x40
Feb 5 16:26:32 [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic]
Feb 5 16:26:32 [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic]
Feb 5 16:26:32 [<f8ef915a>] pppol2tp_recv_core+0x46a/0x960 [pppol2tp]
Feb 5 16:26:32 [<f8ef967e>] pppol2tp_udp_encap_recv+0x2e/0x70 [pppol2tp]
Feb 5 16:26:32 [<c0403fd4>] _read_unlock+0x14/0x20
Feb 5 16:26:32 [<c03dd6b6>] udp_queue_rcv_skb+0x106/0x2a0
Feb 5 16:26:32 [<c03ddc7a>] __udp4_lib_rcv+0x42a/0x7e0
Feb 5 16:26:32 [<f8d5c090>] ipt_hook+0x0/0x20 [iptable_filter]
Feb 5 16:26:32 [<c03bc2fa>] ip_local_deliver_finish+0xca/0x1c0
Feb 5 16:26:32 [<c03bc25e>] ip_local_deliver_finish+0x2e/0x1c0
Feb 5 16:26:32 [<c03bbfcf>] ip_rcv_finish+0xff/0x360
Feb 5 16:26:32 [<c03bc6fc>] ip_rcv+0x20c/0x2a0
Feb 5 16:26:32 [<c03bbed0>] ip_rcv_finish+0x0/0x360
Feb 5 16:26:32 [<c039ad87>] netif_receive_skb+0x317/0x4b0
Feb 5 16:26:32 [<c039ab70>] netif_receive_skb+0x100/0x4b0
Feb 5 16:26:32 [<f8d7e27a>] e1000_clean_rx_irq_ps+0x28a/0x560 [e1000]
Feb 5 16:26:32 [<f8d7dff0>] e1000_clean_rx_irq_ps+0x0/0x560 [e1000]
Feb 5 16:26:32 [<f8d7b84d>] e1000_clean+0x5d/0x290 [e1000]
Feb 5 16:26:32 [<c039d580>] net_rx_action+0x1a0/0x2a0
Feb 5 16:26:32 [<c039d43f>] net_rx_action+0x5f/0x2a0
Feb 5 16:26:32 [<c0131e72>] __do_softirq+0x92/0x120
Feb 5 16:26:32 [<c0131f78>] do_softirq+0x78/0x80
Feb 5 16:26:32 [<c010b15a>] do_IRQ+0x4a/0xa0
Feb 5 16:26:32 [<c0127af0>] finish_task_switch+0x0/0xc0
Feb 5 16:26:32 [<c0108dcc>] common_interrupt+0x24/0x34
Feb 5 16:26:32 [<c0108dd6>] common_interrupt+0x2e/0x34
Feb 5 16:26:32 [<c01062d6>] mwait_idle_with_hints+0x46/0x60
Feb 5 16:26:32 [<c0106550>] mwait_idle+0x0/0x20
Feb 5 16:26:32 [<c0106694>] cpu_idle+0x74/0xe0
Feb 5 16:26:32 [<c0536a9a>] start_kernel+0x30a/0x3a0
Feb 5 16:26:32 [<c0536150>] unknown_bootoption+0x0/0x1f0
Feb 5 16:26:32 [<ffffffff>] 0xffffffff
Feb 5 16:26:32
Feb 5 16:26:32 to a soft-irq-unsafe lock:
Feb 5 16:26:32 (&sk->sk_dst_lock){----}
Feb 5 16:26:32 ... which became soft-irq-unsafe at:
Feb 5 16:26:32 ... [<c014e02e>] mark_held_locks+0x5e/0x80
Feb 5 16:26:32 [<c014ed92>] __lock_acquire+0x6a2/0x10a0
Feb 5 16:26:32 [<c010f5b0>] save_stack_trace+0x20/0x40
Feb 5 16:26:32 [<c014c524>] add_lock_to_list+0x44/0xb0
Feb 5 16:26:32 [<c03dea29>] __udp_lib_get_port+0x19/0x200
Feb 5 16:26:32 [<c014f735>] __lock_acquire+0x1045/0x10a0
Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0
Feb 5 16:26:32 [<c03db0a3>] ip4_datagram_connect+0x53/0x380
Feb 5 16:26:32 [<c040418a>] _write_lock+0x2a/0x40
Feb 5 16:26:32 [<c03db0a3>] ip4_datagram_connect+0x53/0x380
Feb 5 16:26:32 [<c03db0a3>] ip4_datagram_connect+0x53/0x380
Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb 5 16:26:32 [<c0132317>] local_bh_enable_ip+0xa7/0x120
Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb 5 16:26:32 [<c040414f>] _spin_lock_bh+0x2f/0x40
Feb 5 16:26:32 [<c03e4d55>] inet_dgram_connect+0x35/0x80
Feb 5 16:26:32 [<c038ec52>] sys_connect+0x82/0xd0
Feb 5 16:26:32 [<c01455df>] down_read_trylock+0x4f/0x60
Feb 5 16:26:32 [<c011fe9c>] do_page_fault+0xfc/0x940
Feb 5 16:26:32 [<c0404024>] _spin_unlock+0x14/0x20
Feb 5 16:26:32 [<c03905f8>] sys_socketcall+0x98/0x280
Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb 5 16:26:32 [<c02a86ba>] copy_to_user+0x3a/0x70
Feb 5 16:26:32 [<c0108417>] restore_nocheck+0x12/0x15
Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb
Feb 5 16:26:32 [<ffffffff>] 0xffffffff
Feb 5 16:26:32
Feb 5 16:26:32 other info that might help us debug this:
Feb 5 16:26:32
Feb 5 16:26:32 2 locks held by pppd/3224:
Feb 5 16:26:32 #0: (&ppp->wlock){-...}, at: [<f8eb8478>] ppp_xmit_process+0x18/0x630 [ppp_generic]
Feb 5 16:26:32 #1: (&pch->downl){-...}, at: [<f8eb828e>] ppp_push+0x44e/0x620 [ppp_generic]
Feb 5 16:26:32
Feb 5 16:26:32 the soft-irq-safe lock's dependencies:
Feb 5 16:26:32 -> (&pch->upl){-.-+} ops: 0 {
Feb 5 16:26:32 initial-use at:
Feb 5 16:26:32 [<c014e80d>] __lock_acquire+0x11d/0x10a0
Feb 5 16:26:32 [<f8efacf3>] pppol2tp_xmit+0x283/0x460 [pppol2tp]
Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0
Feb 5 16:26:32 [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic]
Feb 5 16:26:32 [<c040425f>] _read_lock_bh+0x2f/0x40
Feb 5 16:26:32 [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic]
Feb 5 16:26:32 [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic]
Feb 5 16:26:32 [<f8eb9ffd>] ppp_write+0xdd/0x110 [ppp_generic]
Feb 5 16:26:32 [<c0181f86>] vfs_write+0xa6/0x140
Feb 5 16:26:32 [<c0108417>] restore_nocheck+0x12/0x15
Feb 5 16:26:32 [<f8eb9f20>] ppp_write+0x0/0x110 [ppp_generic]
Feb 5 16:26:32 [<c0182651>] sys_write+0x41/0x70
Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb
Feb 5 16:26:32 [<ffffffff>] 0xffffffff
Feb 5 16:26:32 hardirq-on-W at:
Feb 5 16:26:32 [<c014ee8e>] __lock_acquire+0x79e/0x10a0
Feb 5 16:26:32 [<c014c524>] add_lock_to_list+0x44/0xb0
Feb 5 16:26:32 [<c014ed68>] __lock_acquire+0x678/0x10a0
Feb 5 16:26:32 [<c014e02e>] mark_held_locks+0x5e/0x80
Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0
Feb 5 16:26:32 [<f8eb7387>] ppp_ioctl+0x647/0xf40 [ppp_generic]
Feb 5 16:26:32 [<c04041cf>] _write_lock_bh+0x2f/0x40
Feb 5 16:26:32 [<f8eb7387>] ppp_ioctl+0x647/0xf40 [ppp_generic]
Feb 5 16:26:32 [<f8eb7387>] ppp_ioctl+0x647/0xf40 [ppp_generic]
Feb 5 16:26:32 [<c0404665>] _spin_unlock_irqrestore+0x55/0x70
Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb 5 16:26:32 [<c0403e4e>] __down+0xde/0xed
Feb 5 16:26:32 [<c0124220>] default_wake_function+0x0/0x10
Feb 5 16:26:32 [<c0403bdb>] __down_failed+0x7/0xc
Feb 5 16:26:32 [<c018d8f8>] do_ioctl+0x78/0x90
Feb 5 16:26:32 [<c018d96c>] vfs_ioctl+0x5c/0x290
Feb 5 16:26:32 [<c018dbdd>] sys_ioctl+0x3d/0x70
Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb
Feb 5 16:26:32 [<ffffffff>] 0xffffffff
Feb 5 16:26:32 in-softirq-R at:
Feb 5 16:26:32 [<c014d87f>] check_usage_backwards+0x1f/0x50
Feb 5 16:26:32 [<c014c479>] save_trace+0x39/0xa0
Feb 5 16:26:32 [<c014edaf>] __lock_acquire+0x6bf/0x10a0
Feb 5 16:26:32 [<c014ee8e>] __lock_acquire+0x79e/0x10a0
Feb 5 16:26:32 [<c014ee8e>] __lock_acquire+0x79e/0x10a0
Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0
Feb 5 16:26:32 [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic]
Feb 5 16:26:32 [<c040425f>] _read_lock_bh+0x2f/0x40
Feb 5 16:26:32 [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic]
Feb 5 16:26:32 [<f8eba092>] ppp_input+0x62/0x140 [ppp_generic]
Feb 5 16:26:32 [<f8ef915a>] pppol2tp_recv_core+0x46a/0x960 [pppol2tp]
Feb 5 16:26:32 [<f8ef967e>] pppol2tp_udp_encap_recv+0x2e/0x70 [pppol2tp]
Feb 5 16:26:32 [<c0403fd4>] _read_unlock+0x14/0x20
Feb 5 16:26:32 [<c03dd6b6>] udp_queue_rcv_skb+0x106/0x2a0
Feb 5 16:26:32 [<c03ddc7a>] __udp4_lib_rcv+0x42a/0x7e0
Feb 5 16:26:32 [<f8d5c090>] ipt_hook+0x0/0x20 [iptable_filter]
Feb 5 16:26:32 [<c03bc2fa>] ip_local_deliver_finish+0xca/0x1c0
Feb 5 16:26:32 [<c03bc25e>] ip_local_deliver_finish+0x2e/0x1c0
Feb 5 16:26:32 [<c03bbfcf>] ip_rcv_finish+0xff/0x360
Feb 5 16:26:32 [<c03bc6fc>] ip_rcv+0x20c/0x2a0
Feb 5 16:26:32 [<c03bbed0>] ip_rcv_finish+0x0/0x360
Feb 5 16:26:32 [<c039ad87>] netif_receive_skb+0x317/0x4b0
Feb 5 16:26:32 [<c039ab70>] netif_receive_skb+0x100/0x4b0
Feb 5 16:26:32 [<f8d7e27a>] e1000_clean_rx_irq_ps+0x28a/0x560 [e1000]
Feb 5 16:26:32 [<f8d7dff0>] e1000_clean_rx_irq_ps+0x0/0x560 [e1000]
Feb 5 16:26:32 [<f8d7b84d>] e1000_clean+0x5d/0x290 [e1000]
Feb 5 16:26:32 [<c039d580>] net_rx_action+0x1a0/0x2a0
Feb 5 16:26:32 [<c039d43f>] net_rx_action+0x5f/0x2a0
Feb 5 16:26:32 [<c0131e72>] __do_softirq+0x92/0x120
Feb 5 16:26:32 [<c0131f78>] do_softirq+0x78/0x80
Feb 5 16:26:32 [<c010b15a>] do_IRQ+0x4a/0xa0
Feb 5 16:26:32 [<c0127af0>] finish_task_switch+0x0/0xc0
Feb 5 16:26:32 [<c0108dcc>] common_interrupt+0x24/0x34
Feb 5 16:26:32 [<c0108dd6>] common_interrupt+0x2e/0x34
Feb 5 16:26:32 [<c01062d6>] mwait_idle_with_hints+0x46/0x60
Feb 5 16:26:32 [<c0106550>] mwait_idle+0x0/0x20
Feb 5 16:26:32 [<c0106694>] cpu_idle+0x74/0xe0
Feb 5 16:26:32 [<c0536a9a>] start_kernel+0x30a/0x3a0
Feb 5 16:26:32 [<c0536150>] unknown_bootoption+0x0/0x1f0
Feb 5 16:26:32 [<ffffffff>] 0xffffffff
Feb 5 16:26:32 hardirq-on-R at:
Feb 5 16:26:32 [<c014da27>] mark_lock+0x77/0x620
Feb 5 16:26:32 [<c014ec2f>] __lock_acquire+0x53f/0x10a0
Feb 5 16:26:32 [<f8efacf3>] pppol2tp_xmit+0x283/0x460 [pppol2tp]
Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0
Feb 5 16:26:32 [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic]
Feb 5 16:26:32 [<c040425f>] _read_lock_bh+0x2f/0x40
Feb 5 16:26:32 [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic]
Feb 5 16:26:32 [<f8eb8af2>] ppp_channel_push+0x62/0x90 [ppp_generic]
Feb 5 16:26:32 [<f8eb9ffd>] ppp_write+0xdd/0x110 [ppp_generic]
Feb 5 16:26:32 [<c0181f86>] vfs_write+0xa6/0x140
Feb 5 16:26:32 [<c0108417>] restore_nocheck+0x12/0x15
Feb 5 16:26:32 [<f8eb9f20>] ppp_write+0x0/0x110 [ppp_generic]
Feb 5 16:26:32 [<c0182651>] sys_write+0x41/0x70
Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb
Feb 5 16:26:32 [<ffffffff>] 0xffffffff
Feb 5 16:26:32 }
Feb 5 16:26:32 ... key at: [<f8ebdf20>] __key.30415+0x0/0xffffc2e7 [ppp_generic]
Feb 5 16:26:32 -> (&ppp->wlock){-...} ops: 0 {
Feb 5 16:26:32 initial-use at:
Feb 5 16:26:32 [<c014e80d>] __lock_acquire+0x11d/0x10a0
Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0
Feb 5 16:26:32 [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic]
Feb 5 16:26:32 [<c040414f>] _spin_lock_bh+0x2f/0x40
Feb 5 16:26:32 [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic]
Feb 5 16:26:32 [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic]
Feb 5 16:26:32 [<c0404665>] _spin_unlock_irqrestore+0x55/0x70
Feb 5 16:26:32 [<c0403e4e>] __down+0xde/0xed
Feb 5 16:26:32 [<c0124220>] default_wake_function+0x0/0x10
Feb 5 16:26:32 [<c0403bdb>] __down_failed+0x7/0xc
Feb 5 16:26:32 [<c018d8f8>] do_ioctl+0x78/0x90
Feb 5 16:26:32 [<c018d96c>] vfs_ioctl+0x5c/0x290
Feb 5 16:26:32 [<c018dbdd>] sys_ioctl+0x3d/0x70
Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb
Feb 5 16:26:32 [<ffffffff>] 0xffffffff
Feb 5 16:26:32 hardirq-on-W at:
Feb 5 16:26:32 [<c014da27>] mark_lock+0x77/0x620
Feb 5 16:26:32 [<c014ed68>] __lock_acquire+0x678/0x10a0
Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0
Feb 5 16:26:32 [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic]
Feb 5 16:26:32 [<c040414f>] _spin_lock_bh+0x2f/0x40
Feb 5 16:26:32 [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic]
Feb 5 16:26:32 [<f8eb7698>] ppp_ioctl+0x958/0xf40 [ppp_generic]
Feb 5 16:26:32 [<c0404665>] _spin_unlock_irqrestore+0x55/0x70
Feb 5 16:26:32 [<c0403e4e>] __down+0xde/0xed
Feb 5 16:26:32 [<c0124220>] default_wake_function+0x0/0x10
Feb 5 16:26:32 [<c0403bdb>] __down_failed+0x7/0xc
Feb 5 16:26:32 [<c018d8f8>] do_ioctl+0x78/0x90
Feb 5 16:26:32 [<c018d96c>] vfs_ioctl+0x5c/0x290
Feb 5 16:26:32 [<c018dbdd>] sys_ioctl+0x3d/0x70
Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb
Feb 5 16:26:32 [<ffffffff>] 0xffffffff
Feb 5 16:26:32 }
Feb 5 16:26:32 ... key at: [<f8ebdf08>] __key.30659+0x0/0xffffc2ff [ppp_generic]
Feb 5 16:26:32 -> (&ppp->rlock){-...} ops: 0 {
Feb 5 16:26:32 initial-use at:
Feb 5 16:26:32 [<c014e80d>] __lock_acquire+0x11d/0x10a0
Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0
Feb 5 16:26:32 [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic]
Feb 5 16:26:32 [<c040414f>] _spin_lock_bh+0x2f/0x40
Feb 5 16:26:32 [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic]
Feb 5 16:26:32 [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic]
Feb 5 16:26:32 [<c0404665>] _spin_unlock_irqrestore+0x55/0x70
Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb 5 16:26:32 [<c0403e4e>] __down+0xde/0xed
Feb 5 16:26:32 [<c0124220>] default_wake_function+0x0/0x10
Feb 5 16:26:32 [<c0403bdb>] __down_failed+0x7/0xc
Feb 5 16:26:32 [<c018d8f8>] do_ioctl+0x78/0x90
Feb 5 16:26:32 [<c018d96c>] vfs_ioctl+0x5c/0x290
Feb 5 16:26:32 [<c018dbdd>] sys_ioctl+0x3d/0x70
Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb
Feb 5 16:26:32 [<ffffffff>] 0xffffffff
Feb 5 16:26:32 hardirq-on-W at:
Feb 5 16:26:32 [<c014da27>] mark_lock+0x77/0x620
Feb 5 16:26:32 [<c014ed68>] __lock_acquire+0x678/0x10a0
Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0
Feb 5 16:26:32 [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic]
Feb 5 16:26:32 [<c040414f>] _spin_lock_bh+0x2f/0x40
Feb 5 16:26:32 [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic]
Feb 5 16:26:32 [<f8eb7661>] ppp_ioctl+0x921/0xf40 [ppp_generic]
Feb 5 16:26:32 [<c0404665>] _spin_unlock_irqrestore+0x55/0x70
Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb 5 16:26:32 [<c0403e4e>] __down+0xde/0xed
Feb 5 16:26:32 [<c0124220>] default_wake_function+0x0/0x10
Feb 5 16:26:32 [<c0403bdb>] __down_failed+0x7/0xc
Feb 5 16:26:32 [<c018d8f8>] do_ioctl+0x78/0x90
Feb 5 16:26:32 [<c018d96c>] vfs_ioctl+0x5c/0x290
Feb 5 16:26:32 [<c018dbdd>] sys_ioctl+0x3d/0x70
Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb
Feb 5 16:26:32 [<ffffffff>] 0xffffffff
Feb 5 16:26:32 }
Feb 5 16:26:32 ... key at: [<f8ebdf10>] __key.30658+0x0/0xffffc2f7 [ppp_generic]
Feb 5 16:26:32 ... acquired at:
Feb 5 16:26:32 [<c014f68f>] __lock_acquire+0xf9f/0x10a0
Feb 5 16:26:32 [<f8eb76a5>] ppp_ioctl+0x965/0xf40 [ppp_generic]
Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0
Feb 5 16:26:32 [<f8eb76a5>] ppp_ioctl+0x965/0xf40 [ppp_generic]
Feb 5 16:26:32 [<c040414f>] _spin_lock_bh+0x2f/0x40
Feb 5 16:26:32 [<f8eb76a5>] ppp_ioctl+0x965/0xf40 [ppp_generic]
Feb 5 16:26:32 [<f8eb76a5>] ppp_ioctl+0x965/0xf40 [ppp_generic]
Feb 5 16:26:32 [<c0404665>] _spin_unlock_irqrestore+0x55/0x70
Feb 5 16:26:32 [<c0403e4e>] __down+0xde/0xed
Feb 5 16:26:32 [<c0124220>] default_wake_function+0x0/0x10
Feb 5 16:26:32 [<c0403bdb>] __down_failed+0x7/0xc
Feb 5 16:26:32 [<c018d8f8>] do_ioctl+0x78/0x90
Feb 5 16:26:32 [<c018d96c>] vfs_ioctl+0x5c/0x290
Feb 5 16:26:32 [<c018dbdd>] sys_ioctl+0x3d/0x70
Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb
Feb 5 16:26:32 [<ffffffff>] 0xffffffff
Feb 5 16:26:32
Feb 5 16:26:32 -> (&list->lock#4){.+..} ops: 0 {
Feb 5 16:26:32 initial-use at:
Feb 5 16:26:32 [<c014e80d>] __lock_acquire+0x11d/0x10a0
Feb 5 16:26:32 [<c0193ba2>] dput+0xa2/0x130
Feb 5 16:26:32 [<c014ee8e>] __lock_acquire+0x79e/0x10a0
Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0
Feb 5 16:26:32 [<c0394b9c>] skb_queue_tail+0x1c/0x50
Feb 5 16:26:32 [<c040450c>] _spin_lock_irqsave+0x4c/0x70
Feb 5 16:26:32 [<c0394b9c>] skb_queue_tail+0x1c/0x50
Feb 5 16:26:32 [<c0394b9c>] skb_queue_tail+0x1c/0x50
Feb 5 16:26:32 [<f8eb9fd5>] ppp_write+0xb5/0x110 [ppp_generic]
Feb 5 16:26:32 [<c0181f86>] vfs_write+0xa6/0x140
Feb 5 16:26:32 [<c0108417>] restore_nocheck+0x12/0x15
Feb 5 16:26:32 [<f8eb9f20>] ppp_write+0x0/0x110 [ppp_generic]
Feb 5 16:26:32 [<c0182651>] sys_write+0x41/0x70
Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb
Feb 5 16:26:32 [<ffffffff>] 0xffffffff
Feb 5 16:26:32 in-softirq-W at:
Feb 5 16:26:32 [<c014c479>] save_trace+0x39/0xa0
Feb 5 16:26:32 [<c014db32>] mark_lock+0x182/0x620
Feb 5 16:26:32 [<c014ed36>] __lock_acquire+0x646/0x10a0
Feb 5 16:26:32 [<c014ef26>] __lock_acquire+0x836/0x10a0
Feb 5 16:26:32 [<c014ee8e>] __lock_acquire+0x79e/0x10a0
Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0
Feb 5 16:26:32 [<c0394b9c>] skb_queue_tail+0x1c/0x50
Feb 5 16:26:32 [<c040450c>] _spin_lock_irqsave+0x4c/0x70
Feb 5 16:26:32 [<c0394b9c>] skb_queue_tail+0x1c/0x50
Feb 5 16:26:32 [<c0394b9c>] skb_queue_tail+0x1c/0x50
Feb 5 16:26:32 [<f8eba0a8>] ppp_input+0x78/0x140 [ppp_generic]
Feb 5 16:26:32 [<f8ef915a>] pppol2tp_recv_core+0x46a/0x960 [pppol2tp]
Feb 5 16:26:32 [<f8ef967e>] pppol2tp_udp_encap_recv+0x2e/0x70 [pppol2tp]
Feb 5 16:26:32 [<c0403fd4>] _read_unlock+0x14/0x20
Feb 5 16:26:32 [<c03dd6b6>] udp_queue_rcv_skb+0x106/0x2a0
Feb 5 16:26:32 [<c03ddc7a>] __udp4_lib_rcv+0x42a/0x7e0
Feb 5 16:26:32 [<f8d5c090>] ipt_hook+0x0/0x20 [iptable_filter]
Feb 5 16:26:32 [<c03bc2fa>] ip_local_deliver_finish+0xca/0x1c0
Feb 5 16:26:32 [<c03bc25e>] ip_local_deliver_finish+0x2e/0x1c0
Feb 5 16:26:32 [<c03bbfcf>] ip_rcv_finish+0xff/0x360
Feb 5 16:26:32 [<c03bc6fc>] ip_rcv+0x20c/0x2a0
Feb 5 16:26:32 [<c03bbed0>] ip_rcv_finish+0x0/0x360
Feb 5 16:26:32 [<c039ad87>] netif_receive_skb+0x317/0x4b0
Feb 5 16:26:32 [<c039ab70>] netif_receive_skb+0x100/0x4b0
Feb 5 16:26:32 [<f8d7e27a>] e1000_clean_rx_irq_ps+0x28a/0x560 [e1000]
Feb 5 16:26:32 [<f8d7dff0>] e1000_clean_rx_irq_ps+0x0/0x560 [e1000]
Feb 5 16:26:32 [<f8d7b84d>] e1000_clean+0x5d/0x290 [e1000]
Feb 5 16:26:32 [<c039d580>] net_rx_action+0x1a0/0x2a0
Feb 5 16:26:32 [<c039d43f>] net_rx_action+0x5f/0x2a0
Feb 5 16:26:32 [<c0131e72>] __do_softirq+0x92/0x120
Feb 5 16:26:32 [<c0131f78>] do_softirq+0x78/0x80
Feb 5 16:26:32 [<c010b15a>] do_IRQ+0x4a/0xa0
Feb 5 16:26:32 [<c0127af0>] finish_task_switch+0x0/0xc0
Feb 5 16:26:32 [<c0108dcc>] common_interrupt+0x24/0x34
Feb 5 16:26:32 [<c0108dd6>] common_interrupt+0x2e/0x34
Feb 5 16:26:32 [<c01062d6>] mwait_idle_with_hints+0x46/0x60
Feb 5 16:26:32 [<c0106550>] mwait_idle+0x0/0x20
Feb 5 16:26:32 [<c0106694>] cpu_idle+0x74/0xe0
Feb 5 16:26:32 [<c0536a9a>] start_kernel+0x30a/0x3a0Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
On Tue, Feb 12, 2008 at 10:58:21AM +0000, James Chapman wrote:
...
Quoted 1 line
Here is a trace from when we had
_bhlocks.
Very nice...
...But since it's quite long, and if you don't know all these paths this could take some time, maybe one question: so if lockdep got these locks right (sometimes it can be wrong when the same structures are nested), then it seems some problem is with this place below. This lock is taken for writing with softirqs enabled here, and IMHO it would be interesting to test if changing this is enough for lockdep. It seems this is in ip4_datagram_connect() during sk_dst_reset() or sk_dst_set(). So maybe you could try with local_bh_disable/enable() around them (or maybe some better idea)? Anyway, I'll try to learn this more in the meantime.
Jarek P.
Quoted 29 lines
Feb 5 16:26:32 to a soft-irq-unsafe lock:
Feb 5 16:26:32 (&sk->sk_dst_lock){----}
Feb 5 16:26:32 ... which became soft-irq-unsafe at:
Feb 5 16:26:32 ... [<c014e02e>] mark_held_locks+0x5e/0x80
Feb 5 16:26:32 [<c014ed92>] __lock_acquire+0x6a2/0x10a0
Feb 5 16:26:32 [<c010f5b0>] save_stack_trace+0x20/0x40
Feb 5 16:26:32 [<c014c524>] add_lock_to_list+0x44/0xb0
Feb 5 16:26:32 [<c03dea29>] __udp_lib_get_port+0x19/0x200
Feb 5 16:26:32 [<c014f735>] __lock_acquire+0x1045/0x10a0
Feb 5 16:26:32 [<c014f804>] lock_acquire+0x74/0xa0
Feb 5 16:26:32 [<c03db0a3>] ip4_datagram_connect+0x53/0x380
Feb 5 16:26:32 [<c040418a>] _write_lock+0x2a/0x40
Feb 5 16:26:32 [<c03db0a3>] ip4_datagram_connect+0x53/0x380
Feb 5 16:26:32 [<c03db0a3>] ip4_datagram_connect+0x53/0x380
Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb 5 16:26:32 [<c0132317>] local_bh_enable_ip+0xa7/0x120
Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb 5 16:26:32 [<c040414f>] _spin_lock_bh+0x2f/0x40
Feb 5 16:26:32 [<c03e4d55>] inet_dgram_connect+0x35/0x80
Feb 5 16:26:32 [<c038ec52>] sys_connect+0x82/0xd0
Feb 5 16:26:32 [<c01455df>] down_read_trylock+0x4f/0x60
Feb 5 16:26:32 [<c011fe9c>] do_page_fault+0xfc/0x940
Feb 5 16:26:32 [<c0404024>] _spin_unlock+0x14/0x20
Feb 5 16:26:32 [<c03905f8>] sys_socketcall+0x98/0x280
Feb 5 16:26:32 [<c014e1c5>] trace_hardirqs_on+0xc5/0x170
Feb 5 16:26:32 [<c02a86ba>] copy_to_user+0x3a/0x70
Feb 5 16:26:32 [<c0108417>] restore_nocheck+0x12/0x15
Feb 5 16:26:32 [<c01083aa>] syscall_call+0x7/0xb
Feb 5 16:26:32 [<ffffffff>] 0xffffffffDavid Miller
maintainerFrom: James Chapman <[email protected]>
Date: Tue, 12 Feb 2008 10:58:21 +0000
Quoted 1 line
Here is a trace from when we had
_bhlocks.
The problem is that the pppol2tp code calls sk_dst_get() in software interrupt context and that is not allowed.
sk_dst_get() grabs sk->sk_dst_lock without any BH enabling or disabling.
It can do that because the usage is to make all the lock taking calls in user context, and in the packet processing paths use __sk_dst_get().
Probably what the pppol2tp code should do is use __sk_dst_check() instead of sk_dst_get(). You then have to be able to handle NULL returns, just like UDP sendmsg() does, which means you'll need to cook up a routing lookup if __sk_dst_check() gives you NULL because the route became obsolete.
Quoted 7 lines · David Miller
From: James Chapman <[email protected]>
Date: Tue, 12 Feb 2008 10:58:21 +0000Quoted 1 line
Here is a trace from when we had
_bhlocks.The problem is that the pppol2tp code calls
sk_dst_get()in software interrupt context and that is not allowed.
Actually, this lockdep report probably warns of something else:sk_dst_lock which is seen in process context with softirqs enabled implicitly endangers some other (ppp_generic) locks, which depend on ppp_generic pch->upl read_lock, which is taken in softirq context.
I can't see on this report any hardirqs dependancies, so it seems even blocking hard interrupts, just like in this patch, shouldn't solve this problem: if BHs were really disabled in exactly the same places, then it seems this warning should trigger in both variants.
I studied long ago some similar problem with pppoe, and it looked like ppp_generic needed some fix, but now I've to recall or re-learn almost all and it needs more time... Anyway, there is a possibility, this warning isn't so dangerous.
Quoted 6 lines
sk_dst_get()grabssk->sk_dst_lockwithout any BH enabling or disabling.It can do that because the usage is to make all the lock taking calls in user context, and in the packet processing paths use
__sk_dst_get().
BTW, does it mean that ip4_datagram_connect() can be used only in user context?
Quoted 5 lines
Probably what the pppol2tp code should do is use
__sk_dst_check()instead ofsk_dst_get(). You then have to be able to handle NULL returns, just like UDPsendmsg()does, which means you'll need to cook up a routing lookup if__sk_dst_check()gives you NULL because the route became obsolete.
I think that without changing ppp_generic or changing ip_queue_xmit() to something else in pppol2tp this would be really hard to avoid such a warning (and maybe not very necessary).
Regards,
Jarek P.
Hi,
It seems, this nice report is still uncomplete: could you check if there could have been something more yet?
Thanks,
Jarek P.
On Tue, Feb 12, 2008 at 10:58:21AM +0000, James Chapman wrote:
...
Quoted 7 lines
Here is a trace from when we had
_bhlocks.Feb 5 16:26:32 ====================================================== Feb 5 16:26:32 [ INFO: soft-safe -> soft-unsafe lock order detected ] Feb 5 16:26:32 2.6.24-core2 #1 Feb 5 16:26:32 ------------------------------------------------------ Feb 5 16:26:32 pppd/3224 [HC0[0]:SC0[2]:HE1:SE0] is trying to acquire:
...
Quoted 20 lines
Feb 5 16:26:32 [<f8d7b84d>] e1000_clean+0x5d/0x290 [e1000] Feb 5 16:26:32 [<c039d580>] net_rx_action+0x1a0/0x2a0 Feb 5 16:26:32 [<c039d43f>] net_rx_action+0x5f/0x2a0 Feb 5 16:26:32 [<c0131e72>] __do_softirq+0x92/0x120 Feb 5 16:26:32 [<c0131f78>] do_softirq+0x78/0x80 Feb 5 16:26:32 [<c010b15a>] do_IRQ+0x4a/0xa0 Feb 5 16:26:32 [<c0127af0>] finish_task_switch+0x0/0xc0 Feb 5 16:26:32 [<c0108dcc>] common_interrupt+0x24/0x34 Feb 5 16:26:32 [<c0108dd6>] common_interrupt+0x2e/0x34 Feb 5 16:26:32 [<c01062d6>] mwait_idle_with_hints+0x46/0x60 Feb 5 16:26:32 [<c0106550>] mwait_idle+0x0/0x20 Feb 5 16:26:32 [<c0106694>] cpu_idle+0x74/0xe0 Feb 5 16:26:32 [<c0536a9a>] start_kernel+0x30a/0x3a0Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
Quoted 4 lines · Jarek Poplawski
Hi,
It seems, this nice report is still uncomplete: could you check if there could have been something more yet?
Unfortunately the ISP's syslog stops. But I've been able to borrow two Quad Xeon boxes and have reproduced the problem.
Here's a new version of the patch. The patch avoids disabling irqs and fixes the sk_dst_get() usage that DaveM mentioned. But even with this patch, lockdep still complains if hundreds of ppp sessions are inserted into a tunnel as rapidly as possible (lockdep trace is below). I can stop these errors by wrapping the call to ppp_input() in pppol2tp_recv_dequeue_skb() with local_irq_save/restore. What is a better fix?
btw, I'm not sending this from my usual mail client so the patch will probably be mangled; I'm sending it only for feedback / discussion at this stage.
drivers/net/pppol2tp.c +0 -0
@@ -302,14 +302,14 @@ pppol2tp_session_find(struct pppol2tp_tu
struct pppol2tp_session *session; struct hlist_node *walk;
- read_lock(&tunnel->hlist_lock);
+ read_lock_bh(&tunnel->hlist_lock);
hlist_for_each_entry(session, walk, session_list, hlist) {
if (session->tunnel_addr.s_session == session_id) {
- read_unlock(&tunnel->hlist_lock);
+ read_unlock_bh(&tunnel->hlist_lock);
return session;
}
}- read_unlock(&tunnel->hlist_lock);
+ read_unlock_bh(&tunnel->hlist_lock);
return NULL;
}
@@ -320,14 +320,14 @@ static struct pppol2tp_tunnel *pppol2tp_
{
struct pppol2tp_tunnel *tunnel = NULL;- read_lock(&pppol2tp_tunnel_list_lock);
+ read_lock_bh(&pppol2tp_tunnel_list_lock);
list_for_each_entry(tunnel, &pppol2tp_tunnel_list, list) {
if (tunnel->stats.tunnel_id == tunnel_id) {
- read_unlock(&pppol2tp_tunnel_list_lock);
+ read_unlock_bh(&pppol2tp_tunnel_list_lock);
return tunnel;
}
}- read_unlock(&pppol2tp_tunnel_list_lock);
+ read_unlock_bh(&pppol2tp_tunnel_list_lock);
return NULL; } @@ -344,7 +344,7 @@ static void pppol2tp_recv_queue_skb(stru struct sk_buff *skbp; u16 ns = PPPOL2TP_SKB_CB(skb)->ns;
- spin_lock(&session->reorder_q.lock);
+ spin_lock_bh(&session->reorder_q.lock);
skb_queue_walk(&session->reorder_q, skbp) {
if (PPPOL2TP_SKB_CB(skbp)->ns > ns) {
__skb_insert(skb, skbp->prev, skbp, &session->reorder_q);
@@ -360,7 +360,7 @@ static void pppol2tp_recv_queue_skb(stru
__skb_queue_tail(&session->reorder_q, skb);out:
- spin_unlock(&session->reorder_q.lock);
+ spin_unlock_bh(&session->reorder_q.lock);
}
/* Dequeue a single skb.
@@ -442,7 +442,7 @@ static void pppol2tp_recv_dequeue(struct
* expect to send up next, dequeue it and any other
* in-sequence packets behind it.
*/
- spin_lock(&session->reorder_q.lock);
+ spin_lock_bh(&session->reorder_q.lock);
skb_queue_walk_safe(&session->reorder_q, skb, tmp) {
if (time_after(jiffies, PPPOL2TP_SKB_CB(skb)->expires)) {
session->stats.rx_seq_discards++;
@@ -469,13 +469,13 @@ static void pppol2tp_recv_dequeue(struct
goto out;
}
}
- spin_unlock(&session->reorder_q.lock);
+ spin_unlock_bh(&session->reorder_q.lock);
pppol2tp_recv_dequeue_skb(session, skb);
- spin_lock(&session->reorder_q.lock);
+ spin_lock_bh(&session->reorder_q.lock);
}out:
- spin_unlock(&session->reorder_q.lock);
+ spin_unlock_bh(&session->reorder_q.lock);
}
/* Internal receive frame. Do the real work of receiving an L2TP data frame @@ -964,6 +964,7 @@ static int pppol2tp_xmit(struct ppp_chan
static const u8 ppph[2] = { 0xff, 0x03 };
struct sock *sk = (struct sock *) chan->private;
struct sock *sk_tun;
+ struct rtable *rt;
int hdr_len;
struct pppol2tp_session *session;
struct pppol2tp_tunnel *tunnel;
@@ -1057,8 +1058,32 @@ static int pppol2tp_xmit(struct ppp_chan
nf_reset(skb);
/* Get routing info from the tunnel socket */- dst_release(skb->dst);
- skb->dst = sk_dst_get(sk_tun);
+ rt = (struct rtable *)__sk_dst_check(sk_tun, 0);
+ if (rt == NULL) {
+ struct flowi fl = { .oif = sk_tun->sk_bound_dev_if,
+ .nl_u = { .ip4_u =
+ { .daddr = inet->daddr,
+ .saddr = inet->saddr,
+ .tos = inet->tos } },
+ .proto = sk_tun->sk_protocol,
+ .uli_u = { .ports =
+ { .sport = inet->sport,
+ .dport = inet->dport } } };
+ security_sk_classify_flow(sk_tun, &fl);
+ rc = ip_route_output_flow(&rt, &fl, sk_tun, 1);
+ if (rc) {
+ if (rc == -ENETUNREACH)
+ IP_INC_STATS_BH(IPSTATS_MIB_OUTNOROUTES);
+ goto abort;
+ }
+
+ sk_dst_set(sk_tun, dst_clone(&rt->u.dst));
+ }
+
+ write_lock_bh(&sk_tun->sk_dst_lock);
+ skb->dst = __sk_dst_get(sk_tun);
+ dst_hold(skb->dst);
+ write_unlock_bh(&sk_tun->sk_dst_lock); skb_orphan(skb); skb->sk = sk_tun;
@@ -1106,7 +1131,7 @@ static void pppol2tp_tunnel_closeall(str
PRINTK(tunnel->debug, PPPOL2TP_MSG_CONTROL, KERN_INFO, "%s: closing all sessions...\n", tunnel->name);
- write_lock(&tunnel->hlist_lock);
+ write_lock_bh(&tunnel->hlist_lock);
for (hash = 0; hash < PPPOL2TP_HASH_SIZE; hash++) {
again:
hlist_for_each_safe(walk, tmp, &tunnel->session_hlist[hash]) {
@@ -1126,7 +1151,7 @@ again:
* disappear as we're jumping between locks.
*/
sock_hold(sk);
- write_unlock(&tunnel->hlist_lock);
+ write_unlock_bh(&tunnel->hlist_lock);
lock_sock(sk);
if (sk->sk_state & (PPPOX_CONNECTED | PPPOX_BOUND)) {@@ -1148,11 +1173,11 @@ again:
* list so we are guaranteed to make forward
* progress.
*/
- write_lock(&tunnel->hlist_lock);
+ write_lock_bh(&tunnel->hlist_lock); goto again; } }
- write_unlock(&tunnel->hlist_lock);
+ write_unlock_bh(&tunnel->hlist_lock);
}
/* Really kill the tunnel.
@@ -1161,9 +1186,9 @@ again:
static void pppol2tp_tunnel_free(struct pppol2tp_tunnel *tunnel)
{
/* Remove from socket list */
- write_lock(&pppol2tp_tunnel_list_lock);
+ write_lock_bh(&pppol2tp_tunnel_list_lock);
list_del_init(&tunnel->list);- write_unlock(&pppol2tp_tunnel_list_lock);
+ write_unlock_bh(&pppol2tp_tunnel_list_lock);
atomic_dec(&pppol2tp_tunnel_count); kfree(tunnel); @@ -1239,9 +1264,9 @@ static void pppol2tp_session_destruct(st /* Delete the session socket from the * hash */ - write_lock(&tunnel->hlist_lock); + write_lock_bh(&tunnel->hlist_lock); hlist_del_init(&session->hlist);
- write_unlock(&tunnel->hlist_lock);
+ write_unlock_bh(&tunnel->hlist_lock);
atomic_dec(&pppol2tp_session_count); }
@@ -1386,9 +1411,9 @@ static struct sock *pppol2tp_prepare_tun
/* Add tunnel to our list */
INIT_LIST_HEAD(&tunnel->list);
- write_lock(&pppol2tp_tunnel_list_lock);
+ write_lock_bh(&pppol2tp_tunnel_list_lock);
list_add(&tunnel->list, &pppol2tp_tunnel_list);
- write_unlock(&pppol2tp_tunnel_list_lock);
+ write_unlock_bh(&pppol2tp_tunnel_list_lock);
atomic_inc(&pppol2tp_tunnel_count);
/* Bump the reference count. The tunnel context is deleted
@@ -1546,14 +1571,15 @@ static int pppol2tp_connect(struct socke
session->mtu = session->mru = 1500 - PPPOL2TP_HEADER_OVERHEAD;
/* If PMTU discovery was enabled, use the MTU that was discovered */
- dst = sk_dst_get(sk);
+ read_lock_bh(&tunnel_sock->sk_dst_lock);
+ dst = __sk_dst_get(sk);
if (dst != NULL) {
- u32 pmtu = dst_mtu(__sk_dst_get(sk));
+ u32 pmtu = dst_mtu(dst);
if (pmtu != 0)
session->mtu = session->mru = pmtu -
PPPOL2TP_HEADER_OVERHEAD;
- dst_release(dst);
}+ read_unlock_bh(&tunnel_sock->sk_dst_lock);
/* Special case: if source & dest session_id == 0x0000, this socket is * being created to manage the tunnel. Don't add the session to the @@ -1593,11 +1619,11 @@ static int pppol2tp_connect(struct socke sk->sk_user_data = session; /* Add session to the tunnel's hash list */ - write_lock(&tunnel->hlist_lock); + write_lock_bh(&tunnel->hlist_lock); hlist_add_head(&session->hlist, pppol2tp_session_id_hash(tunnel, session->tunnel_addr.s_session));
- write_unlock(&tunnel->hlist_lock);
+ write_unlock_bh(&tunnel->hlist_lock);
atomic_inc(&pppol2tp_session_count);
@@ -2199,7 +2225,7 @@ static struct pppol2tp_session *next_ses
int next = 0; int i;
- read_lock(&tunnel->hlist_lock);
+ read_lock_bh(&tunnel->hlist_lock);
for (i = 0; i < PPPOL2TP_HASH_SIZE; i++) {
hlist_for_each_entry(session, walk, &tunnel->session_hlist[i], hlist) {
if (curr == NULL) {
@@ -2217,7 +2243,7 @@ static struct pppol2tp_session *next_ses
}
}
out:
- read_unlock(&tunnel->hlist_lock);
+ read_unlock_bh(&tunnel->hlist_lock);
if (!found)
session = NULL;@@ -2228,13 +2254,13 @@ static struct pppol2tp_tunnel *next_tunn
{
struct pppol2tp_tunnel *tunnel = NULL;- read_lock(&pppol2tp_tunnel_list_lock);
+ read_lock_bh(&pppol2tp_tunnel_list_lock);
if (list_is_last(&curr->list, &pppol2tp_tunnel_list)) {
goto out;
}
tunnel = list_entry(curr->list.next, struct pppol2tp_tunnel, list);out:
- read_unlock(&pppol2tp_tunnel_list_lock);
+ read_unlock_bh(&pppol2tp_tunnel_list_lock);
return tunnel; }
......................................................
Here's the lockdep trace (long).
Feb 18 20:56:18 localhost kernel: Feb 18 20:56:18 localhost kernel: ======================================================
Feb 18 20:56:18 localhost kernel: [ INFO: soft-safe -> soft-unsafe lock order detected ]
Feb 18 20:56:18 localhost kernel: 2.6.24.2 #1
Feb 18 20:56:18 localhost kernel: ------------------------------------------------------
Feb 18 20:56:18 localhost kernel: pppd/3241 [HC0[0]:SC0[2]:HE1:SE0] is trying to acquire:
Feb 18 20:56:18 localhost kernel: (&pch->downl){-...}, at: [<f8c566db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 20:56:18 localhost kernel: Feb 18 20:56:18 localhost kernel: and this task is already holding:
Feb 18 20:56:18 localhost kernel: (&ppp->wlock){-...}, at: [<f8c56f10>] ppp_xmit_process+0x15/0x5a1 [ppp_generic]
Feb 18 20:56:18 localhost kernel: which would create a new lock dependency:
Feb 18 20:56:18 localhost kernel: (&ppp->wlock){-...} -> (&pch->downl){-...}
Feb 18 20:56:18 localhost kernel: Feb 18 20:56:18 localhost kernel: but this new dependency connects a soft-irq-safe lock:
Feb 18 20:56:18 localhost kernel: (&pch->upl){-.-+}
Feb 18 20:56:18 localhost kernel: ... which became soft-irq-safe at:
Feb 18 20:56:18 localhost kernel: [<c0448013>] check_usage_backwards+0x19/0x41
Feb 18 20:56:18 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:18 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:18 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:18 localhost kernel: [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 20:56:18 localhost kernel: [<c061b373>] _read_lock_bh+0x2e/0x39
Feb 18 20:56:18 localhost kernel: [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 20:56:18 localhost kernel: [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 20:56:18 localhost kernel: [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 20:56:18 localhost kernel: [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 20:56:18 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 20:56:18 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 20:56:18 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 20:56:18 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 20:56:18 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 20:56:18 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 20:56:18 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:18 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 20:56:18 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 20:56:18 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:18 localhost kernel: [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 20:56:18 localhost kernel: [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 20:56:18 localhost kernel: [<f8984f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 20:56:19 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 20:56:19 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 20:56:19 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:19 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:19 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:19 localhost kernel: Feb 18 20:56:19 localhost kernel: to a soft-irq-unsafe lock:
Feb 18 20:56:19 localhost kernel: (&sk->sk_dst_lock){----}
Feb 18 20:56:19 localhost kernel: ... which became soft-irq-unsafe at:
Feb 18 20:56:19 localhost kernel: ... [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 18 20:56:19 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 20:56:19 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 20:56:19 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 20:56:19 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:19 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:56:19 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 20:56:19 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:56:19 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:56:19 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 20:56:19 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 20:56:19 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 20:56:19 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 20:56:19 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:19 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 20:56:19 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:19 localhost kernel: Feb 18 20:56:19 localhost kernel: other info that might help us debug this:
Feb 18 20:56:19 localhost kernel: Feb 18 20:56:19 localhost kernel: 1 lock held by pppd/3241:
Feb 18 20:56:19 localhost kernel: #0: (&ppp->wlock){-...}, at: [<f8c56f10>] ppp_xmit_process+0x15/0x5a1 [ppp_generic]
Feb 18 20:56:19 localhost kernel: Feb 18 20:56:19 localhost kernel: the soft-irq-safe lock's dependencies:
Feb 18 20:56:19 localhost kernel: -> (&pch->upl){-.-+} ops: 13 {
Feb 18 20:56:19 localhost kernel: initial-use at:
Feb 18 20:56:19 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:56:19 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:19 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:19 localhost kernel: [<f8c57b7c>] ppp_ioctl+0x4df/0xc06 [ppp_generic]
Feb 18 20:56:19 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:19 localhost kernel: [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 20:56:19 localhost kernel: [<c061b300>] _write_lock_bh+0x2e/0x39
Feb 18 20:56:20 localhost kernel: [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 20:56:20 localhost kernel: [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 20:56:20 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:20 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:20 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:20 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:20 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:56:20 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc
Feb 18 20:56:20 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 20:56:20 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 20:56:20 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 20:56:20 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:56:20 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:20 localhost kernel: hardirq-on-W at:
Feb 18 20:56:20 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:20 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:56:20 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 20:56:20 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:20 localhost kernel: [<f8c57b7c>] ppp_ioctl+0x4df/0xc06 [ppp_generic]
Feb 18 20:56:20 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:20 localhost kernel: [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 20:56:20 localhost kernel: [<c061b300>] _write_lock_bh+0x2e/0x39
Feb 18 20:56:20 localhost kernel: [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 20:56:20 localhost kernel: [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 20:56:20 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:20 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:20 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:20 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:20 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:56:20 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc
Feb 18 20:56:20 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 20:56:20 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 20:56:20 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 20:56:20 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:56:20 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:20 localhost kernel: in-softirq-R at:
Feb 18 20:56:20 localhost kernel: [<c0448013>] check_usage_backwards+0x19/0x41
Feb 18 20:56:20 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:20 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:20 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:20 localhost kernel: [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 20:56:21 localhost kernel: [<c061b373>] _read_lock_bh+0x2e/0x39
Feb 18 20:56:21 localhost kernel: [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 20:56:21 localhost kernel: [<f8c5940d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 20:56:21 localhost kernel: [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 20:56:21 localhost kernel: [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 20:56:21 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 20:56:21 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 20:56:21 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 20:56:21 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 20:56:21 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 20:56:21 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 20:56:21 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:21 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 20:56:21 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 20:56:21 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:21 localhost kernel: [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 20:56:21 localhost kernel: [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 20:56:21 localhost kernel: [<f8984f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 20:56:21 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 20:56:21 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 20:56:21 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:21 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:21 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:21 localhost kernel: hardirq-on-R at:
Feb 18 20:56:21 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 20:56:21 localhost kernel: [<c0449001>] __lock_acquire+0x446/0xbf1
Feb 18 20:56:21 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:21 localhost kernel: [<f8c5834b>] ppp_channel_push+0x71/0x90 [ppp_generic]
Feb 18 20:56:21 localhost kernel: [<c061b373>] _read_lock_bh+0x2e/0x39
Feb 18 20:56:21 localhost kernel: [<f8c5834b>] ppp_channel_push+0x71/0x90 [ppp_generic]
Feb 18 20:56:21 localhost kernel: [<f8c5834b>] ppp_channel_push+0x71/0x90 [ppp_generic]
Feb 18 20:56:21 localhost kernel: [<f8c593bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 20:56:21 localhost kernel: [<f8c592ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 20:56:21 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 20:56:21 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67
Feb 18 20:56:21 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:56:21 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:21 localhost kernel: }
Feb 18 20:56:21 localhost kernel: ... key at: [<f8c5dd20>] __key.29741+0x0/0xffffb822 [ppp_generic]
Feb 18 20:56:21 localhost kernel: -> (&ppp->wlock){-...} ops: 11 {
Feb 18 20:56:22 localhost kernel: initial-use at:
Feb 18 20:56:22 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:22 localhost kernel: [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 20:56:22 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:22 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:56:22 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 20:56:22 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:56:22 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:56:22 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:22 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:22 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:22 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:22 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:56:22 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc
Feb 18 20:56:22 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 20:56:22 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 20:56:22 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 20:56:22 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:56:22 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:22 localhost kernel: hardirq-on-W at:
Feb 18 20:56:22 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 20:56:22 localhost kernel: [<f8c57ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 20:56:22 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:22 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:56:22 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 20:56:22 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:56:22 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:56:22 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:22 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:22 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:22 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:22 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:56:22 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc
Feb 18 20:56:22 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 20:56:22 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 20:56:22 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 20:56:22 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:56:22 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:22 localhost kernel: }
Feb 18 20:56:22 localhost kernel: ... key at: [<f8c5dd08>] __key.29980+0x0/0xffffb83a [ppp_generic]
Feb 18 20:56:22 localhost kernel: -> (&ppp->rlock){-+..} ops: 5 {
Feb 18 20:56:22 localhost kernel: initial-use at:
Feb 18 20:56:22 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:23 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:56:23 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:23 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:56:23 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 20:56:23 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:56:23 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:56:23 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:23 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:23 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:23 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:23 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:56:23 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc
Feb 18 20:56:23 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 20:56:23 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 20:56:23 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 20:56:23 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:56:23 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:23 localhost kernel: in-softirq-W at:
Feb 18 20:56:23 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:56:23 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:23 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:23 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:23 localhost kernel: [<f8c59473>] ppp_input+0xab/0xef [ppp_generic]
Feb 18 20:56:23 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 20:56:23 localhost kernel: [<f8c59473>] ppp_input+0xab/0xef [ppp_generic]
Feb 18 20:56:23 localhost kernel: [<f8c59473>] ppp_input+0xab/0xef [ppp_generic]
Feb 18 20:56:23 localhost kernel: [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 20:56:23 localhost kernel: [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 20:56:23 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 20:56:23 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 20:56:23 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 20:56:23 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 20:56:23 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 20:56:23 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 20:56:23 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:23 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 20:56:23 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 20:56:23 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:23 localhost kernel: [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 20:56:23 localhost kernel: [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 20:56:23 localhost kernel: [<f8984f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 20:56:24 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 20:56:24 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 20:56:24 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:24 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:24 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:24 localhost kernel: hardirq-on-W at:
Feb 18 20:56:24 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 20:56:24 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:56:24 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:24 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:56:24 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 20:56:24 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:56:24 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:56:24 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:24 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:24 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:24 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 20:56:24 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:56:24 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc
Feb 18 20:56:24 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 20:56:24 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 20:56:24 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 20:56:24 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:56:24 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:24 localhost kernel: }
Feb 18 20:56:24 localhost kernel: ... key at: [<f8c5dd10>] __key.29979+0x0/0xffffb832 [ppp_generic]
Feb 18 20:56:24 localhost kernel: -> (&list->lock#7){.+..} ops: 31 {
Feb 18 20:56:24 localhost kernel: initial-use at:
Feb 18 20:56:24 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:24 localhost kernel: [<c046b874>] __do_fault+0x31b/0x35d
Feb 18 20:56:24 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:24 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:24 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:24 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:24 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:24 localhost kernel: [<f8c5939f>] ppp_write+0xb2/0xdb [ppp_generic]
Feb 18 20:56:24 localhost kernel: [<f8c592ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 20:56:24 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 20:56:24 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67
Feb 18 20:56:24 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:56:24 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:24 localhost kernel: in-softirq-W at:
Feb 18 20:56:24 localhost kernel: [<c0446ab0>] save_trace+0x37/0x8b
Feb 18 20:56:25 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:25 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:25 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:25 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:25 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:25 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:25 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:25 localhost kernel: [<f8c59433>] ppp_input+0x6b/0xef [ppp_generic]
Feb 18 20:56:25 localhost kernel: [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 20:56:25 localhost kernel: [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 20:56:25 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 20:56:25 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 20:56:25 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 20:56:25 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 20:56:25 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 20:56:25 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 20:56:25 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:25 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 20:56:25 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 20:56:25 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:25 localhost kernel: [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 20:56:25 localhost kernel: [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 20:56:25 localhost kernel: [<f8984f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 20:56:25 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 20:56:25 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 20:56:25 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:25 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:25 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:25 localhost kernel: }
Feb 18 20:56:25 localhost kernel: ... key at: [<f8c5dd18>] __key.21159+0x0/0xffffb82a [ppp_generic]
Feb 18 20:56:25 localhost kernel: ... acquired at:
Feb 18 20:56:25 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:56:25 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:25 localhost kernel: [<c0446ab0>] save_trace+0x37/0x8b
Feb 18 20:56:25 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:25 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:25 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:25 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:25 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:56:25 localhost kernel: [<f8c589f2>] ppp_receive_nonmp_frame+0x4e6/0x70a [ppp_generic]
Feb 18 20:56:25 localhost kernel: [<f8c59473>] ppp_input+0xab/0xef [ppp_generic]
Feb 18 20:56:26 localhost kernel: [<f8c59487>] ppp_input+0xbf/0xef [ppp_generic]
Feb 18 20:56:26 localhost kernel: [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 20:56:26 localhost kernel: [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 20:56:26 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 20:56:26 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 20:56:26 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 20:56:26 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 20:56:26 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 20:56:26 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 20:56:26 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:26 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 20:56:26 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 20:56:26 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:56:26 localhost kernel: [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 20:56:26 localhost kernel: [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 20:56:26 localhost kernel: [<f8984f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 20:56:26 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 20:56:26 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 20:56:26 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:26 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:26 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:26 localhost kernel: Feb 18 20:56:26 localhost kernel: -> (&q->lock){++..} ops: 468037 {
Feb 18 20:56:26 localhost kernel: initial-use at:
Feb 18 20:56:26 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:56:26 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:26 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:56:26 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 20:56:26 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:26 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:26 localhost kernel: [<c0619856>] wait_for_common+0x2a/0xe0
Feb 18 20:56:26 localhost kernel: [<c061b4a0>] _spin_lock_irq+0x2f/0x3a
Feb 18 20:56:26 localhost kernel: [<c0619856>] wait_for_common+0x2a/0xe0
Feb 18 20:56:26 localhost kernel: [<c0619856>] wait_for_common+0x2a/0xe0
Feb 18 20:56:26 localhost kernel: [<c043d4b2>] kthread_create+0x70/0xa4
Feb 18 20:56:26 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:26 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:56:26 localhost kernel: [<c04265bc>] migration_call+0x49/0x364
Feb 18 20:56:26 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:26 localhost kernel: [<c075e01b>] migration_init+0x19/0x40
Feb 18 20:56:26 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:27 localhost kernel: [<c074e39e>] kernel_init+0x55/0x2af
Feb 18 20:56:27 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:56:27 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:27 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:27 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:27 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:27 localhost kernel: in-hardirq-W at:
Feb 18 20:56:27 localhost kernel: [<c04480df>] check_usage_forwards+0x19/0x41
Feb 18 20:56:27 localhost kernel: [<c0446ab0>] save_trace+0x37/0x8b
Feb 18 20:56:27 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:56:27 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:27 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:27 localhost kernel: [<c04235da>] complete+0x12/0x44
Feb 18 20:56:27 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:27 localhost kernel: [<c04235da>] complete+0x12/0x44
Feb 18 20:56:27 localhost kernel: [<c04235da>] complete+0x12/0x44
Feb 18 20:56:27 localhost kernel: [<c058446e>] i8042_aux_test_irq+0x44/0x5c
Feb 18 20:56:27 localhost kernel: [<c045bcde>] handle_IRQ_event+0x13/0x3d
Feb 18 20:56:27 localhost kernel: [<c045cd38>] handle_edge_irq+0xc2/0xff
Feb 18 20:56:27 localhost kernel: [<c045cc76>] handle_edge_irq+0x0/0xff
Feb 18 20:56:27 localhost kernel: [<c04071b4>] do_IRQ+0xac/0xd4
Feb 18 20:56:27 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:27 localhost kernel: in-softirq-W at:
Feb 18 20:56:27 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:27 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:27 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:27 localhost kernel: [<c04235da>] complete+0x12/0x44
Feb 18 20:56:27 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:27 localhost kernel: [<c04235da>] complete+0x12/0x44
Feb 18 20:56:27 localhost kernel: [<c04235da>] complete+0x12/0x44
Feb 18 20:56:27 localhost kernel: [<c0430b61>] tasklet_action+0x23/0xa4
Feb 18 20:56:27 localhost kernel: [<c043b647>] __rcu_process_callbacks+0xfc/0x16e
Feb 18 20:56:27 localhost kernel: [<c043b6d1>] rcu_process_callbacks+0x18/0x30
Feb 18 20:56:27 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:56:27 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:27 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:27 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:27 localhost kernel: }
Feb 18 20:56:27 localhost kernel: ... key at: [<c0804610>] __key.17392+0x0/0x8
Feb 18 20:56:27 localhost kernel: -> (&rq->rq_lock_key){++..} ops: 138094 {
Feb 18 20:56:28 localhost kernel: initial-use at:
Feb 18 20:56:28 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:28 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:28 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:28 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:28 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:28 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:28 localhost kernel: [<c075e3c9>] sched_init+0x27d/0x292
Feb 18 20:56:28 localhost kernel: [<c074e756>] start_kernel+0x15e/0x327
Feb 18 20:56:28 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:28 localhost kernel: in-hardirq-W at:
Feb 18 20:56:28 localhost kernel: [<c04480a2>] find_usage_forwards+0x67/0x8b
Feb 18 20:56:28 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:56:28 localhost kernel: [<c0442f77>] clocksource_get_next+0xa/0x3f
Feb 18 20:56:28 localhost kernel: [<c0442fa6>] clocksource_get_next+0x39/0x3f
Feb 18 20:56:28 localhost kernel: [<c044204f>] update_wall_time+0x58d/0x71c
Feb 18 20:56:28 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:28 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:28 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:28 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:28 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:28 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:56:28 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:56:28 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:56:28 localhost kernel: [<c0407b80>] timer_interrupt+0x44/0x4a
Feb 18 20:56:28 localhost kernel: [<c045bcde>] handle_IRQ_event+0x13/0x3d
Feb 18 20:56:28 localhost kernel: [<c045d17e>] handle_level_irq+0x78/0xbe
Feb 18 20:56:28 localhost kernel: [<c045d106>] handle_level_irq+0x0/0xbe
Feb 18 20:56:28 localhost kernel: [<c04071b4>] do_IRQ+0xac/0xd4
Feb 18 20:56:28 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:28 localhost kernel: in-softirq-W at:
Feb 18 20:56:28 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 20:56:28 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 20:56:28 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:28 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:28 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:28 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:56:28 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:28 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:56:28 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:56:28 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 20:56:29 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 20:56:29 localhost kernel: [<c0430afa>] __do_softirq+0xc9/0xde
Feb 18 20:56:29 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:29 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:29 localhost kernel: }
Feb 18 20:56:29 localhost kernel: ... key at: [<c2a22488>] 0xc2a22488
Feb 18 20:56:29 localhost kernel: -> (&rq->rq_lock_key#2){++..} ops: 382046 {
Feb 18 20:56:29 localhost kernel: initial-use at:
Feb 18 20:56:29 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:29 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:29 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:29 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:29 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:29 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:29 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:56:29 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:56:29 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:56:29 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:56:29 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:56:29 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:56:29 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:29 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:29 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:29 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:29 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:56:29 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:56:29 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:29 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:29 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:56:29 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:56:29 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:29 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:29 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:29 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:29 localhost kernel: in-hardirq-W at:
Feb 18 20:56:29 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:56:29 localhost kernel: [<c040a9d6>] save_stack_trace+0x20/0x3a
Feb 18 20:56:29 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:56:29 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:29 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:29 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:30 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:30 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:30 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:30 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:30 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:56:30 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:56:30 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:56:30 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:56:30 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:56:30 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:56:30 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:30 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:56:30 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:30 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:56:30 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:56:30 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:56:30 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:30 localhost kernel: in-softirq-W at:
Feb 18 20:56:30 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:30 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 20:56:30 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 20:56:30 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:30 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:30 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:56:30 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:30 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:56:30 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:56:30 localhost kernel: [<c0433efb>] process_timeout+0x0/0x5
Feb 18 20:56:30 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 20:56:30 localhost kernel: [<c0433d59>] run_timer_softirq+0x113/0x188
Feb 18 20:56:30 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:56:30 localhost kernel: [<c0433efb>] process_timeout+0x0/0x5
Feb 18 20:56:30 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 20:56:30 localhost kernel: [<c0433efb>] process_timeout+0x0/0x5
Feb 18 20:56:30 localhost kernel: [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 20:56:30 localhost kernel: [<c0433efb>] process_timeout+0x0/0x5
Feb 18 20:56:30 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 20:56:30 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:30 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:30 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:30 localhost kernel: }
Feb 18 20:56:31 localhost kernel: ... key at: [<c2a2b488>] 0xc2a2b488
Feb 18 20:56:31 localhost kernel: -> (&rq->rq_lock_key#3){++..} ops: 250064 {
Feb 18 20:56:31 localhost kernel: initial-use at:
Feb 18 20:56:31 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:31 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:31 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:31 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:31 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:31 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:31 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:56:31 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:56:31 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:56:31 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:56:31 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:56:31 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:56:31 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:31 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:31 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:31 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:31 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:56:31 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:56:31 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:31 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:31 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:56:31 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:56:31 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:31 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:31 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:31 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:31 localhost kernel: in-hardirq-W at:
Feb 18 20:56:31 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:56:31 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:31 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:31 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:31 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:31 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:31 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:31 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:31 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:56:31 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:56:31 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:56:32 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:56:32 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:56:32 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:56:32 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:32 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:56:32 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:32 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:56:32 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:56:32 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:56:32 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:32 localhost kernel: in-softirq-W at:
Feb 18 20:56:32 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:56:32 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:32 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:32 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:56:32 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:32 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:56:32 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:56:32 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:56:32 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:56:32 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:56:32 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:32 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:32 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:32 localhost kernel: }
Feb 18 20:56:32 localhost kernel: ... key at: [<c2a34488>] 0xc2a34488
Feb 18 20:56:32 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 233519 {
Feb 18 20:56:32 localhost kernel: initial-use at:
Feb 18 20:56:32 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:32 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:32 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:32 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:32 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:32 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:32 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:56:32 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:56:32 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:56:32 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:56:32 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:56:32 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:56:32 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:32 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:33 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:33 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:33 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:56:33 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:56:33 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:33 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:33 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:56:33 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:56:33 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:33 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:33 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:33 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:33 localhost kernel: in-hardirq-W at:
Feb 18 20:56:33 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:56:33 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:33 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:33 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:33 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:33 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:33 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:33 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:33 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:56:33 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:56:33 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:56:33 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:56:33 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:56:33 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:56:33 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:33 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:56:33 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:33 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:56:33 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:56:33 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:56:33 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:33 localhost kernel: in-softirq-W at:
Feb 18 20:56:33 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:33 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:33 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:33 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:33 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:33 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:33 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:56:34 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:56:34 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:56:34 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:34 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:34 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:34 localhost kernel: }
Feb 18 20:56:34 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488
Feb 18 20:56:34 localhost kernel: ... acquired at:
Feb 18 20:56:34 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:56:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:34 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:34 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:34 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:56:34 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:34 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:56:34 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:34 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:56:34 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:56:34 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:34 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:34 localhost kernel: Feb 18 20:56:34 localhost kernel: ... acquired at:
Feb 18 20:56:34 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:56:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:34 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:34 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:34 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:56:34 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:34 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:56:34 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:34 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:56:34 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:56:34 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:34 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:35 localhost kernel: Feb 18 20:56:35 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 233519 {
Feb 18 20:56:35 localhost kernel: initial-use at:
Feb 18 20:56:35 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:35 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:35 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:35 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:35 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:35 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:35 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:56:35 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:56:35 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:56:35 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:56:35 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:56:35 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:56:35 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:35 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:35 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:35 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:35 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:56:35 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:56:35 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:35 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:35 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:56:35 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:56:35 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:35 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:35 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:35 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:35 localhost kernel: in-hardirq-W at:
Feb 18 20:56:35 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:56:35 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:35 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:35 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:35 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:35 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:35 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:35 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:35 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:56:35 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:56:35 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:56:36 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:56:36 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:56:36 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:56:36 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:36 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:56:36 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:36 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:56:36 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:56:36 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:56:36 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:36 localhost kernel: in-softirq-W at:
Feb 18 20:56:36 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:36 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:36 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:36 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:56:36 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:56:36 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:56:36 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:36 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:36 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:36 localhost kernel: }
Feb 18 20:56:36 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488
Feb 18 20:56:36 localhost kernel: ... acquired at:
Feb 18 20:56:36 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:56:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:36 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:36 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:36 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:56:36 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:36 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:56:36 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:36 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:56:36 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:56:36 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:36 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:37 localhost kernel: Feb 18 20:56:37 localhost kernel: ... acquired at:
Feb 18 20:56:37 localhost kernel: [<c0446b6a>] add_lock_to_list+0x66/0x89
Feb 18 20:56:37 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:56:37 localhost kernel: [<c0619307>] schedule+0x22e/0x6ec
Feb 18 20:56:37 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:37 localhost kernel: [<c0619307>] schedule+0x22e/0x6ec
Feb 18 20:56:37 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:37 localhost kernel: [<c0619307>] schedule+0x22e/0x6ec
Feb 18 20:56:37 localhost kernel: [<c0619307>] schedule+0x22e/0x6ec
Feb 18 20:56:37 localhost kernel: [<c0619a92>] schedule_timeout+0x13/0x8d
Feb 18 20:56:37 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 20:56:37 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:56:37 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:37 localhost kernel: [<c06198d2>] wait_for_common+0xa6/0xe0
Feb 18 20:56:37 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:56:37 localhost kernel: [<c0439c5c>] call_usermodehelper_exec+0x7a/0xbe
Feb 18 20:56:37 localhost kernel: [<c04ecbd6>] kobject_uevent_env+0x348/0x37c
Feb 18 20:56:37 localhost kernel: [<c055f4c0>] device_add+0x318/0x54c
Feb 18 20:56:37 localhost kernel: [<c055fa67>] device_create+0x7a/0x9a
Feb 18 20:56:37 localhost kernel: [<c053c13e>] tty_register_device+0xb5/0xbd
Feb 18 20:56:37 localhost kernel: [<c04efab1>] vsnprintf+0x440/0x47c
Feb 18 20:56:37 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 20:56:37 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:37 localhost kernel: [<c042c9c6>] register_console+0x4b/0x1fe
Feb 18 20:56:37 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:37 localhost kernel: [<c055aff0>] serial8250_pm+0x0/0xb2
Feb 18 20:56:37 localhost kernel: [<c0558795>] uart_change_pm+0x23/0x2b
Feb 18 20:56:37 localhost kernel: [<c055998a>] uart_add_one_port+0x29d/0x2f6
Feb 18 20:56:37 localhost kernel: [<c05623c9>] platform_device_add+0xee/0x11c
Feb 18 20:56:37 localhost kernel: [<c055ed94>] device_initialize+0x7b/0x93
Feb 18 20:56:37 localhost kernel: [<c05624ee>] platform_device_alloc+0x40/0x51
Feb 18 20:56:37 localhost kernel: [<c076a545>] serial8250_init+0xca/0x110
Feb 18 20:56:37 localhost kernel: [<c074e491>] kernel_init+0x148/0x2af
Feb 18 20:56:37 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:37 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:37 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:37 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:37 localhost kernel: Feb 18 20:56:37 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 233519 {
Feb 18 20:56:37 localhost kernel: initial-use at:
Feb 18 20:56:38 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:38 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:38 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:38 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:38 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:38 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:38 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:56:38 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:56:38 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:56:38 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:56:38 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:56:38 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:56:38 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:38 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:38 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:38 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:38 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:56:38 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:56:38 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:38 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:38 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:56:38 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:56:38 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:38 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:38 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:38 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:38 localhost kernel: in-hardirq-W at:
Feb 18 20:56:38 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:56:38 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:38 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:38 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:38 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:38 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:38 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:38 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:38 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:56:38 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:56:38 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:56:38 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:56:38 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:56:38 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:56:39 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:39 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:56:39 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:39 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:56:39 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:56:39 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:56:39 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:39 localhost kernel: in-softirq-W at:
Feb 18 20:56:39 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:39 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:39 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:39 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:39 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:39 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:39 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:56:39 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:56:39 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:56:39 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:39 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:39 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:39 localhost kernel: }
Feb 18 20:56:39 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488
Feb 18 20:56:39 localhost kernel: ... acquired at:
Feb 18 20:56:39 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:56:39 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:39 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:39 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:39 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:39 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:39 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:56:39 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:56:39 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:39 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:56:39 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:56:39 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:56:39 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:56:39 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:39 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:39 localhost kernel: Feb 18 20:56:39 localhost kernel: -> (&rq->rq_lock_key#3){++..} ops: 250064 {
Feb 18 20:56:39 localhost kernel: initial-use at:
Feb 18 20:56:40 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:40 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:40 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:40 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:40 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:40 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:40 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:56:40 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:56:40 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:56:40 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:56:40 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:56:40 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:56:40 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:40 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:40 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:40 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:40 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:56:40 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:56:40 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:40 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:40 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:56:40 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:56:40 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:40 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:56:40 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:56:40 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:40 localhost kernel: in-hardirq-W at:
Feb 18 20:56:40 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:56:40 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:40 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:56:40 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:40 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:40 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:40 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:40 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:56:40 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:56:40 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:56:40 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:56:40 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:56:40 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:56:40 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:56:41 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:41 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:56:41 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:56:41 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:56:41 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:56:41 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:56:41 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:41 localhost kernel: in-softirq-W at:
Feb 18 20:56:41 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:56:41 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:56:41 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:41 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:56:41 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:56:41 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:56:41 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:56:41 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:56:41 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:56:41 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:56:41 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:56:41 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:56:41 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:56:41 localhost kernel: }
Feb 18 20:56:41 localhost kernel: ... key at: [<c2a34488>] 0xc2a34488
Feb 18 20:56:41 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 233519 {
Feb 18 20:56:41 localhost kernel: initial-use at:
Feb 18 20:56:41 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:56:41 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:56:41 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:41 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:56:41 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:41 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:56:41 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:56:41 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:56:41 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:56:41 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:56:41 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:56:41 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:56:41 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:42 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:56:43 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:56:48 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:56:53 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:56:58 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:03 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:09 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:13 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:57:18 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:57:21 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:25 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:26 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:27 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:27 localhost kernel: in-hardirq-W at:
Feb 18 20:57:27 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:57:27 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:27 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:27 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:27 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:27 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:27 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:27 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:27 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:57:27 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:57:27 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:57:27 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:57:27 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:57:27 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:57:27 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:27 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:57:27 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:27 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:57:27 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:57:27 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:57:27 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:27 localhost kernel: in-softirq-W at:
Feb 18 20:57:27 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:27 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:27 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:27 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:27 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:57:28 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:57:28 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:57:28 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:28 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:28 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:28 localhost kernel: }
Feb 18 20:57:28 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488
Feb 18 20:57:28 localhost kernel: ... acquired at:
Feb 18 20:57:28 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:57:28 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:28 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:57:28 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:28 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:57:28 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:28 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:28 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:28 localhost kernel: Feb 18 20:57:28 localhost kernel: ... acquired at:
Feb 18 20:57:28 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:28 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:57:28 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:28 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:57:28 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:28 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:57:28 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:28 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:29 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:29 localhost kernel: Feb 18 20:57:29 localhost kernel: ... acquired at:
Feb 18 20:57:29 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:29 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:29 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:29 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:29 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:29 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:29 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:29 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:29 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 20:57:29 localhost kernel: [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 20:57:29 localhost kernel: [<c04235fe>] complete+0x36/0x44
Feb 18 20:57:29 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:29 localhost kernel: [<c043d502>] kthread+0x1c/0x5e
Feb 18 20:57:29 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:29 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:29 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:29 localhost kernel: Feb 18 20:57:29 localhost kernel: -> (&rq->rq_lock_key#2){++..} ops: 382046 {
Feb 18 20:57:29 localhost kernel: initial-use at:
Feb 18 20:57:29 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:29 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:29 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:29 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:29 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:29 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:29 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:57:29 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:57:29 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:57:29 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:57:29 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:57:29 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:57:29 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:29 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:29 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:29 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:29 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:57:29 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:29 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:29 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:30 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:57:30 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:57:30 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:30 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:30 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:30 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:30 localhost kernel: in-hardirq-W at:
Feb 18 20:57:30 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:57:30 localhost kernel: [<c040a9d6>] save_stack_trace+0x20/0x3a
Feb 18 20:57:30 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:57:30 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:30 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:30 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:30 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:30 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:30 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:30 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:30 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:57:30 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:57:30 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:57:30 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:57:30 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:57:30 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:57:30 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:30 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:57:30 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:30 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:57:30 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:57:30 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:57:30 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:30 localhost kernel: in-softirq-W at:
Feb 18 20:57:30 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:30 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 20:57:30 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 20:57:30 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:30 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:30 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:30 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:30 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:30 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:30 localhost kernel: [<c0433efb>] process_timeout+0x0/0x5
Feb 18 20:57:31 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 20:57:31 localhost kernel: [<c0433d59>] run_timer_softirq+0x113/0x188
Feb 18 20:57:31 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:31 localhost kernel: [<c0433efb>] process_timeout+0x0/0x5
Feb 18 20:57:31 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 20:57:31 localhost kernel: [<c0433efb>] process_timeout+0x0/0x5
Feb 18 20:57:31 localhost kernel: [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 20:57:31 localhost kernel: [<c0433efb>] process_timeout+0x0/0x5
Feb 18 20:57:31 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 20:57:31 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:31 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:31 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:31 localhost kernel: }
Feb 18 20:57:31 localhost kernel: ... key at: [<c2a2b488>] 0xc2a2b488
Feb 18 20:57:31 localhost kernel: -> (&rq->rq_lock_key#3){++..} ops: 250064 {
Feb 18 20:57:31 localhost kernel: initial-use at:
Feb 18 20:57:31 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:31 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:31 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:31 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:31 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:31 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:31 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:57:31 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:57:31 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:57:31 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:57:31 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:57:31 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:57:31 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:31 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:31 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:31 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:31 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:57:31 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:31 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:31 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:31 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:57:31 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:57:31 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:31 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:31 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:32 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:32 localhost kernel: in-hardirq-W at:
Feb 18 20:57:32 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:57:32 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:32 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:32 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:32 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:32 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:32 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:32 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:32 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:57:32 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:57:32 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:57:32 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:57:32 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:57:32 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:57:32 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:32 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:57:32 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:32 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:57:32 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:57:32 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:57:32 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:32 localhost kernel: in-softirq-W at:
Feb 18 20:57:32 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:57:32 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:32 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:32 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:57:32 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:32 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:57:32 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:57:32 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:57:32 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:57:32 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:57:32 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:32 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:32 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:32 localhost kernel: }
Feb 18 20:57:32 localhost kernel: ... key at: [<c2a34488>] 0xc2a34488
Feb 18 20:57:32 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 233519 {
Feb 18 20:57:32 localhost kernel: initial-use at:
Feb 18 20:57:33 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:33 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:33 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:33 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:33 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:33 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:33 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:57:33 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:57:33 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:57:33 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:57:33 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:57:33 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:57:33 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:33 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:33 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:33 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:33 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:57:33 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:33 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:33 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:33 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:57:33 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:57:33 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:33 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:33 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:33 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:33 localhost kernel: in-hardirq-W at:
Feb 18 20:57:33 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:57:33 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:33 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:33 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:33 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:33 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:33 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:33 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:33 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:57:33 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:57:33 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:57:33 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:57:33 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:57:33 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:57:33 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:33 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:57:34 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:34 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:57:34 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:57:34 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:57:34 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:34 localhost kernel: in-softirq-W at:
Feb 18 20:57:34 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:34 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:34 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:34 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:57:34 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:57:34 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:57:34 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:34 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:34 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:34 localhost kernel: }
Feb 18 20:57:34 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488
Feb 18 20:57:34 localhost kernel: ... acquired at:
Feb 18 20:57:34 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:34 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:34 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:34 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:57:34 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:34 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:57:34 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:34 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:57:34 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:34 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:34 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:34 localhost kernel: Feb 18 20:57:34 localhost kernel: ... acquired at:
Feb 18 20:57:34 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:34 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:34 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:35 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:35 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:35 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:35 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:57:35 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:35 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:57:35 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:35 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:57:35 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:35 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:35 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:35 localhost kernel: Feb 18 20:57:35 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 233519 {
Feb 18 20:57:35 localhost kernel: initial-use at:
Feb 18 20:57:35 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:35 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:35 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:35 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:35 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:35 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:35 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:57:35 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:57:35 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:57:35 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:57:35 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:57:35 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:57:35 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:35 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:35 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:35 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:35 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:57:35 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:35 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:35 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:35 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:57:35 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:57:35 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:35 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:35 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:35 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:35 localhost kernel: in-hardirq-W at:
Feb 18 20:57:35 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:57:36 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:36 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:36 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:36 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:36 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:36 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:36 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:36 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:57:36 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:57:36 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:57:36 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:57:36 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:57:36 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:57:36 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:36 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:57:36 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:36 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:57:36 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:57:36 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:57:36 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:36 localhost kernel: in-softirq-W at:
Feb 18 20:57:36 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:36 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:36 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:36 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:57:36 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:57:36 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:57:36 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:36 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:36 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:36 localhost kernel: }
Feb 18 20:57:36 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488
Feb 18 20:57:36 localhost kernel: ... acquired at:
Feb 18 20:57:36 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:36 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:36 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:37 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:37 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:37 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:37 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:57:37 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:37 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:57:37 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:37 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:57:37 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:37 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:37 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:37 localhost kernel: Feb 18 20:57:37 localhost kernel: ... acquired at:
Feb 18 20:57:37 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:37 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:37 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:37 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:37 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:37 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:37 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:37 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 20:57:37 localhost kernel: [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 20:57:37 localhost kernel: [<c04235fe>] complete+0x36/0x44
Feb 18 20:57:37 localhost kernel: [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 20:57:37 localhost kernel: [<c043d502>] kthread+0x1c/0x5e
Feb 18 20:57:37 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:37 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:37 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:37 localhost kernel: Feb 18 20:57:37 localhost kernel: -> (&rq->rq_lock_key#3){++..} ops: 250064 {
Feb 18 20:57:37 localhost kernel: initial-use at:
Feb 18 20:57:37 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:37 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:37 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:37 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:37 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:37 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:37 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:57:37 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:57:37 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:57:38 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:57:38 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:57:38 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:57:38 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:38 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:38 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:38 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:38 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:57:38 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:38 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:38 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:38 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:57:38 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:57:38 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:38 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:38 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:38 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:38 localhost kernel: in-hardirq-W at:
Feb 18 20:57:38 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:57:38 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:38 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:38 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:38 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:38 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:38 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:38 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:38 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:57:38 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:57:38 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:57:38 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:57:38 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:57:38 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:57:38 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:38 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:57:38 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:38 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:57:38 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:57:38 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:57:38 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:38 localhost kernel: in-softirq-W at:
Feb 18 20:57:38 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 20:57:38 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:39 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:39 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:57:39 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:39 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:57:39 localhost kernel: [<c042313a>] double_rq_lock+0x29/0x40
Feb 18 20:57:39 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:57:39 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:57:39 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:57:39 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:39 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:39 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:39 localhost kernel: }
Feb 18 20:57:39 localhost kernel: ... key at: [<c2a34488>] 0xc2a34488
Feb 18 20:57:39 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 233519 {
Feb 18 20:57:39 localhost kernel: initial-use at:
Feb 18 20:57:39 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:39 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:39 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:39 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:39 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:39 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:39 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:57:39 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:57:39 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:57:39 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:57:39 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:57:39 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:57:39 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:39 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:39 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:39 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:39 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:57:39 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:39 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:39 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:39 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:57:39 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:57:39 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:39 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:39 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:39 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:40 localhost kernel: in-hardirq-W at:
Feb 18 20:57:40 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:57:40 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:40 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:40 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:40 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:40 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:40 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:40 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:40 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:57:40 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:57:40 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:57:40 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:57:40 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:57:40 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:57:40 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:40 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:57:40 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:40 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:57:40 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:57:40 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:57:40 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:40 localhost kernel: in-softirq-W at:
Feb 18 20:57:40 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:40 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:40 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:40 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:40 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:40 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:40 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:57:40 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:57:40 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:57:40 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:40 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:40 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:40 localhost kernel: }
Feb 18 20:57:40 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488
Feb 18 20:57:40 localhost kernel: ... acquired at:
Feb 18 20:57:40 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:40 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:40 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:40 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:41 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:41 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:41 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:41 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 20:57:41 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:41 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 20:57:41 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:41 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:57:41 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:41 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:41 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:41 localhost kernel: Feb 18 20:57:41 localhost kernel: ... acquired at:
Feb 18 20:57:41 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:41 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:41 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:41 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:41 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:41 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:41 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:41 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 20:57:41 localhost kernel: [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 20:57:41 localhost kernel: [<c04235fe>] complete+0x36/0x44
Feb 18 20:57:41 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:41 localhost kernel: [<c0428996>] migration_thread+0x186/0x1d2
Feb 18 20:57:41 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:41 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:57:41 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:41 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:41 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:41 localhost kernel: Feb 18 20:57:41 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 233519 {
Feb 18 20:57:41 localhost kernel: initial-use at:
Feb 18 20:57:41 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:41 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:41 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:41 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:41 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:41 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 20:57:41 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 20:57:41 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 20:57:42 localhost kernel: [<c041a8a3>] setup_local_APIC+0x9e/0x28f
Feb 18 20:57:42 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 20:57:42 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 20:57:42 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 20:57:42 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:42 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:42 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:42 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 20:57:42 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 20:57:42 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 20:57:42 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:42 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:42 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 20:57:42 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 20:57:42 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:42 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 20:57:42 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:42 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:42 localhost kernel: in-hardirq-W at:
Feb 18 20:57:42 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 20:57:42 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:42 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:42 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:42 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:42 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:42 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:42 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 20:57:42 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 20:57:42 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 20:57:42 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 20:57:42 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 20:57:42 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 20:57:42 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 20:57:42 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:42 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 20:57:42 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 20:57:42 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 20:57:42 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 20:57:42 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 20:57:42 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:42 localhost kernel: in-softirq-W at:
Feb 18 20:57:43 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:43 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:43 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:43 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:43 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:43 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 20:57:43 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 20:57:43 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 20:57:43 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 20:57:43 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:43 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:43 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:43 localhost kernel: }
Feb 18 20:57:43 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488
Feb 18 20:57:43 localhost kernel: ... acquired at:
Feb 18 20:57:43 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:43 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:43 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:43 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:43 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 20:57:43 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:43 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 20:57:43 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 20:57:43 localhost kernel: [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 20:57:43 localhost kernel: [<c04235fe>] complete+0x36/0x44
Feb 18 20:57:43 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:43 localhost kernel: [<c0428996>] migration_thread+0x186/0x1d2
Feb 18 20:57:43 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 20:57:43 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 20:57:43 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 20:57:43 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 20:57:43 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:43 localhost kernel: Feb 18 20:57:43 localhost kernel: ... acquired at:
Feb 18 20:57:43 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:43 localhost kernel: [<c042366f>] __wake_up+0x18/0x42
Feb 18 20:57:43 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:43 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:57:43 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:43 localhost kernel: [<c042366f>] __wake_up+0x18/0x42
Feb 18 20:57:43 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:44 localhost kernel: [<c042366f>] __wake_up+0x18/0x42
Feb 18 20:57:44 localhost kernel: [<c042366f>] __wake_up+0x18/0x42
Feb 18 20:57:44 localhost kernel: [<f8c58a2b>] ppp_receive_nonmp_frame+0x51f/0x70a [ppp_generic]
Feb 18 20:57:44 localhost kernel: [<f8c59487>] ppp_input+0xbf/0xef [ppp_generic]
Feb 18 20:57:44 localhost kernel: [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 20:57:44 localhost kernel: [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 20:57:44 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 20:57:44 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 20:57:44 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 20:57:44 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 20:57:44 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 20:57:44 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 20:57:44 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:57:44 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 20:57:44 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 20:57:44 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:57:44 localhost kernel: [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 20:57:44 localhost kernel: [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 20:57:44 localhost kernel: [<f8984f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 20:57:44 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 20:57:44 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 20:57:44 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:44 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:44 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:44 localhost kernel: Feb 18 20:57:44 localhost kernel: ... acquired at:
Feb 18 20:57:44 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:44 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:57:44 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:44 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:57:44 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 20:57:44 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:57:44 localhost kernel: [<f8c57bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 20:57:44 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 20:57:44 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:44 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:44 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 20:57:44 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:57:44 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc
Feb 18 20:57:44 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 20:57:44 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 20:57:45 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 20:57:45 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:57:45 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:45 localhost kernel: Feb 18 20:57:45 localhost kernel: -> (&list->lock#7){.+..} ops: 31 {
Feb 18 20:57:45 localhost kernel: initial-use at:
Feb 18 20:57:45 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:45 localhost kernel: [<c046b874>] __do_fault+0x31b/0x35d
Feb 18 20:57:45 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:45 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:57:45 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:45 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:57:45 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:57:45 localhost kernel: [<f8c5939f>] ppp_write+0xb2/0xdb [ppp_generic]
Feb 18 20:57:45 localhost kernel: [<f8c592ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 20:57:45 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 20:57:45 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67
Feb 18 20:57:45 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:57:45 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:45 localhost kernel: in-softirq-W at:
Feb 18 20:57:45 localhost kernel: [<c0446ab0>] save_trace+0x37/0x8b
Feb 18 20:57:45 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 20:57:45 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:45 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:45 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:57:45 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:45 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:57:45 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 20:57:45 localhost kernel: [<f8c59433>] ppp_input+0x6b/0xef [ppp_generic]
Feb 18 20:57:45 localhost kernel: [<f8c60263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 20:57:45 localhost kernel: [<f8c6037a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 20:57:45 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 20:57:45 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 20:57:45 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 20:57:45 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 20:57:45 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 20:57:45 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 20:57:45 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:57:45 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 20:57:45 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 20:57:45 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 20:57:45 localhost kernel: [<f89877c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 20:57:45 localhost kernel: [<f8987453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 20:57:46 localhost kernel: [<f8984f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 20:57:46 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 20:57:46 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 20:57:46 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 20:57:46 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 20:57:46 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:46 localhost kernel: }
Feb 18 20:57:46 localhost kernel: ... key at: [<f8c5dd18>] __key.21159+0x0/0xffffb82a [ppp_generic]
Feb 18 20:57:46 localhost kernel: ... acquired at:
Feb 18 20:57:46 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:46 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 20:57:46 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:46 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:46 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 20:57:46 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 20:57:46 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 20:57:46 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 20:57:46 localhost kernel: [<f8c57424>] ppp_xmit_process+0x529/0x5a1 [ppp_generic]
Feb 18 20:57:46 localhost kernel: [<f8c5834b>] ppp_channel_push+0x71/0x90 [ppp_generic]
Feb 18 20:57:46 localhost kernel: [<c061b373>] _read_lock_bh+0x2e/0x39
Feb 18 20:57:46 localhost kernel: [<f8c5835a>] ppp_channel_push+0x80/0x90 [ppp_generic]
Feb 18 20:57:46 localhost kernel: [<f8c593bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 20:57:46 localhost kernel: [<f8c592ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 20:57:46 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 20:57:46 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67
Feb 18 20:57:46 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:57:46 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:46 localhost kernel: Feb 18 20:57:46 localhost kernel: ... acquired at:
Feb 18 20:57:46 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 20:57:46 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:57:46 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:46 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:57:46 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 20:57:46 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:57:46 localhost kernel: [<f8c57bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 20:57:46 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 20:57:46 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:46 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:46 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 20:57:46 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:57:47 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc
Feb 18 20:57:47 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 20:57:47 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 20:57:47 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 20:57:47 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:57:47 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:47 localhost kernel: Feb 18 20:57:47 localhost kernel: Feb 18 20:57:47 localhost kernel: the soft-irq-unsafe lock's dependencies:
Feb 18 20:57:47 localhost kernel: -> (&sk->sk_dst_lock){----} ops: 284 {
Feb 18 20:57:47 localhost kernel: initial-use at:
Feb 18 20:57:47 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 20:57:47 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 20:57:47 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 20:57:47 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 20:57:47 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:47 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:47 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 20:57:47 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:47 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:47 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 20:57:47 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 20:57:47 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 20:57:47 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 20:57:47 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:47 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 20:57:47 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:47 localhost kernel: softirq-on-W at:
Feb 18 20:57:47 localhost kernel: [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 18 20:57:47 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 20:57:47 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 20:57:47 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 20:57:47 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:47 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:47 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 20:57:47 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:47 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:47 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 20:57:47 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 20:57:47 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 20:57:47 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 20:57:47 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:48 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 20:57:48 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:48 localhost kernel: hardirq-on-W at:
Feb 18 20:57:48 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 20:57:48 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 20:57:48 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 20:57:48 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 20:57:48 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:48 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:48 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 20:57:48 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:48 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 20:57:48 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 20:57:48 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 20:57:48 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 20:57:48 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 20:57:48 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:48 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 20:57:48 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:48 localhost kernel: softirq-on-R at:
Feb 18 20:57:48 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:48 localhost kernel: [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 18 20:57:48 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:48 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:48 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 20:57:48 localhost kernel: [<c061b3a7>] _read_lock+0x29/0x34
Feb 18 20:57:48 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 20:57:48 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 20:57:48 localhost kernel: [<f8a6a9b1>] ipv6_chk_addr+0x8a/0x96 [ipv6]
Feb 18 20:57:48 localhost kernel: [<f8a649b0>] ip6_sk_dst_lookup+0x2d/0x17a [ipv6]
Feb 18 20:57:48 localhost kernel: [<c0489d61>] __pollwait+0x0/0xac
Feb 18 20:57:48 localhost kernel: [<f8a75e6d>] udpv6_sendmsg+0x4c1/0x8a1 [ipv6]
Feb 18 20:57:48 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:57:48 localhost last message repeated 2 times
Feb 18 20:57:48 localhost kernel: [<c05f5fc7>] inet_sendmsg+0x3b/0x45
Feb 18 20:57:48 localhost kernel: [<c05ab8cd>] sock_sendmsg+0xc9/0xe4
Feb 18 20:57:48 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 20:57:48 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:48 localhost kernel: [<c04f05f4>] copy_from_user+0x32/0x5e
Feb 18 20:57:49 localhost kernel: [<c04f05f4>] copy_from_user+0x32/0x5e
Feb 18 20:57:49 localhost kernel: [<c05aba7a>] sys_sendmsg+0x192/0x1f7
Feb 18 20:57:49 localhost kernel: [<c048f764>] file_update_time+0x22/0x6a
Feb 18 20:57:49 localhost kernel: [<c0483d90>] pipe_write+0x392/0x3ef
Feb 18 20:57:49 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:49 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 20:57:49 localhost kernel: [<c05acd2b>] sys_socketcall+0x240/0x261
Feb 18 20:57:49 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:57:49 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:49 localhost kernel: hardirq-on-R at:
Feb 18 20:57:49 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:49 localhost kernel: [<c0449001>] __lock_acquire+0x446/0xbf1
Feb 18 20:57:49 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:49 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:49 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 20:57:49 localhost kernel: [<c061b3a7>] _read_lock+0x29/0x34
Feb 18 20:57:49 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 20:57:49 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 20:57:49 localhost kernel: [<f8a6a9b1>] ipv6_chk_addr+0x8a/0x96 [ipv6]
Feb 18 20:57:49 localhost kernel: [<f8a649b0>] ip6_sk_dst_lookup+0x2d/0x17a [ipv6]
Feb 18 20:57:49 localhost kernel: [<c0489d61>] __pollwait+0x0/0xac
Feb 18 20:57:49 localhost kernel: [<f8a75e6d>] udpv6_sendmsg+0x4c1/0x8a1 [ipv6]
Feb 18 20:57:49 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8
Feb 18 20:57:49 localhost last message repeated 2 times
Feb 18 20:57:49 localhost kernel: [<c05f5fc7>] inet_sendmsg+0x3b/0x45
Feb 18 20:57:49 localhost kernel: [<c05ab8cd>] sock_sendmsg+0xc9/0xe4
Feb 18 20:57:49 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 20:57:49 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:49 localhost kernel: [<c04f05f4>] copy_from_user+0x32/0x5e
Feb 18 20:57:49 localhost kernel: [<c04f05f4>] copy_from_user+0x32/0x5e
Feb 18 20:57:49 localhost kernel: [<c05aba7a>] sys_sendmsg+0x192/0x1f7
Feb 18 20:57:49 localhost kernel: [<c048f764>] file_update_time+0x22/0x6a
Feb 18 20:57:49 localhost kernel: [<c0483d90>] pipe_write+0x392/0x3ef
Feb 18 20:57:49 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 20:57:49 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 20:57:49 localhost kernel: [<c05acd2b>] sys_socketcall+0x240/0x261
Feb 18 20:57:49 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:57:49 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 20:57:49 localhost kernel: }
Feb 18 20:57:49 localhost kernel: ... key at: [<c0a3dc28>] __key.35996+0x0/0x8
Feb 18 20:57:49 localhost kernel: Feb 18 20:57:49 localhost kernel: stack backtrace:
Feb 18 20:57:50 localhost kernel: Pid: 3241, comm: pppd Not tainted 2.6.24.2 #1
Feb 18 20:57:50 localhost kernel: [<c0448b1e>] check_usage+0x24e/0x258
Feb 18 20:57:50 localhost kernel: [<c0449555>] __lock_acquire+0x99a/0xbf1
Feb 18 20:57:50 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 20:57:50 localhost kernel: [<f8c566db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 20:57:50 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 20:57:50 localhost kernel: [<f8c566db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 20:57:50 localhost kernel: [<f8c566db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 20:57:50 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 20:57:50 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 20:57:50 localhost kernel: [<f8c573f1>] ppp_xmit_process+0x4f6/0x5a1 [ppp_generic]
Feb 18 20:57:50 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 20:57:50 localhost kernel: [<f8c593b4>] ppp_write+0xc7/0xdb [ppp_generic]
Feb 18 20:57:50 localhost kernel: [<f8c592ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 20:57:50 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 20:57:50 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67
Feb 18 20:57:50 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 20:57:50 localhost kernel: =======================Quoted 27 lines
Thanks,
Jarek P.On Tue, Feb 12, 2008 at 10:58:21AM +0000, James Chapman wrote:
...Quoted 7 lines
Here is a trace from when we had
_bhlocks.Feb 5 16:26:32 ====================================================== Feb 5 16:26:32 [ INFO: soft-safe -> soft-unsafe lock order detected ] Feb 5 16:26:32 2.6.24-core2 #1 Feb 5 16:26:32 ------------------------------------------------------ Feb 5 16:26:32 pppd/3224 [HC0[0]:SC0[2]:HE1:SE0] is trying to acquire:...
Quoted 13 lines
Feb 5 16:26:32 [<f8d7b84d>] e1000_clean+0x5d/0x290 [e1000] Feb 5 16:26:32 [<c039d580>] net_rx_action+0x1a0/0x2a0 Feb 5 16:26:32 [<c039d43f>] net_rx_action+0x5f/0x2a0 Feb 5 16:26:32 [<c0131e72>] __do_softirq+0x92/0x120 Feb 5 16:26:32 [<c0131f78>] do_softirq+0x78/0x80 Feb 5 16:26:32 [<c010b15a>] do_IRQ+0x4a/0xa0 Feb 5 16:26:32 [<c0127af0>] finish_task_switch+0x0/0xc0 Feb 5 16:26:32 [<c0108dcc>] common_interrupt+0x24/0x34 Feb 5 16:26:32 [<c0108dd6>] common_interrupt+0x2e/0x34 Feb 5 16:26:32 [<c01062d6>] mwait_idle_with_hints+0x46/0x60 Feb 5 16:26:32 [<c0106550>] mwait_idle+0x0/0x20 Feb 5 16:26:32 [<c0106694>] cpu_idle+0x74/0xe0 Feb 5 16:26:32 [<c0536a9a>] start_kernel+0x30a/0x3a0
Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
Quoted 9 lines · James Chapman
Quoted 4 lines · Jarek Poplawski
Hi,
It seems, this nice report is still uncomplete: could you check if there could have been something more yet?
Unfortunately the ISP's syslog stops. But I've been able to borrow two Quad Xeon boxes and have reproduced the problem.
Quoted 7 lines
Here's a new version of the patch. The patch avoids disabling irqs and fixes the
sk_dst_get()usage that DaveM mentioned. But even with this patch, lockdep still complains if hundreds of ppp sessions are inserted into a tunnel as rapidly as possible (lockdep trace is below). I can stop these errors by wrapping the call toppp_input()inpppol2tp_recv_dequeue_skb()with local_irq_save/restore. What is a better fix?
Hmm... This is a really long report and quite a bit different from the previous one. I need some time for this. BTW: you sent before a lockdep report with hlist_lock problem. I think this could be fixed in some independent patch to make this all more readable. Are all the other changes in this current patch only because of this or previous lockdep report or for some other reasons (or reports) yet?
Regards,
Jarek P.
Quoted 17 lines · Jarek Poplawski
Quoted 8 lines · James Chapman
Quoted 4 lines · Jarek Poplawski
Hi,
It seems, this nice report is still uncomplete: could you check if there could have been something more yet?
Unfortunately the ISP's syslog stops. But I've been able to borrow two Quad Xeon boxes and have reproduced the problem.
Quoted 7 lines
Here's a new version of the patch. The patch avoids disabling irqs and fixes the
sk_dst_get()usage that DaveM mentioned. But even with this patch, lockdep still complains if hundreds of ppp sessions are inserted into a tunnel as rapidly as possible (lockdep trace is below). I can stop these errors by wrapping the call toppp_input()inpppol2tp_recv_dequeue_skb()with local_irq_save/restore. What is a better fix?
Quoted 6 lines
Hmm... This is a really long report and quite a bit different from the previous one. I need some time for this. BTW: you sent before a lockdep report with
hlist_lockproblem. I think this could be fixed in some independent patch to make this all more readable. Are all the other changes in this current patch only because of this or previous lockdep report or for some other reasons (or reports) yet?
As I mentioned in my reply to davem, modifying the pppol2tp driver as described in the patch I sent made the original lockdep problems go away.
Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
David Miller
maintainerFrom: James Chapman <[email protected]>
Date: Mon, 18 Feb 2008 22:09:24 +0000
Quoted 7 lines
Here's a new version of the patch. The patch avoids disabling irqs and fixes the
sk_dst_get()usage that DaveM mentioned. But even with this patch, lockdep still complains if hundreds of ppp sessions are inserted into a tunnel as rapidly as possible (lockdep trace is below). I can stop these errors by wrapping the call toppp_input()inpppol2tp_recv_dequeue_skb()with local_irq_save/restore. What is a better fix?
Firstly, let's fix one thing at a time. Leave the sk_dst_get() thing alone until we can prove that it's part of the lockdep traces.
Next, I can't see why ppp_input() needs to be invoked with interrupts disabled. There are many other things that invoke that in software interrupt context, such as pppoe.
Please provide the lockdep traces without the ppp_input() IRQ disabling so this can be properly analyzed.
Quoted 11 lines · David Miller
From: James Chapman <[email protected]>
Date: Mon, 18 Feb 2008 22:09:24 +0000Quoted 7 lines
Here's a new version of the patch. The patch avoids disabling irqs and fixes the
sk_dst_get()usage that DaveM mentioned. But even with this patch, lockdep still complains if hundreds of ppp sessions are inserted into a tunnel as rapidly as possible (lockdep trace is below). I can stop these errors by wrapping the call toppp_input()inpppol2tp_recv_dequeue_skb()with local_irq_save/restore. What is a better fix?
Quoted 3 lines
Firstly, let's fix one thing at a time. Leave the
sk_dst_get()thing alone until we can prove that it's part of the lockdep traces.
In reproducing the problem, I obtained several lockdep traces that implicated sk_dst_get(). I changed the code to use __sk_dst_check() as you suggested and they went away. At that point, I was hopeful the locking issues were fixed. But after several minutes of creating/deleting hundreds of ppp sessions, lockdep dumped another error. It is that error that I posted yesterday.
Quoted 3 lines
Next, I can't see why
ppp_input()needs to be invoked with interrupts disabled. There are many other things that invoke that in software interrupt context, such as pppoe.
I agree. I'm seeking advice on what the underlying cause is of this new trace.
Quoted 2 lines
Please provide the lockdep traces without the
ppp_input()IRQ disabling so this can be properly analyzed.
The trace _was_ without ppp_input IRQ disabling. The trace doesn't occur if I disable IRQs around the ppp_input() call. The patch I sent showed the changes I made before running the tests that created the new lockdep trace. I'm sorry this wasn't clear.
Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
Quoted 12 lines · James Chapman
Quoted 11 lines · David Miller
From: James Chapman <[email protected]>
Date: Mon, 18 Feb 2008 22:09:24 +0000Quoted 7 lines
Here's a new version of the patch. The patch avoids disabling irqs and fixes the
sk_dst_get()usage that DaveM mentioned. But even with this patch, lockdep still complains if hundreds of ppp sessions are inserted into a tunnel as rapidly as possible (lockdep trace is below). I can stop these errors by wrapping the call toppp_input()inpppol2tp_recv_dequeue_skb()with local_irq_save/restore. What is a better fix?
Quoted 6 lines
Quoted 3 lines
Firstly, let's fix one thing at a time. Leave the
sk_dst_get()thing alone until we can prove that it's part of the lockdep traces.In reproducing the problem, I obtained several lockdep traces that implicated
sk_dst_get().
As a matter of fact I missed just that kind information on previous lockdep report, so if you could send them too this should be still helpful.
...
Quoted 2 lines
I agree. I'm seeking advice on what the underlying cause is of this new trace.
IMHO, just like I wrote earlier, the main problem is in ppp_generic(), especially ppp_connect_channel(), where main tx & rx locks are used. I didn't know enough about this sk_dst_lock traces yet. I hope I could help with this, but after these changes I need some time to figure this out again.
Jarek P.
On Tue, Feb 19, 2008 at 10:30:47AM +0000, Jarek Poplawski wrote:
...
Quoted 1 line
IMHO, just like I wrote earlier, the main problem is in
ppp_generic(),
...or maybe ppp_generic.c? Whatever...
Jarek P.
Quoted 12 lines · Jarek Poplawski
Quoted 11 lines · James Chapman
Quoted 10 lines · David Miller
From: James Chapman <[email protected]>
Date: Mon, 18 Feb 2008 22:09:24 +0000Quoted 7 lines
Here's a new version of the patch. The patch avoids disabling irqs and fixes the
sk_dst_get()usage that DaveM mentioned. But even with this patch, lockdep still complains if hundreds of ppp sessions are inserted into a tunnel as rapidly as possible (lockdep trace is below). I can stop these errors by wrapping the call toppp_input()inpppol2tp_recv_dequeue_skb()with local_irq_save/restore. What is a better fix?
Quoted 8 lines
Quoted 4 lines
Quoted 3 lines
Firstly, let's fix one thing at a time. Leave the
sk_dst_get()thing alone until we can prove that it's part of the lockdep traces.In reproducing the problem, I obtained several lockdep traces that implicated
sk_dst_get().As a matter of fact I missed just that kind information on previous lockdep report, so if you could send them too this should be still helpful.
Here you go.
Just to be clear, the initial lockdep report was with non-bh locks. Yesterday's report was with _bh locks and the sk_dst_get() changes. This (older) report is with _bh locks but without the sk_dst_get() changes.
Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
Feb 18 17:25:42 localhost kernel: ======================================================
Feb 18 17:25:42 localhost kernel: [ INFO: soft-safe -> soft-unsafe lock order detected ]
Feb 18 17:25:42 localhost kernel: 2.6.24.2 #1
Feb 18 17:25:42 localhost kernel: ------------------------------------------------------
Feb 18 17:25:42 localhost kernel: pppd/13863 [HC0[0]:SC0[1]:HE1:SE0] is trying to acquire:
Feb 18 17:25:42 localhost kernel: (&sk->sk_dst_lock){----}, at: [<f8bbf73e>] pppol2tp_xmit+0x3a5/0x4a2 [pppol2tp]
Feb 18 17:25:42 localhost kernel:
Feb 18 17:25:42 localhost kernel: and this task is already holding:
Feb 18 17:25:42 localhost kernel: (&pch->downl){-...}, at: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:25:42 localhost kernel: which would create a new lock dependency:
Feb 18 17:25:42 localhost pppd[13863]: Connection terminated.
Feb 18 17:25:42 localhost kernel: (&pch->downl){-...} -> (&sk->sk_dst_lock){----}
Feb 18 17:25:42 localhost pppd[13863]: Exit.
Feb 18 17:25:42 localhost kernel:
Feb 18 17:25:42 localhost kernel: but this new dependency connects a soft-irq-safe lock:
Feb 18 17:25:42 localhost kernel: (&pch->upl){-.-+}
Feb 18 17:25:42 localhost kernel: ... which became soft-irq-safe at:
Feb 18 17:25:43 localhost kernel: [<c0448013>] check_usage_backwards+0x19/0x41
Feb 18 17:25:43 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:43 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:43 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:43 localhost kernel: [<f8bb840d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 17:25:43 localhost kernel: [<c061b373>] _read_lock_bh+0x2e/0x39
Feb 18 17:25:43 localhost kernel: [<f8bb840d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 17:25:43 localhost kernel: [<f8bb840d>] ppp_input+0x45/0xef [ppp_generic]
Feb 18 17:25:43 localhost kernel: [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:25:43 localhost kernel: [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:25:43 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:25:43 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:25:43 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:25:43 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:25:43 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:25:43 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:25:43 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:25:43 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:25:43 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:25:43 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:25:43 localhost kernel: [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:25:43 localhost kernel: [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:25:43 localhost kernel: [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:25:43 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:25:43 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:25:43 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:43 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:43 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:43 localhost kernel:
Feb 18 17:25:43 localhost kernel: to a soft-irq-unsafe lock:
Feb 18 17:25:43 localhost kernel: (&sk->sk_dst_lock){----}
Feb 18 17:25:43 localhost kernel: ... which became soft-irq-unsafe at:
Feb 18 17:25:43 localhost kernel: ... [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 18 17:25:43 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:25:43 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 17:25:43 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 17:25:43 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:43 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:25:43 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 17:25:43 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:25:43 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:25:43 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 17:25:44 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 17:25:44 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 17:25:44 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 17:25:44 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:44 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:25:44 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:44 localhost kernel:
Feb 18 17:25:44 localhost kernel: other info that might help us debug this:
Feb 18 17:25:44 localhost kernel:
Feb 18 17:25:44 localhost kernel: 1 lock held by pppd/13863:
Feb 18 17:25:44 localhost kernel: #0: (&pch->downl){-...}, at: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:25:44 localhost kernel:
Feb 18 17:25:44 localhost kernel: the soft-irq-safe lock's dependencies:
Feb 18 17:25:44 localhost kernel: -> (&pch->upl){-.-+} ops: 13 {
Feb 18 17:25:44 localhost kernel: initial-use at:
Feb 18 17:25:44 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:44 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:44 localhost kernel: [<f8bb6b7c>] ppp_ioctl+0x4df/0xc06 [ppp_generic]
Feb 18 17:25:44 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:44 localhost kernel: [<f8bb6ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 17:25:44 localhost kernel: [<c061b300>] _write_lock_bh+0x2e/0x39
Feb 18 17:25:44 localhost kernel: [<f8bb6ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 17:25:44 localhost kernel: [<f8bb6ba1>] ppp_ioctl+0x504/0xc06 [ppp_generic]
Feb 18 17:25:44 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 17:25:44 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:44 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:44 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 17:25:44 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8
Feb 18 17:25:44 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc
Feb 18 17:25:44 localhost kernel: [<c0488854>] do_ioctl+0x4ebalance_domains+0x28/0x99
Feb 18 17:25:44 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:44 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:44 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:44 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:44 localhost kernel: }
Feb 18 17:25:44 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:25:44 localhost kernel: ... acquired at:
Feb 18 17:25:44 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:44 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:44 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:45 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:45 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:45 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:45 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:45 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:45 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:45 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:45 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:45 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:45 localhost kernel:
Feb 18 17:25:45 localhost kernel: ... acquired at:
Feb 18 17:25:45 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:45 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:45 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:45 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:45 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:45 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:45 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:45 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:45 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:45 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:45 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:45 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:45 localhost kernel:
Feb 18 17:25:45 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:25:45 localhost kernel: initial-use at:
Feb 18 17:25:45 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:45 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:45 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:45 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:45 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:45 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:45 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:45 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:45 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:46 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:46 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:46 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:46 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:46 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:46 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:46 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:46 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:46 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:46 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:46 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:46 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:46 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:46 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:46 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:46 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:46 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:46 localhost kernel: in-hardirq-W at:
Feb 18 17:25:46 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:46 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:46 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:46 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:46 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:46 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:46 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:46 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:46 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:46 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:46 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:46 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:46 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:46 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:46 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:46 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:46 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:46 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:46 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:46 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:46 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:46 localhost kernel: in-softirq-W at:
Feb 18 17:25:46 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:25:46 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:47 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:47 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:47 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:47 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:47 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:47 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:47 localhost kernel: }
Feb 18 17:25:47 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:25:47 localhost kernel: ... acquired at:
Feb 18 17:25:47 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:47 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:47 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:47 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:47 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:47 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:47 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:47 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:47 localhost kernel:
Feb 18 17:25:47 localhost kernel: ... acquired at:
Feb 18 17:25:47 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:47 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:47 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:48 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:48 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:48 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:48 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:48 localhost kernel:
Feb 18 17:25:48 localhost kernel: -> (&rq->rq_lock_key#3){++..} ops: 1928539 {
Feb 18 17:25:48 localhost kernel: initial-use at:
Feb 18 17:25:48 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:48 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:48 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:48 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:48 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:48 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:48 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:48 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:48 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:48 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:48 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:48 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:48 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:48 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:48 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:48 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:48 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:48 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:48 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:48 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:48 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:48 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:48 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:48 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:48 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:48 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:48 localhost kernel: in-hardirq-W at:
Feb 18 17:25:48 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:48 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:48 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:48 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:48 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:48 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:48 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:48 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:48 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:49 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:49 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:49 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:49 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:49 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:49 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:49 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:49 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:49 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:49 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:49 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:49 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:49 localhost kernel: in-softirq-W at:
Feb 18 17:25:49 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:49 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:25:49 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:49 localhost kernel: [<c04422f1>] getnstimeofday+0x30/0xbc
Feb 18 17:25:49 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:49 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:49 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:49 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:49 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:49 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:49 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:25:49 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:25:49 localhost kernel: [<c0430afa>] __do_softirq+0xc9/0xde
Feb 18 17:25:49 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:49 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:49 localhost kernel: }
Feb 18 17:25:49 localhost kernel: ... key at: [<c2a34488>] 0xc2a34488
Feb 18 17:25:49 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:25:49 localhost kernel: initial-use at:
Feb 18 17:25:49 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:49 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:49 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:49 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:49 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:49 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:49 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:50 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:50 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:50 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:50 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:50 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:50 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:50 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:50 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:50 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:50 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:50 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:50 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:50 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:50 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:50 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:50 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:50 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:50 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:50 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:50 localhost kernel: in-hardirq-W at:
Feb 18 17:25:50 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:50 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:50 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:50 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:50 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:50 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:50 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:50 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:50 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:50 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:50 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:50 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:50 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:50 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:50 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:50 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:50 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:50 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:50 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:50 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:50 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:51 localhost kernel: in-softirq-W at:
Feb 18 17:25:51 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:25:51 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:51 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:51 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:51 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:51 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:51 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:51 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:51 localhost kernel: }
Feb 18 17:25:51 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:25:51 localhost kernel: ... acquired at:
Feb 18 17:25:51 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:51 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:51 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:51 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:51 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:51 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:51 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:51 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:51 localhost kernel:
Feb 18 17:25:51 localhost kernel: ... acquired at:
Feb 18 17:25:51 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:51 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:52 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:52 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:52 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:52 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:52 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:52 localhost kernel:
Feb 18 17:25:52 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:25:52 localhost kernel: initial-use at:
Feb 18 17:25:52 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:52 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:52 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:52 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:52 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:52 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:52 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:52 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:52 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:52 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:52 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:52 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:52 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:52 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:52 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:52 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:52 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:52 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:52 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:52 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:52 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:52 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:52 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:52 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:52 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:52 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:52 localhost kernel: in-hardirq-W at:
Feb 18 17:25:52 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:52 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:52 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:52 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:53 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:53 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:53 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:53 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:53 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:53 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:53 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:53 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:53 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:53 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:53 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:53 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:53 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:53 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:53 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:53 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:53 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:53 localhost kernel: in-softirq-W at:
Feb 18 17:25:53 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:25:53 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:53 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:53 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:53 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:53 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:53 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:53 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:53 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:53 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:53 localhost kernel: }
Feb 18 17:25:53 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:25:53 localhost kernel: ... acquired at:
Feb 18 17:25:53 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:53 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:53 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:53 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:53 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:54 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:54 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:54 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:54 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:54 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:54 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:54 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:54 localhost kernel:
Feb 18 17:25:54 localhost kernel: ... acquired at:
Feb 18 17:25:54 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:54 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:54 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:54 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:54 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:54 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:54 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:54 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:54 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:25:54 localhost kernel: [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 17:25:54 localhost kernel: [<c04235fe>] complete+0x36/0x44
Feb 18 17:25:54 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:54 localhost kernel: [<c043d502>] kthread+0x1c/0x5e
Feb 18 17:25:54 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:25:54 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:54 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:54 localhost kernel:
Feb 18 17:25:54 localhost kernel: -> (&rq->rq_lock_key#2){++..} ops: 1235688 {
Feb 18 17:25:54 localhost kernel: initial-use at:
Feb 18 17:25:54 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:54 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:54 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:54 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:54 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:54 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:54 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:54 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:54 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:54 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:54 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:54 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:55 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:55 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:55 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:55 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:55 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:55 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:55 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:55 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:55 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:55 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:55 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:55 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:55 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:55 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:55 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:55 localhost kernel: in-hardirq-W at:
Feb 18 17:25:55 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:25:55 localhost kernel: [<c040a9d6>] save_stack_trace+0x20/0x3a
Feb 18 17:25:55 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:55 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:55 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:55 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:55 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:55 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:55 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:55 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:55 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:55 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:55 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:55 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:55 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:55 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:55 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:55 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:55 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:55 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:55 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:55 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:55 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:55 localhost kernel: in-softirq-W at:
Feb 18 17:25:55 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:56 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:56 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:56 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:56 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:56 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:56 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:56 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:56 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:56 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:56 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:56 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:56 localhost kernel: }
Feb 18 17:25:56 localhost kernel: ... key at: [<c2a2b488>] 0xc2a2b488
Feb 18 17:25:56 localhost kernel: -> (&rq->rq_lock_key#3){++..} ops: 1928539 {
Feb 18 17:25:56 localhost kernel: initial-use at:
Feb 18 17:25:56 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:56 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:56 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:56 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:56 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:56 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:56 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:56 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:56 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:56 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:56 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:56 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:56 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:56 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:56 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:56 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:56 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:56 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:56 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:56 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:56 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:56 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:56 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:56 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:56 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:56 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:56 localhost kernel: in-hardirq-W at:
Feb 18 17:25:57 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:57 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:57 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:57 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:57 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:57 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:57 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:57 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:57 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:57 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:57 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:57 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:57 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:57 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:57 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:57 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:57 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:57 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:57 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:57 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:57 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:57 localhost kernel: in-softirq-W at:
Feb 18 17:25:57 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:57 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:25:57 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:57 localhost kernel: [<c04422f1>] getnstimeofday+0x30/0xbc
Feb 18 17:25:57 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:57 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:57 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:57 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:57 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:57 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:25:57 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:25:57 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:25:57 localhost kernel: [<c0430afa>] __do_softirq+0xc9/0xde
Feb 18 17:25:57 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:57 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:57 localhost kernel: }
Feb 18 17:25:57 localhost kernel: ... key at: [<c2a34488>] 0xc2a34488
Feb 18 17:25:57 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:25:58 localhost kernel: initial-use at:
Feb 18 17:25:58 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:25:58 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:58 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:58 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:25:58 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:58 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:25:58 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:25:58 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:25:58 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:25:58 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:25:58 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:25:58 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:25:58 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:58 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:25:58 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:58 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:25:58 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:25:58 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:25:58 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:25:58 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:58 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:25:58 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:25:58 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:58 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:25:58 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:25:58 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:58 localhost kernel: in-hardirq-W at:
Feb 18 17:25:58 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:25:58 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:58 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:25:58 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:58 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:58 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:58 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:58 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:25:58 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:25:58 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:25:58 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:25:58 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:25:58 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:25:59 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:25:59 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:59 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:25:59 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:25:59 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:25:59 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:25:59 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:25:59 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:59 localhost kernel: in-softirq-W at:
Feb 18 17:25:59 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:25:59 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:25:59 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:59 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:59 localhost openl2tpd: Loading plugin ppp_unix.so, version V1.0
Feb 18 17:25:59 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost openl2tpd[14004]: Start, trace_flags=00000000
Feb 18 17:25:59 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost openl2tpd[14004]: OpenL2TP V1.0, (c) Copyright 2004,2005,2006,2007,2008 Katalix Systems Ltd.
Feb 18 17:25:59 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:25:59 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:25:59 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:25:59 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:25:59 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:25:59 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:25:59 localhost kernel: }
Feb 18 17:25:59 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:25:59 localhost kernel: ... acquired at:
Feb 18 17:25:59 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:25:59 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:25:59 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:25:59 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:25:59 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:25:59 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:59 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:25:59 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:25:59 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:25:59 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:00 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:00 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:00 localhost kernel:
Feb 18 17:26:00 localhost kernel: ... acquired at:
Feb 18 17:26:00 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:00 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:00 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:00 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:00 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:00 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:00 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:00 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:26:00 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:00 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:26:00 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:00 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:00 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:00 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:00 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:00 localhost kernel:
Feb 18 17:26:00 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:26:00 localhost kernel: initial-use at:
Feb 18 17:26:00 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:00 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:00 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:00 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:00 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:00 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:00 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:26:00 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:26:00 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:26:00 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:26:00 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:26:00 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:26:00 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:00 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:00 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:00 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:00 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:26:00 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:26:01 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:01 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:01 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:26:01 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:26:01 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:01 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:01 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:01 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:01 localhost kernel: in-hardirq-W at:
Feb 18 17:26:01 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:26:01 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:01 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:01 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:01 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:01 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:01 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:01 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:01 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:26:01 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:26:01 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:26:01 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:26:01 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:26:01 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:26:01 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:01 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:26:01 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:01 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:26:01 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:26:01 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:26:01 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:01 localhost kernel: in-softirq-W at:
Feb 18 17:26:01 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:01 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:01 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:01 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:01 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:01 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:01 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:01 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:26:01 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:26:01 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:01 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:02 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:02 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:02 localhost kernel: }
Feb 18 17:26:02 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:26:02 localhost kernel: ... acquired at:
Feb 18 17:26:02 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:02 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:02 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:02 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:02 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:02 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:02 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:02 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:26:02 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:02 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:26:02 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:02 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:02 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:02 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:02 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:02 localhost kernel:
Feb 18 17:26:02 localhost kernel: ... acquired at:
Feb 18 17:26:02 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:02 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:02 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:02 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:02 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:02 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:02 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:02 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:26:02 localhost kernel: [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 17:26:02 localhost kernel: [<c04235fe>] complete+0x36/0x44
Feb 18 17:26:02 localhost kernel: [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 17:26:02 localhost kernel: [<c043d502>] kthread+0x1c/0x5e
Feb 18 17:26:02 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:02 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:02 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:02 localhost kernel:
Feb 18 17:26:02 localhost kernel: -> (&rq->rq_lock_key#3){++..} ops: 1928539 {
Feb 18 17:26:02 localhost kernel: initial-use at:
Feb 18 17:26:02 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:03 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:03 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:03 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:03 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:03 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:03 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:26:03 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:26:03 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:26:03 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:26:03 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:26:03 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:26:03 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:03 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:03 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:03 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:03 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:26:03 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:26:03 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:03 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:03 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:26:03 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:26:03 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:03 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:03 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:03 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:03 localhost kernel: in-hardirq-W at:
Feb 18 17:26:03 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:26:03 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:03 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:03 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:03 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:03 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:03 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:03 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:03 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:26:03 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:26:03 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:26:03 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:26:03 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:26:03 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:26:03 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:04 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:26:04 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:04 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:26:04 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:26:04 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:26:04 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:04 localhost kernel: in-softirq-W at:
Feb 18 17:26:04 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:26:04 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:26:04 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:04 localhost kernel: [<c04422f1>] getnstimeofday+0x30/0xbc
Feb 18 17:26:04 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:04 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:04 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:04 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:04 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:04 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:04 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:26:04 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:04 localhost kernel: [<c0430afa>] __do_softirq+0xc9/0xde
Feb 18 17:26:04 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:04 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:04 localhost kernel: }
Feb 18 17:26:04 localhost kernel: ... key at: [<c2a34488>] 0xc2a34488
Feb 18 17:26:04 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:26:04 localhost kernel: initial-use at:
Feb 18 17:26:04 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:04 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:04 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:04 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:04 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:04 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:04 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:26:04 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:26:04 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:26:04 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:26:04 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:26:04 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:26:04 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:04 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:04 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:05 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:05 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:26:05 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:26:05 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:05 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:05 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:26:05 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:26:05 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:05 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:05 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:05 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:05 localhost kernel: in-hardirq-W at:
Feb 18 17:26:05 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:26:05 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:05 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:05 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:05 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:05 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:05 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:05 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:05 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:26:05 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:26:05 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:26:05 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:26:05 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:26:05 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:26:05 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:05 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:26:05 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:05 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:26:05 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:26:05 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:26:05 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:05 localhost kernel: in-softirq-W at:
Feb 18 17:26:05 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:05 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:05 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:05 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:05 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:05 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:05 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:05 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:26:06 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:26:06 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:06 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:06 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:06 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:06 localhost kernel: }
Feb 18 17:26:06 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:26:06 localhost kernel: ... acquired at:
Feb 18 17:26:06 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:06 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:06 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:06 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:06 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:06 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:06 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:06 localhost kernel: [<c04231b1>] __migrate_task+0x45/0xc0
Feb 18 17:26:06 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:06 localhost kernel: [<c0428988>] migration_thread+0x178/0x1d2
Feb 18 17:26:06 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:06 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:06 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:06 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:06 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:06 localhost kernel:
Feb 18 17:26:06 localhost kernel: ... acquired at:
Feb 18 17:26:06 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:06 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:06 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:06 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:06 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:06 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:06 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:06 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:26:06 localhost kernel: [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 17:26:06 localhost kernel: [<c04235fe>] complete+0x36/0x44
Feb 18 17:26:06 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:06 localhost kernel: [<c0428996>] migration_thread+0x186/0x1d2
Feb 18 17:26:06 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:06 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:06 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:07 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:07 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:07 localhost kernel:
Feb 18 17:26:07 localhost kernel: -> (&rq->rq_lock_key#4){++..} ops: 1787266 {
Feb 18 17:26:07 localhost kernel: initial-use at:
Feb 18 17:26:07 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:07 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:07 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:07 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:07 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:07 localhost kernel: [<c0423559>] init_idle+0x77/0xa1
Feb 18 17:26:07 localhost kernel: [<c042b8d9>] fork_idle+0x45/0x4d
Feb 18 17:26:07 localhost kernel: [<c04191c1>] do_boot_cpu+0x3d/0x4b9
Feb 18 17:26:07 localhost kernel: [<c04eb4c4>] __next_cpu+0x12/0x21
Feb 18 17:26:07 localhost kernel: [<c041aa84>] setup_local_APIC+0x27f/0x28f
Feb 18 17:26:07 localhost kernel: [<c075a26f>] verify_local_APIC+0x89/0x137
Feb 18 17:26:07 localhost kernel: [<c07593e0>] native_smp_prepare_cpus+0x34b/0x4a3
Feb 18 17:26:07 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:07 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:07 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:07 localhost kernel: [<c04284c5>] set_cpus_allowed+0x85/0x8d
Feb 18 17:26:07 localhost kernel: [<c0427e1d>] finish_task_switch+0x50/0xbb
Feb 18 17:26:07 localhost kernel: [<c061b505>] _spin_unlock_irq+0x20/0x23
Feb 18 17:26:07 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:07 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:07 localhost kernel: [<c074e399>] kernel_init+0x50/0x2af
Feb 18 17:26:07 localhost kernel: [<c0404f5b>] restore_nocheck+0x12/0x15
Feb 18 17:26:07 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:07 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:07 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:07 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:07 localhost kernel: in-hardirq-W at:
Feb 18 17:26:07 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:26:07 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:07 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:07 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:07 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:07 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:07 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:07 localhost kernel: [<c0427ec7>] scheduler_tick+0x3f/0x19d
Feb 18 17:26:07 localhost kernel: [<c0434365>] update_process_times+0x3a/0x44
Feb 18 17:26:08 localhost kernel: [<c04449ea>] tick_periodic+0x63/0x6d
Feb 18 17:26:08 localhost kernel: [<c0444a0b>] tick_handle_periodic+0x17/0x5c
Feb 18 17:26:08 localhost kernel: [<c0445962>] tick_nohz_stop_sched_tick+0x294/0x2a1
Feb 18 17:26:08 localhost kernel: [<c041af93>] smp_apic_timer_interrupt+0x6f/0x80
Feb 18 17:26:08 localhost kernel: [<c04059e1>] apic_timer_interrupt+0x29/0x38
Feb 18 17:26:08 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:08 localhost kernel: [<c04059eb>] apic_timer_interrupt+0x33/0x38
Feb 18 17:26:08 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:08 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:26:08 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:26:08 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:26:08 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:08 localhost kernel: in-softirq-W at:
Feb 18 17:26:08 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:08 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:08 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:08 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:08 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:08 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:08 localhost kernel: [<c0423141>] double_rq_lock+0x30/0x40
Feb 18 17:26:08 localhost kernel: [<c0425611>] rebalance_domains+0x182/0x393
Feb 18 17:26:08 localhost kernel: [<c0429203>] run_rebalance_domains+0x28/0x99
Feb 18 17:26:08 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:08 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:08 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:08 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:08 localhost kernel: }
Feb 18 17:26:08 localhost kernel: ... key at: [<c2a3d488>] 0xc2a3d488
Feb 18 17:26:08 localhost kernel: ... acquired at:
Feb 18 17:26:08 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:08 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:08 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:08 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:08 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:08 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:08 localhost kernel: [<c0423379>] task_rq_lock+0x2d/0x50
Feb 18 17:26:08 localhost kernel: [<c04250c8>] try_to_wake_up+0x19/0x3c5
Feb 18 17:26:08 localhost kernel: [<c0421394>] __wake_up_common+0x32/0x5c
Feb 18 17:26:08 localhost kernel: [<c04235fe>] complete+0x36/0x44
Feb 18 17:26:08 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:09 localhost kernel: [<c0428996>] migration_thread+0x186/0x1d2
Feb 18 17:26:09 localhost kernel: [<c0428810>] migration_thread+0x0/0x1d2
Feb 18 17:26:09 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:09 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:09 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:09 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:09 localhost kernel:
Feb 18 17:26:09 localhost kernel: ... acquired at:
Feb 18 17:26:09 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:09 localhost kernel: [<c042366f>] __wake_up+0x18/0x42
Feb 18 17:26:09 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:09 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:09 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:09 localhost kernel: [<c042366f>] __wake_up+0x18/0x42
Feb 18 17:26:09 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:09 localhost kernel: [<c042366f>] __wake_up+0x18/0x42
Feb 18 17:26:09 localhost kernel: [<c042366f>] __wake_up+0x18/0x42
Feb 18 17:26:09 localhost kernel: [<f8bb7a2b>] ppp_receive_nonmp_frame+0x51f/0x70a [ppp_generic]
Feb 18 17:26:09 localhost kernel: [<f8bb8487>] ppp_input+0xbf/0xef [ppp_generic]
Feb 18 17:26:09 localhost kernel: [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:26:09 localhost kernel: [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:26:09 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:09 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:26:09 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:26:09 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:26:09 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:26:09 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:26:09 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:09 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:26:09 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:09 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:09 localhost kernel: [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:09 localhost kernel: [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:09 localhost kernel: [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:09 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:09 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:09 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:09 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:09 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:09 localhost kernel:
Feb 18 17:26:09 localhost kernel: ... acquired at:
Feb 18 17:26:09 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:10 localhost kernel: [<f8bb6bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 17:26:10 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:10 localhost kernel: [<f8bb6bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 17:26:10 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:10 localhost kernel: [<f8bb6bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 17:26:10 localhost kernel: [<f8bb6bc0>] ppp_ioctl+0x523/0xc06 [ppp_generic]
Feb 18 17:26:10 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 17:26:10 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:10 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:10 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 17:26:10 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8
Feb 18 17:26:10 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc
Feb 18 17:26:10 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 17:26:10 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 17:26:10 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 17:26:10 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:10 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:10 localhost kernel:
Feb 18 17:26:10 localhost kernel: -> (&list->lock#8){.+..} ops: 46 {
Feb 18 17:26:10 localhost kernel: initial-use at:
Feb 18 17:26:10 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:10 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:10 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:10 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:10 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:10 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:10 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:10 localhost kernel: [<f8bb8433>] ppp_input+0x6b/0xef [ppp_generic]
Feb 18 17:26:10 localhost kernel: [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:26:10 localhost kernel: [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:26:10 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:10 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:26:10 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:26:10 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:26:10 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:26:10 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:26:10 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:10 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:26:10 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:10 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:10 localhost kernel: [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:11 localhost kernel: [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:11 localhost kernel: [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:11 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:11 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:11 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:11 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:11 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:11 localhost kernel: in-softirq-W at:
Feb 18 17:26:11 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:11 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:11 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:11 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:11 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:11 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:11 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:11 localhost kernel: [<f8bb8433>] ppp_input+0x6b/0xef [ppp_generic]
Feb 18 17:26:11 localhost kernel: [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:26:11 localhost kernel: [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:26:11 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:11 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:26:11 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:26:11 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:26:11 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:26:11 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:26:11 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:11 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:26:11 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:11 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:11 localhost kernel: [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:11 localhost kernel: [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:11 localhost kernel: [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:11 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:11 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:11 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:11 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:11 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:11 localhost kernel: }
Feb 18 17:26:11 localhost kernel: ... key at: [<f8bbcd18>] __key.21159+0x0/0xffffb82a [ppp_generic]
Feb 18 17:26:11 localhost kernel: ... acquired at:
Feb 18 17:26:11 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:11 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:12 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:12 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:12 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:12 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:12 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:12 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:12 localhost kernel: [<f8bb6424>] ppp_xmit_process+0x529/0x5a1 [ppp_generic]
Feb 18 17:26:12 localhost kernel: [<f8bb734b>] ppp_channel_push+0x71/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel: [<c061b373>] _read_lock_bh+0x2e/0x39
Feb 18 17:26:12 localhost kernel: [<f8bb735a>] ppp_channel_push+0x80/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel: [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:12 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:12 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:12 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:12 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:12 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:12 localhost kernel:
Feb 18 17:26:12 localhost kernel: -> (&pch->downl){-...} ops: 8 {
Feb 18 17:26:12 localhost kernel: initial-use at:
Feb 18 17:26:12 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:12 localhost kernel: [<c046b874>] __do_fault+0x31b/0x35d
Feb 18 17:26:12 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:12 localhost kernel: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:12 localhost kernel: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel: [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:12 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:12 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:12 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:12 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:12 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:12 localhost kernel: hardirq-on-W at:
Feb 18 17:26:12 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:12 localhost kernel: [<c046b874>] __do_fault+0x31b/0x35d
Feb 18 17:26:12 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:12 localhost kernel: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:12 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:13 localhost kernel: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:13 localhost kernel: [<f8bb72ed>] ppp_channel_push+0x13/0x90 [ppp_generic]
Feb 18 17:26:13 localhost kernel: [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:13 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:13 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:13 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:13 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:13 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:13 localhost kernel: }
Feb 18 17:26:13 localhost kernel: ... key at: [<f8bbcd28>] __key.29740+0x0/0xffffb81a [ppp_generic]
Feb 18 17:26:13 localhost kernel: -> (&list->lock#8){.+..} ops: 46 {
Feb 18 17:26:13 localhost kernel: initial-use at:
Feb 18 17:26:13 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:13 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:13 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:13 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:13 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:13 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:13 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:13 localhost kernel: [<f8bb8433>] ppp_input+0x6b/0xef [ppp_generic]
Feb 18 17:26:13 localhost kernel: [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:26:13 localhost kernel: [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:26:13 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:13 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:26:13 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:26:13 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:26:13 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:26:13 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:26:13 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:13 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:26:13 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:13 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:13 localhost kernel: [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:13 localhost kernel: [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:13 localhost kernel: [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:13 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:13 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:13 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:13 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:13 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:13 localhost kernel: in-softirq-W at:
Feb 18 17:26:14 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:14 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:14 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:14 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:14 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:14 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:14 localhost kernel: [<c05b0641>] skb_queue_tail+0x11/0x2d
Feb 18 17:26:14 localhost kernel: [<f8bb8433>] ppp_input+0x6b/0xef [ppp_generic]
Feb 18 17:26:14 localhost kernel: [<f8bbf263>] pppol2tp_recv_core+0x75c/0x82d [pppol2tp]
Feb 18 17:26:14 localhost kernel: [<f8bbf37a>] pppol2tp_udp_encap_recv+0x46/0x65 [pppol2tp]
Feb 18 17:26:14 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:14 localhost kernel: [<c05ef9df>] udp_queue_rcv_skb+0xba/0x259
Feb 18 17:26:14 localhost kernel: [<c05efffe>] __udp4_lib_rcv+0x480/0x758
Feb 18 17:26:14 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 18 17:26:14 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 18 17:26:14 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 18 17:26:14 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:14 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 18 17:26:14 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:14 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:14 localhost kernel: [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:14 localhost kernel: [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:14 localhost kernel: [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:14 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:14 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:14 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:14 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:14 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:14 localhost kernel: }
Feb 18 17:26:14 localhost kernel: ... key at: [<f8bbcd18>] __key.21159+0x0/0xffffb82a [ppp_generic]
Feb 18 17:26:14 localhost kernel: ... acquired at:
Feb 18 17:26:14 localhost kernel: [<c0446b6a>] add_lock_to_list+0x66/0x89
Feb 18 17:26:14 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:14 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:14 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:14 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:14 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:14 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:14 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:14 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:14 localhost kernel: [<f8bb72ff>] ppp_channel_push+0x25/0x90 [ppp_generic]
Feb 18 17:26:14 localhost kernel: [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:14 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:15 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:15 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:15 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:15 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:15 localhost kernel:
Feb 18 17:26:15 localhost kernel: -> (inet_peer_idlock){-+..} ops: 1947 {
Feb 18 17:26:15 localhost kernel: initial-use at:
Feb 18 17:26:15 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:15 localhost kernel: [<c05d0e17>] rt_bind_peer+0x1b/0x50
Feb 18 17:26:15 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:15 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:15 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:15 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:15 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel: [<c05d4ea2>] ip_push_pending_frames+0x206/0x33b
Feb 18 17:26:15 localhost kernel: [<c05f368d>] icmp_send+0x310/0x386
Feb 18 17:26:15 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:15 localhost kernel: [<c05bcd7a>] neigh_timer_handler+0x14/0x276
Feb 18 17:26:15 localhost kernel: [<c05ce07c>] ipv4_link_failure+0x1c/0x44
Feb 18 17:26:15 localhost kernel: [<c05f1375>] arp_error_report+0x1c/0x24
Feb 18 17:26:15 localhost kernel: [<c05bcf16>] neigh_timer_handler+0x1b0/0x276
Feb 18 17:26:15 localhost kernel: [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:15 localhost kernel: [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 17:26:15 localhost kernel: [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:15 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:15 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:15 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:15 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:15 localhost kernel: in-softirq-W at:
Feb 18 17:26:15 localhost kernel: [<c0446ab0>] save_trace+0x37/0x8b
Feb 18 17:26:15 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:15 localhost kernel: [<c05d0e17>] rt_bind_peer+0x1b/0x50
Feb 18 17:26:15 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:15 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:15 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:15 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:15 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:15 localhost kernel: [<c05d4ea2>] ip_push_pending_frames+0x206/0x33b
Feb 18 17:26:16 localhost kernel: [<c05f368d>] icmp_send+0x310/0x386
Feb 18 17:26:16 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:16 localhost kernel: [<c05bcd7a>] neigh_timer_handler+0x14/0x276
Feb 18 17:26:16 localhost kernel: [<c05ce07c>] ipv4_link_failure+0x1c/0x44
Feb 18 17:26:16 localhost kernel: [<c05f1375>] arp_error_report+0x1c/0x24
Feb 18 17:26:16 localhost kernel: [<c05bcf16>] neigh_timer_handler+0x1b0/0x276
Feb 18 17:26:16 localhost kernel: [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:16 localhost kernel: [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 17:26:16 localhost kernel: [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:16 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:16 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:16 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:16 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:16 localhost kernel: hardirq-on-W at:
Feb 18 17:26:16 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:16 localhost kernel: [<c05d0e17>] rt_bind_peer+0x1b/0x50
Feb 18 17:26:16 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:16 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:16 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:16 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:16 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:16 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:16 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:16 localhost kernel: [<c05d4ea2>] ip_push_pending_frames+0x206/0x33b
Feb 18 17:26:16 localhost kernel: [<c05f368d>] icmp_send+0x310/0x386
Feb 18 17:26:16 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:16 localhost kernel: [<c05bcd7a>] neigh_timer_handler+0x14/0x276
Feb 18 17:26:16 localhost kernel: [<c05ce07c>] ipv4_link_failure+0x1c/0x44
Feb 18 17:26:16 localhost kernel: [<c05f1375>] arp_error_report+0x1c/0x24
Feb 18 17:26:16 localhost kernel: [<c05bcf16>] neigh_timer_handler+0x1b0/0x276
Feb 18 17:26:16 localhost kernel: [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:16 localhost kernel: [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 17:26:16 localhost kernel: [<c05bcd66>] neigh_timer_handler+0x0/0x276
Feb 18 17:26:16 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:16 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:16 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:16 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:16 localhost kernel: }
Feb 18 17:26:16 localhost kernel: ... key at: [<c07380f0>] inet_peer_idlock+0x10/0x1c
Feb 18 17:26:16 localhost kernel: ... acquired at:
Feb 18 17:26:16 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:16 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:17 localhost kernel: [<c048d217>] __d_lookup+0x0/0x12a
Feb 18 17:26:17 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:17 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:17 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:17 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:17 localhost kernel: [<c05d1015>] __ip_select_ident+0x3b/0xaa
Feb 18 17:26:17 localhost kernel: [<c043badc>] __kernel_text_address+0x18/0x23
Feb 18 17:26:17 localhost kernel: [<c05d6ec6>] ip_queue_xmit+0x27b/0x34d
Feb 18 17:26:17 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:17 localhost kernel: [<c047a1bd>] kfree+0x9f/0xa6
Feb 18 17:26:17 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:17 localhost kernel: [<c05b0c83>] pskb_expand_head+0xde/0x144
Feb 18 17:26:17 localhost kernel: [<f8bbf7bf>] pppol2tp_xmit+0x426/0x4a2 [pppol2tp]
Feb 18 17:26:17 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:17 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:17 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:17 localhost kernel: [<f8bb730e>] ppp_channel_push+0x34/0x90 [ppp_generic]
Feb 18 17:26:17 localhost kernel: [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:17 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:17 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:17 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:17 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:17 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:17 localhost kernel:
Feb 18 17:26:17 localhost kernel: -> (&dev->queue_lock#2){-+..} ops: 47607 {
Feb 18 17:26:17 localhost kernel: initial-use at:
Feb 18 17:26:17 localhost kernel: [<c047a1bd>] kfree+0x9f/0xa6
Feb 18 17:26:17 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:17 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:17 localhost kernel: [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:17 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:17 localhost kernel: [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:17 localhost kernel: [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:17 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:17 localhost kernel: [<c05b680a>] register_netdevice+0x29f/0x2f9
Feb 18 17:26:17 localhost kernel: [<c05b6893>] register_netdev+0x2f/0x3b
Feb 18 17:26:17 localhost kernel: [<c076aded>] loopback_net_init+0x36/0x68
Feb 18 17:26:17 localhost kernel: [<c05b3fa0>] register_pernet_operations+0x10/0x11
Feb 18 17:26:17 localhost kernel: [<c05b3ffc>] register_pernet_device+0x1c/0x48
Feb 18 17:26:17 localhost kernel: [<c074e491>] kernel_init+0x148/0x2af
Feb 18 17:26:17 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:18 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:18 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:18 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:18 localhost kernel: in-softirq-W at:
Feb 18 17:26:18 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:18 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:18 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:18 localhost kernel: [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:18 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:18 localhost kernel: [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:18 localhost kernel: [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:18 localhost kernel: [<c05b7e60>] dev_queue_xmit+0x125/0x2e9
Feb 18 17:26:18 localhost kernel: [<c05c341e>] eth_header+0x0/0xb6
Feb 18 17:26:18 localhost kernel: [<f8f3a487>] mld_sendpack+0x242/0x355 [ipv6]
Feb 18 17:26:18 localhost kernel: [<f8f3b453>] mld_ifc_timer_expire+0x143/0x208 [ipv6]
Feb 18 17:26:18 localhost kernel: [<f8f3b4e5>] mld_ifc_timer_expire+0x1d5/0x208 [ipv6]
Feb 18 17:26:18 localhost kernel: [<f8f3b4f0>] mld_ifc_timer_expire+0x1e0/0x208 [ipv6]
Feb 18 17:26:18 localhost kernel: [<f8f3b310>] mld_ifc_timer_expire+0x0/0x208 [ipv6]
Feb 18 17:26:18 localhost kernel: [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 17:26:18 localhost kernel: [<f8f3b310>] mld_ifc_timer_expire+0x0/0x208 [ipv6]
Feb 18 17:26:18 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:18 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:18 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:18 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:18 localhost kernel: hardirq-on-W at:
Feb 18 17:26:18 localhost kernel: [<c047a1bd>] kfree+0x9f/0xa6
Feb 18 17:26:18 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:18 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:18 localhost kernel: [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:18 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:18 localhost kernel: [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:18 localhost kernel: [<c05c47ad>] qdisc_lock_tree+0x14/0x1c
Feb 18 17:26:18 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:18 localhost kernel: [<c05b680a>] register_netdevice+0x29f/0x2f9
Feb 18 17:26:18 localhost kernel: [<c05b6893>] register_netdev+0x2f/0x3b
Feb 18 17:26:18 localhost kernel: [<c076aded>] loopback_net_init+0x36/0x68
Feb 18 17:26:18 localhost kernel: [<c05b3fa0>] register_pernet_operations+0x10/0x11
Feb 18 17:26:18 localhost kernel: [<c05b3ffc>] register_pernet_device+0x1c/0x48
Feb 18 17:26:18 localhost kernel: [<c074e491>] kernel_init+0x148/0x2af
Feb 18 17:26:18 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:18 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:18 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:19 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:19 localhost kernel: }
Feb 18 17:26:19 localhost kernel: ... key at: [<c0a3e1c4>] __key.31614+0x0/0x8
Feb 18 17:26:19 localhost kernel: -> (&dev->ingress_lock){-...} ops: 208 {
Feb 18 17:26:19 localhost kernel: initial-use at:
Feb 18 17:26:19 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:19 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:19 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:19 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel: [<c05b680a>] register_netdevice+0x29f/0x2f9
Feb 18 17:26:19 localhost kernel: [<c05b6893>] register_netdev+0x2f/0x3b
Feb 18 17:26:19 localhost kernel: [<c076aded>] loopback_net_init+0x36/0x68
Feb 18 17:26:19 localhost kernel: [<c05b3fa0>] register_pernet_operations+0x10/0x11
Feb 18 17:26:19 localhost kernel: [<c05b3ffc>] register_pernet_device+0x1c/0x48
Feb 18 17:26:19 localhost kernel: [<c074e491>] kernel_init+0x148/0x2af
Feb 18 17:26:19 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:19 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:19 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:19 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:19 localhost kernel: hardirq-on-W at:
Feb 18 17:26:19 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:19 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:19 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:19 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:19 localhost kernel: [<c05b680a>] register_netdevice+0x29f/0x2f9
Feb 18 17:26:19 localhost kernel: [<c05b6893>] register_netdev+0x2f/0x3b
Feb 18 17:26:19 localhost kernel: [<c076aded>] loopback_net_init+0x36/0x68
Feb 18 17:26:19 localhost kernel: [<c05b3fa0>] register_pernet_operations+0x10/0x11
Feb 18 17:26:19 localhost kernel: [<c05b3ffc>] register_pernet_device+0x1c/0x48
Feb 18 17:26:19 localhost kernel: [<c074e491>] kernel_init+0x148/0x2af
Feb 18 17:26:19 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:19 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:19 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:19 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:19 localhost kernel: }
Feb 18 17:26:19 localhost kernel: ... key at: [<c0a3e1b4>] __key.31616+0x0/0x8
Feb 18 17:26:19 localhost kernel: -> (&list->lock#6){....} ops: 675 {
Feb 18 17:26:19 localhost kernel: initial-use at:
Feb 18 17:26:19 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:19 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:20 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:20 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel: [<c05b0afd>] skb_queue_purge+0x11/0x17
Feb 18 17:26:20 localhost kernel: [<c05c4eca>] pfifo_fast_reset+0x14/0x2f
Feb 18 17:26:20 localhost kernel: [<c05c4712>] qdisc_reset+0x10/0x11
Feb 18 17:26:20 localhost kernel: [<c05c4b83>] dev_deactivate+0x2a/0xfd
Feb 18 17:26:20 localhost kernel: [<c05bf1f4>] __linkwatch_run_queue+0x143/0x17a
Feb 18 17:26:20 localhost kernel: [<c05bf248>] linkwatch_event+0x1d/0x22
Feb 18 17:26:20 localhost kernel: [<c043a0f3>] run_workqueue+0xdb/0x1ae
Feb 18 17:26:20 localhost kernel: [<c043a09f>] run_workqueue+0x87/0x1ae
Feb 18 17:26:20 localhost kernel: [<c05bf22b>] linkwatch_event+0x0/0x22
Feb 18 17:26:20 localhost kernel: [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 17:26:20 localhost kernel: [<c043ab92>] worker_thread+0xba/0xc4
Feb 18 17:26:20 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:20 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:20 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:20 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:20 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:20 localhost kernel: }
Feb 18 17:26:20 localhost kernel: ... key at: [<c0a3e7d4>] __key.21167+0x0/0x8
Feb 18 17:26:20 localhost kernel: ... acquired at:
Feb 18 17:26:20 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:20 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:20 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:20 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:20 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:20 localhost kernel: [<c05b0afd>] skb_queue_purge+0x11/0x17
Feb 18 17:26:20 localhost kernel: [<c05c4eca>] pfifo_fast_reset+0x14/0x2f
Feb 18 17:26:20 localhost kernel: [<c05c48f7>] qdisc_destroy+0x36/0x69
Feb 18 17:26:20 localhost kernel: [<c05c4954>] dev_shutdown+0x2a/0x7b
Feb 18 17:26:20 localhost kernel: [<c05b6210>] rollback_registered+0xb8/0x11e
Feb 18 17:26:20 localhost kernel: [<c05b627e>] unregister_netdevice+0x8/0x35
Feb 18 17:26:20 localhost kernel: [<c05b62ba>] unregister_netdev+0xf/0x15
Feb 18 17:26:20 localhost kernel: [<f8bb523f>] ppp_shutdown_interface+0x58/0xad [ppp_generic]
Feb 18 17:26:20 localhost kernel: [<f8bb5484>] ppp_release+0x29/0x56 [ppp_generic]
Feb 18 17:26:20 localhost kernel: [<c047f08a>] __fput+0xba/0x172
Feb 18 17:26:20 localhost kernel: [<c047c7df>] filp_close+0x51/0x58
Feb 18 17:26:21 localhost kernel: [<c047d9c4>] sys_close+0x70/0xab
Feb 18 17:26:21 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:21 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:21 localhost kernel:
Feb 18 17:26:21 localhost kernel: ... acquired at:
Feb 18 17:26:21 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:21 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:21 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:21 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:21 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:21 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:21 localhost kernel: [<c05c47d9>] dev_init_scheduler+0x8/0x4f
Feb 18 17:26:21 localhost kernel: [<c05b680a>] register_netdevice+0x29f/0x2f9
Feb 18 17:26:21 localhost kernel: [<c05b6893>] register_netdev+0x2f/0x3b
Feb 18 17:26:21 localhost kernel: [<c076aded>] loopback_net_init+0x36/0x68
Feb 18 17:26:21 localhost kernel: [<c05b3fa0>] register_pernet_operations+0x10/0x11
Feb 18 17:26:21 localhost kernel: [<c05b3ffc>] register_pernet_device+0x1c/0x48
Feb 18 17:26:21 localhost kernel: [<c074e491>] kernel_init+0x148/0x2af
Feb 18 17:26:21 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:21 localhost kernel: [<c074e349>] kernel_init+0x0/0x2af
Feb 18 17:26:21 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:21 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:21 localhost kernel:
Feb 18 17:26:21 localhost kernel: -> (&list->lock#6){....} ops: 675 {
Feb 18 17:26:21 localhost kernel: initial-use at:
Feb 18 17:26:21 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:21 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:21 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:21 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:21 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:21 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:21 localhost kernel: [<c05b0afd>] skb_queue_purge+0x11/0x17
Feb 18 17:26:21 localhost kernel: [<c05c4eca>] pfifo_fast_reset+0x14/0x2f
Feb 18 17:26:21 localhost kernel: [<c05c4712>] qdisc_reset+0x10/0x11
Feb 18 17:26:21 localhost kernel: [<c05c4b83>] dev_deactivate+0x2a/0xfd
Feb 18 17:26:21 localhost kernel: [<c05bf1f4>] __linkwatch_run_queue+0x143/0x17a
Feb 18 17:26:21 localhost kernel: [<c05bf248>] linkwatch_event+0x1d/0x22
Feb 18 17:26:21 localhost kernel: [<c043a0f3>] run_workqueue+0xdb/0x1ae
Feb 18 17:26:21 localhost kernel: [<c043a09f>] run_workqueue+0x87/0x1ae
Feb 18 17:26:21 localhost kernel: [<c05bf22b>] linkwatch_event+0x0/0x22
Feb 18 17:26:21 localhost kernel: [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 17:26:21 localhost kernel: [<c043ab92>] worker_thread+0xba/0xc4
Feb 18 17:26:22 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:22 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:22 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:22 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:22 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:22 localhost kernel: }
Feb 18 17:26:22 localhost kernel: ... key at: [<c0a3e7d4>] __key.21167+0x0/0x8
Feb 18 17:26:22 localhost kernel: ... acquired at:
Feb 18 17:26:22 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:22 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:22 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:22 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:22 localhost kernel: [<c061b55d>] _spin_lock_irqsave+0x32/0x41
Feb 18 17:26:22 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:22 localhost kernel: [<c05b06e1>] skb_dequeue+0xf/0x3f
Feb 18 17:26:22 localhost kernel: [<c05b0afd>] skb_queue_purge+0x11/0x17
Feb 18 17:26:22 localhost kernel: [<c05c4eca>] pfifo_fast_reset+0x14/0x2f
Feb 18 17:26:22 localhost kernel: [<c05c4712>] qdisc_reset+0x10/0x11
Feb 18 17:26:22 localhost kernel: [<c05c4b83>] dev_deactivate+0x2a/0xfd
Feb 18 17:26:22 localhost kernel: [<c05bf1f4>] __linkwatch_run_queue+0x143/0x17a
Feb 18 17:26:22 localhost kernel: [<c05bf248>] linkwatch_event+0x1d/0x22
Feb 18 17:26:22 localhost kernel: [<c043a0f3>] run_workqueue+0xdb/0x1ae
Feb 18 17:26:22 localhost kernel: [<c043a09f>] run_workqueue+0x87/0x1ae
Feb 18 17:26:22 localhost kernel: [<c05bf22b>] linkwatch_event+0x0/0x22
Feb 18 17:26:22 localhost kernel: [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 17:26:22 localhost kernel: [<c043ab92>] worker_thread+0xba/0xc4
Feb 18 17:26:22 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:22 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:22 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:22 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:22 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:22 localhost kernel:
Feb 18 17:26:22 localhost kernel: ... acquired at:
Feb 18 17:26:22 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:22 localhost kernel: [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:22 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:22 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:22 localhost kernel: [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:22 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:22 localhost kernel: [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:22 localhost kernel: [<c05b7eb4>] dev_queue_xmit+0x179/0x2e9
Feb 18 17:26:22 localhost kernel: [<c05b7e60>] dev_queue_xmit+0x125/0x2e9
Feb 18 17:26:23 localhost kernel: [<c05d62e7>] ip_finish_output+0x1e6/0x21e
Feb 18 17:26:23 localhost kernel: [<c05d6f58>] ip_queue_xmit+0x30d/0x34d
Feb 18 17:26:23 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:23 localhost kernel: [<c047a1bd>] kfree+0x9f/0xa6
Feb 18 17:26:23 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:23 localhost kernel: [<c05b0c83>] pskb_expand_head+0xde/0x144
Feb 18 17:26:23 localhost kernel: [<f8bbf7bf>] pppol2tp_xmit+0x426/0x4a2 [pppol2tp]
Feb 18 17:26:23 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:23 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:23 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:23 localhost kernel: [<f8bb730e>] ppp_channel_push+0x34/0x90 [ppp_generic]
Feb 18 17:26:23 localhost kernel: [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:23 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:23 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:23 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:23 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:23 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:23 localhost kernel:
Feb 18 17:26:23 localhost kernel: -> (&rt_hash_locks[i]){-+..} ops: 24708841 {
Feb 18 17:26:23 localhost kernel: initial-use at:
Feb 18 17:26:23 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:23 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:23 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:23 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:23 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:23 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:23 localhost kernel: [<c05f90b6>] ip_mc_inc_group+0x184/0x1c1
Feb 18 17:26:23 localhost kernel: [<c05f9134>] ip_mc_up+0x41/0x59
Feb 18 17:26:23 localhost kernel: [<c05f4f9f>] inetdev_event+0x257/0x46e
Feb 18 17:26:23 localhost kernel: [<c05c258a>] fib_rules_event+0x120/0x12a
Feb 18 17:26:23 localhost kernel: [<c05c24b4>] fib_rules_event+0x4a/0x12a
Feb 18 17:26:23 localhost kernel: [<c061d06c>] notifier_call_chain+0x2a/0x47
Feb 18 17:26:23 localhost kernel: [<c0441084>] raw_notifier_call_chain+0x17/0x1a
Feb 18 17:26:23 localhost kernel: [<c05b73cf>] dev_open+0x71/0x77
Feb 18 17:26:23 localhost kernel: [<c05b5ffa>] dev_change_flags+0x9d/0x149
Feb 18 17:26:23 localhost kernel: [<c05b49b5>] __dev_get_by_name+0x67/0x72
Feb 18 17:26:23 localhost kernel: [<c05f54f7>] devinet_ioctl+0x22e/0x539
Feb 18 17:26:23 localhost kernel: [<c05b7188>] dev_ioctl+0x46f/0x5e0
Feb 18 17:26:23 localhost kernel: [<c05ab078>] sock_ioctl+0x1bb/0x1e0
Feb 18 17:26:23 localhost kernel: [<c05aaebd>] sock_ioctl+0x0/0x1e0
Feb 18 17:26:23 localhost kernel: [<c0488827>] do_ioctl+0x1f/0x62
Feb 18 17:26:23 localhost kernel: [<c061ccb2>] do_page_fault+0x234/0x5c4
Feb 18 17:26:23 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 17:26:24 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 17:26:24 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:24 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:24 localhost kernel: in-softirq-W at:
Feb 18 17:26:24 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:24 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:24 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:24 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel: [<c05cee70>] rt_run_flush+0x0/0x8b
Feb 18 17:26:24 localhost kernel: [<c0433d67>] run_timer_softirq+0x121/0x188
Feb 18 17:26:24 localhost kernel: [<c05cee70>] rt_run_flush+0x0/0x8b
Feb 18 17:26:24 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:24 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:24 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:24 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:24 localhost kernel: hardirq-on-W at:
Feb 18 17:26:24 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:24 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:24 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:24 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel: [<c05ceeb3>] rt_run_flush+0x43/0x8b
Feb 18 17:26:24 localhost kernel: [<c05f90b6>] ip_mc_inc_group+0x184/0x1c1
Feb 18 17:26:24 localhost kernel: [<c05f9134>] ip_mc_up+0x41/0x59
Feb 18 17:26:24 localhost kernel: [<c05f4f9f>] inetdev_event+0x257/0x46e
Feb 18 17:26:24 localhost kernel: [<c05c258a>] fib_rules_event+0x120/0x12a
Feb 18 17:26:24 localhost kernel: [<c05c24b4>] fib_rules_event+0x4a/0x12a
Feb 18 17:26:24 localhost kernel: [<c061d06c>] notifier_call_chain+0x2a/0x47
Feb 18 17:26:24 localhost kernel: [<c0441084>] raw_notifier_call_chain+0x17/0x1a
Feb 18 17:26:24 localhost kernel: [<c05b73cf>] dev_open+0x71/0x77
Feb 18 17:26:24 localhost kernel: [<c05b5ffa>] dev_change_flags+0x9d/0x149
Feb 18 17:26:24 localhost kernel: [<c05b49b5>] __dev_get_by_name+0x67/0x72
Feb 18 17:26:24 localhost kernel: [<c05f54f7>] devinet_ioctl+0x22e/0x539
Feb 18 17:26:24 localhost kernel: [<c05b7188>] dev_ioctl+0x46f/0x5e0
Feb 18 17:26:24 localhost kernel: [<c05ab078>] sock_ioctl+0x1bb/0x1e0
Feb 18 17:26:24 localhost kernel: [<c05aaebd>] sock_ioctl+0x0/0x1e0
Feb 18 17:26:24 localhost kernel: [<c0488827>] do_ioctl+0x1f/0x62
Feb 18 17:26:24 localhost kernel: [<c061ccb2>] do_page_fault+0x234/0x5c4
Feb 18 17:26:24 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 17:26:24 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 17:26:24 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:25 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:25 localhost kernel: }
Feb 18 17:26:25 localhost kernel: ... key at: [<c0a3ea90>] __key.39046+0x0/0x8
Feb 18 17:26:25 localhost kernel: -> (&parent->list_lock){++..} ops: 252663 {
Feb 18 17:26:25 localhost kernel: initial-use at:
Feb 18 17:26:25 localhost kernel: [<c040a9d6>] save_stack_trace+0x20/0x3a
Feb 18 17:26:25 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:25 localhost kernel: [<c043badc>] __kernel_text_address+0x18/0x23
Feb 18 17:26:25 localhost kernel: [<c0448013>] check_usage_backwards+0x19/0x41
Feb 18 17:26:25 localhost kernel: [<c0446ab0>] save_trace+0x37/0x8b
Feb 18 17:26:25 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:25 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:25 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:25 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:25 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:25 localhost kernel: [<c047b89c>] kmem_cache_create+0x62/0x38e
Feb 18 17:26:25 localhost kernel: [<c047a790>] kmem_cache_alloc+0x53/0x9f
Feb 18 17:26:25 localhost kernel: [<c047b99d>] kmem_cache_create+0x163/0x38e
Feb 18 17:26:25 localhost kernel: [<c0479da2>] cache_estimate+0x6c/0x89
Feb 18 17:26:25 localhost kernel: [<c0762241>] kmem_cache_init+0x182/0x3f6
Feb 18 17:26:25 localhost kernel: [<c074e88b>] start_kernel+0x293/0x327
Feb 18 17:26:25 localhost kernel: [<c074e0e0>] unknown_bootoption+0x0/0x195
Feb 18 17:26:25 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:25 localhost kernel: in-hardirq-W at:
Feb 18 17:26:25 localhost kernel: [<c0448fc0>] __lock_acquire+0x405/0xbf1
Feb 18 17:26:25 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:25 localhost kernel: [<c047b7ec>] do_drain+0x1e/0x49
Feb 18 17:26:25 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:25 localhost kernel: [<c047b7ec>] do_drain+0x1e/0x49
Feb 18 17:26:25 localhost kernel: [<c047b7ec>] do_drain+0x1e/0x49
Feb 18 17:26:25 localhost kernel: [<c047b7ce>] do_drain+0x0/0x49
Feb 18 17:26:25 localhost kernel: [<c0418ed5>] smp_call_function_interrupt+0x37/0x62
Feb 18 17:26:25 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:25 localhost kernel: [<c04059b3>] call_function_interrupt+0x33/0x38
Feb 18 17:26:25 localhost kernel: [<c0403c9b>] default_idle+0x0/0x54
Feb 18 17:26:25 localhost kernel: [<c041007b>] cyrix_get_arr+0xbc/0x126
Feb 18 17:26:25 localhost kernel: [<c0403cd8>] default_idle+0x3d/0x54
Feb 18 17:26:25 localhost kernel: [<c0403583>] cpu_idle+0x9f/0xc0
Feb 18 17:26:25 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:25 localhost kernel: in-softirq-W at:
Feb 18 17:26:25 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:26:25 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:26 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:26 localhost kernel: [<c04488b6>] debug_check_no_locks_freed+0x105/0x11f
Feb 18 17:26:26 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:26 localhost kernel: [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:26 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:26 localhost kernel: [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:26 localhost kernel: [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:26 localhost kernel: [<c047a1fa>] kmem_cache_free+0x36/0x5e
Feb 18 17:26:26 localhost kernel: [<c043b647>] __rcu_process_callbacks+0xfc/0x16e
Feb 18 17:26:26 localhost kernel: [<c043b6d1>] rcu_process_callbacks+0x18/0x30
Feb 18 17:26:26 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:26 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:26 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:26 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:26 localhost kernel: }
Feb 18 17:26:26 localhost kernel: ... key at: [<c0a2bbe8>] __key.19871+0x0/0x8
Feb 18 17:26:26 localhost kernel: -> (&zone->lock){.+..} ops: 63132 {
Feb 18 17:26:26 localhost kernel: initial-use at:
Feb 18 17:26:26 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:26 localhost kernel: [<c04ef19a>] number+0x147/0x215
Feb 18 17:26:26 localhost kernel: [<c04efab1>] vsnprintf+0x440/0x47c
Feb 18 17:26:26 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:26 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:26 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:26 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:26 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:26 localhost kernel: [<c04645c0>] free_hot_cold_page+0x133/0x175
Feb 18 17:26:26 localhost kernel: [<c07606b8>] free_all_bootmem_core+0xdd/0x158
Feb 18 17:26:26 localhost kernel: [<c075d82e>] mem_init+0x8c/0x383
Feb 18 17:26:26 localhost kernel: [<c042c8c7>] printk+0x1b/0x1f
Feb 18 17:26:26 localhost kernel: [<c0761a7b>] alloc_large_system_hash+0x1f7/0x222
Feb 18 17:26:26 localhost kernel: [<c04f2f65>] __spin_lock_init+0x29/0x49
Feb 18 17:26:26 localhost kernel: [<c076287c>] inode_init_early+0x49/0x72
Feb 18 17:26:26 localhost kernel: [<c074e886>] start_kernel+0x28e/0x327
Feb 18 17:26:26 localhost kernel: [<c074e0e0>] unknown_bootoption+0x0/0x195
Feb 18 17:26:26 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:26 localhost kernel: in-softirq-W at:
Feb 18 17:26:26 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:26 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:26:26 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:26 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:27 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:27 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:27 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:27 localhost kernel: [<c04645c0>] free_hot_cold_page+0x133/0x175
Feb 18 17:26:27 localhost kernel: [<c042a030>] free_task+0x18/0x25
Feb 18 17:26:27 localhost kernel: [<c043b647>] __rcu_process_callbacks+0xfc/0x16e
Feb 18 17:26:27 localhost kernel: [<c043b6d1>] rcu_process_callbacks+0x18/0x30
Feb 18 17:26:27 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:27 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:27 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:27 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:27 localhost kernel: }
Feb 18 17:26:27 localhost kernel: ... key at: [<c0a21fb8>] __key.25233+0x0/0x8
Feb 18 17:26:27 localhost kernel: ... acquired at:
Feb 18 17:26:27 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:27 localhost kernel: [<c046472e>] __free_pages_ok+0x101/0x2a8
Feb 18 17:26:27 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:27 localhost kernel: [<c046472e>] __free_pages_ok+0x101/0x2a8
Feb 18 17:26:27 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:27 localhost kernel: [<c046472e>] __free_pages_ok+0x101/0x2a8
Feb 18 17:26:27 localhost kernel: [<c046472e>] __free_pages_ok+0x101/0x2a8
Feb 18 17:26:27 localhost kernel: [<c0479f5b>] kmem_freepages+0x4e/0xa5
Feb 18 17:26:27 localhost kernel: [<c047a2a2>] slab_destroy+0x2e/0x49
Feb 18 17:26:27 localhost kernel: [<c047a370>] free_block+0xb3/0xec
Feb 18 17:26:27 localhost kernel: [<c047a433>] drain_array+0x8a/0xb5
Feb 18 17:26:27 localhost kernel: [<c047b4ec>] cache_reap+0x7e/0xf4
Feb 18 17:26:27 localhost kernel: [<c043a0f3>] run_workqueue+0xdb/0x1ae
Feb 18 17:26:27 localhost kernel: [<c043a09f>] run_workqueue+0x87/0x1ae
Feb 18 17:26:27 localhost kernel: [<c047b46e>] cache_reap+0x0/0xf4
Feb 18 17:26:27 localhost kernel: [<c043aad8>] worker_thread+0x0/0xc4
Feb 18 17:26:27 localhost kernel: [<c043ab92>] worker_thread+0xba/0xc4
Feb 18 17:26:27 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:27 localhost kernel: [<c043d51e>] kthread+0x38/0x5e
Feb 18 17:26:27 localhost kernel: [<c043d4e6>] kthread+0x0/0x5e
Feb 18 17:26:27 localhost kernel: [<c0405b83>] kernel_thread_helper+0x7/0x10
Feb 18 17:26:27 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:27 localhost kernel:
Feb 18 17:26:27 localhost kernel: -> (&on_slab_l3_key){.+..} ops: 28386 {
Feb 18 17:26:27 localhost kernel: initial-use at:
Feb 18 17:26:27 localhost kernel: [<c044816c>] mark_lock+0x65/0x454
Feb 18 17:26:27 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:27 localhost kernel: [<c040a9d6>] save_stack_trace+0x20/0x3a
Feb 18 17:26:27 localhost kernel: [<c0464b77>] get_page_from_freelist+0x24b/0x37a
Feb 18 17:26:28 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:28 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:28 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:28 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel: [<c046a328>] mod_zone_page_state+0x35/0x3c
Feb 18 17:26:28 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:28 localhost kernel: [<c047a790>] kmem_cache_alloc+0x53/0x9f
Feb 18 17:26:28 localhost kernel: [<c047ab07>] cache_alloc_refill+0x32b/0x493
Feb 18 17:26:28 localhost kernel: [<c047a790>] kmem_cache_alloc+0x53/0x9f
Feb 18 17:26:28 localhost kernel: [<c075ee99>] pidmap_init+0x15/0x137
Feb 18 17:26:28 localhost kernel: [<c074e8a0>] start_kernel+0x2a8/0x327
Feb 18 17:26:28 localhost kernel: [<c074e0e0>] unknown_bootoption+0x0/0x195
Feb 18 17:26:28 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:28 localhost kernel: in-softirq-W at:
Feb 18 17:26:28 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:28 localhost kernel: [<c05fdd2b>] fn_hash_lookup+0xaa/0xb4
Feb 18 17:26:28 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:28 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:28 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:28 localhost kernel: [<c047acf5>] __kmalloc+0x86/0xd2
Feb 18 17:26:28 localhost kernel: [<c05bb6b9>] neigh_create+0x2e4/0x41a
Feb 18 17:26:28 localhost kernel: [<c05baadb>] neigh_lookup+0x94/0x9d
Feb 18 17:26:28 localhost kernel: [<c05bb82b>] neigh_event_ns+0x3c/0x76
Feb 18 17:26:28 localhost kernel: [<c05f2116>] arp_process+0x218/0x60c
Feb 18 17:26:28 localhost kernel: [<c05f1efe>] arp_process+0x0/0x60c
Feb 18 17:26:28 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:28 localhost kernel: [<c05f250a>] arp_rcv+0x0/0x105
Feb 18 17:26:28 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 18 17:26:28 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 18 17:26:28 localhost kernel: [<f89637c7>] e1000_clean_rx_irq+0x374/0x44a [e1000]
Feb 18 17:26:28 localhost kernel: [<f8963453>] e1000_clean_rx_irq+0x0/0x44a [e1000]
Feb 18 17:26:28 localhost kernel: [<f8960f13>] e1000_clean+0x63/0x203 [e1000]
Feb 18 17:26:28 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 18 17:26:28 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 18 17:26:28 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:28 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:28 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:28 localhost kernel: }
Feb 18 17:26:29 localhost kernel: ... key at: [<c0a2bc0c>] on_slab_l3_key+0x0/0x8
Feb 18 17:26:29 localhost kernel: -> (&zone->lock){.+..} ops: 63132 {
Feb 18 17:26:29 localhost kernel: initial-use at:
Feb 18 17:26:29 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:29 localhost kernel: [<c04ef19a>] number+0x147/0x215
Feb 18 17:26:29 localhost kernel: [<c04efab1>] vsnprintf+0x440/0x47c
Feb 18 17:26:29 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:29 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:29 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel: [<c04645c0>] free_hot_cold_page+0x133/0x175
Feb 18 17:26:29 localhost kernel: [<c07606b8>] free_all_bootmem_core+0xdd/0x158
Feb 18 17:26:29 localhost kernel: [<c075d82e>] mem_init+0x8c/0x383
Feb 18 17:26:29 localhost kernel: [<c042c8c7>] printk+0x1b/0x1f
Feb 18 17:26:29 localhost kernel: [<c0761a7b>] alloc_large_system_hash+0x1f7/0x222
Feb 18 17:26:29 localhost kernel: [<c04f2f65>] __spin_lock_init+0x29/0x49
Feb 18 17:26:29 localhost kernel: [<c076287c>] inode_init_early+0x49/0x72
Feb 18 17:26:29 localhost kernel: [<c074e886>] start_kernel+0x28e/0x327
Feb 18 17:26:29 localhost kernel: [<c074e0e0>] unknown_bootoption+0x0/0x195
Feb 18 17:26:29 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:29 localhost kernel: in-softirq-W at:
Feb 18 17:26:29 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 18 17:26:29 localhost kernel: [<c0422625>] find_busiest_group+0x204/0x5f3
Feb 18 17:26:29 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:29 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:29 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel: [<c04645c0>] free_hot_cold_page+0x133/0x175
Feb 18 17:26:29 localhost kernel: [<c042a030>] free_task+0x18/0x25
Feb 18 17:26:29 localhost kernel: [<c043b647>] __rcu_process_callbacks+0xfc/0x16e
Feb 18 17:26:29 localhost kernel: [<c043b6d1>] rcu_process_callbacks+0x18/0x30
Feb 18 17:26:29 localhost kernel: [<c0430b8e>] tasklet_action+0x50/0xa4
Feb 18 17:26:29 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 18 17:26:29 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 18 17:26:29 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:29 localhost kernel: }
Feb 18 17:26:29 localhost kernel: ... key at: [<c0a21fb8>] __key.25233+0x0/0x8
Feb 18 17:26:29 localhost kernel: ... acquired at:
Feb 18 17:26:29 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:29 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:29 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:29 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:30 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:30 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:30 localhost kernel: [<c0464182>] free_pages_bulk+0x28/0x1b6
Feb 18 17:26:30 localhost kernel: [<c04645c0>] free_hot_cold_page+0x133/0x175
Feb 18 17:26:30 localhost kernel: [<c047a2a2>] slab_destroy+0x2e/0x49
Feb 18 17:26:30 localhost kernel: [<c047a370>] free_block+0xb3/0xec
Feb 18 17:26:30 localhost kernel: [<c047a0f2>] cache_flusharray+0x74/0xa0
Feb 18 17:26:30 localhost kernel: [<c047a19c>] kfree+0x7e/0xa6
Feb 18 17:26:30 localhost kernel: [<c05f594c>] snmp_mib_free+0x1a/0x29
Feb 18 17:26:30 localhost kernel: [<f8f283ac>] in6_dev_finish_destroy+0x8e/0xb9 [ipv6]
Feb 18 17:26:30 localhost kernel: [<f8f28dd9>] addrconf_ifdown+0x2df/0x2eb [ipv6]
Feb 18 17:26:30 localhost kernel: [<f8f2b0f5>] addrconf_notify+0x738/0x7c2 [ipv6]
Feb 18 17:26:30 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:30 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:30 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:30 localhost kernel: [<c0614a16>] packet_notifier+0x149/0x151
Feb 18 17:26:30 localhost kernel: [<c06148f3>] packet_notifier+0x26/0x151
Feb 18 17:26:30 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:30 localhost kernel: [<c0614a16>] packet_notifier+0x149/0x151
Feb 18 17:26:30 localhost kernel: [<c061d06c>] notifier_call_chain+0x2a/0x47
Feb 18 17:26:30 localhost kernel: [<c0441084>] raw_notifier_call_chain+0x17/0x1a
Feb 18 17:26:30 localhost kernel: [<c05b621c>] rollback_registered+0xc4/0x11e
Feb 18 17:26:30 localhost kernel: [<c05b627e>] unregister_netdevice+0x8/0x35
Feb 18 17:26:30 localhost kernel: [<c05b62ba>] unregister_netdev+0xf/0x15
Feb 18 17:26:30 localhost kernel: [<f8bb523f>] ppp_shutdown_interface+0x58/0xad [ppp_generic]
Feb 18 17:26:30 localhost kernel: [<f8bb5484>] ppp_release+0x29/0x56 [ppp_generic]
Feb 18 17:26:30 localhost kernel: [<c047f08a>] __fput+0xba/0x172
Feb 18 17:26:30 localhost kernel: [<c047c7df>] filp_close+0x51/0x58
Feb 18 17:26:30 localhost kernel: [<c047d9c4>] sys_close+0x70/0xab
Feb 18 17:26:30 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:30 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:30 localhost kernel:
Feb 18 17:26:30 localhost kernel: ... acquired at:
Feb 18 17:26:30 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:30 localhost kernel: [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:30 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:30 localhost kernel: [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:30 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:30 localhost kernel: [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:30 localhost kernel: [<c047a0a0>] cache_flusharray+0x22/0xa0
Feb 18 17:26:30 localhost kernel: [<c047a1fa>] kmem_cache_free+0x36/0x5e
Feb 18 17:26:30 localhost kernel: [<c047a370>] free_block+0xb3/0xec
Feb 18 17:26:30 localhost kernel: [<c047a0f2>] cache_flusharray+0x74/0xa0
Feb 18 17:26:31 localhost kernel: [<c047a19c>] kfree+0x7e/0xa6
Feb 18 17:26:31 localhost kernel: [<f8f28d8e>] addrconf_ifdown+0x294/0x2eb [ipv6]
Feb 18 17:26:31 localhost kernel: [<f8f2b0f5>] addrconf_notify+0x738/0x7c2 [ipv6]
Feb 18 17:26:31 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:31 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:31 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:31 localhost kernel: [<c0614a16>] packet_notifier+0x149/0x151
Feb 18 17:26:31 localhost kernel: [<c06148f3>] packet_notifier+0x26/0x151
Feb 18 17:26:31 localhost kernel: [<c061b171>] _read_unlock+0x14/0x1c
Feb 18 17:26:31 localhost kernel: [<c0614a16>] packet_notifier+0x149/0x151
Feb 18 17:26:31 localhost kernel: [<c061d06c>] notifier_call_chain+0x2a/0x47
Feb 18 17:26:31 localhost kernel: [<c0441084>] raw_notifier_call_chain+0x17/0x1a
Feb 18 17:26:31 localhost kernel: [<c05b621c>] rollback_registered+0xc4/0x11e
Feb 18 17:26:31 localhost kernel: [<c05b627e>] unregister_netdevice+0x8/0x35
Feb 18 17:26:31 localhost kernel: [<c05b62ba>] unregister_netdev+0xf/0x15
Feb 18 17:26:31 localhost kernel: [<f8bb523f>] ppp_shutdown_interface+0x58/0xad [ppp_generic]
Feb 18 17:26:31 localhost kernel: [<f8bb5484>] ppp_release+0x29/0x56 [ppp_generic]
Feb 18 17:26:31 localhost kernel: [<c047f08a>] __fput+0xba/0x172
Feb 18 17:26:31 localhost kernel: [<c047c7df>] filp_close+0x51/0x58
Feb 18 17:26:31 localhost kernel: [<c047d9c4>] sys_close+0x70/0xab
Feb 18 17:26:31 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:31 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:31 localhost kernel:
Feb 18 17:26:31 localhost kernel: ... acquired at:
Feb 18 17:26:31 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:31 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:31 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:31 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:31 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:31 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:31 localhost kernel: [<c047a834>] cache_alloc_refill+0x58/0x493
Feb 18 17:26:31 localhost kernel: [<c047a790>] kmem_cache_alloc+0x53/0x9f
Feb 18 17:26:31 localhost kernel: [<c05bb516>] neigh_create+0x141/0x41a
Feb 18 17:26:31 localhost kernel: [<c05baadb>] neigh_lookup+0x94/0x9d
Feb 18 17:26:31 localhost kernel: [<c05f144f>] arp_bind_neighbour+0x62/0x74
Feb 18 17:26:31 localhost kernel: [<c05cf248>] rt_intern_hash+0x236/0x322
Feb 18 17:26:31 localhost kernel: [<c05cfab6>] __ip_route_output_key+0x782/0x847
Feb 18 17:26:31 localhost kernel: [<c05cf948>] __ip_route_output_key+0x614/0x847
Feb 18 17:26:31 localhost kernel: [<c05eace7>] tcp_v4_connect+0xef/0x5a6
Feb 18 17:26:31 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:31 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 17:26:32 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:32 localhost kernel: [<c05f6944>] inet_stream_connect+0x7f/0x208
Feb 18 17:26:32 localhost kernel: [<c04f05f4>] copy_from_user+0x32/0x5e
Feb 18 17:26:32 localhost kernel: [<c05ac371>] sys_connect+0x7b/0xa6
Feb 18 17:26:32 localhost kernel: [<c05aea5a>] sock_setsockopt+0x492/0x4aa
Feb 18 17:26:32 localhost kernel: [<c05aea5a>] sock_setsockopt+0x492/0x4aa
Feb 18 17:26:32 localhost kernel: [<c05db937>] tcp_setsockopt+0x321/0x339
Feb 18 17:26:32 localhost kernel: [<c05ab618>] sock_map_fd+0x41/0x4a
Feb 18 17:26:32 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 18 17:26:32 localhost kernel: [<c05ab47f>] sys_setsockopt+0x62/0x9c
Feb 18 17:26:32 localhost kernel: [<c05acb97>] sys_socketcall+0xac/0x261
Feb 18 17:26:32 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:32 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:32 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:32 localhost kernel:
Feb 18 17:26:32 localhost kernel: ... acquired at:
Feb 18 17:26:32 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:32 localhost kernel: [<c05cf06f>] rt_intern_hash+0x5d/0x322
Feb 18 17:26:32 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:32 localhost kernel: [<c05cf06f>] rt_intern_hash+0x5d/0x322
Feb 18 17:26:32 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:32 localhost kernel: [<c05cf06f>] rt_intern_hash+0x5d/0x322
Feb 18 17:26:32 localhost kernel: [<c05cf06f>] rt_intern_hash+0x5d/0x322
Feb 18 17:26:32 localhost kernel: [<c05cf000>] rt_set_nexthop+0x105/0x117
Feb 18 17:26:32 localhost kernel: [<c05cfab6>] __ip_route_output_key+0x782/0x847
Feb 18 17:26:32 localhost kernel: [<c05cf948>] __ip_route_output_key+0x614/0x847
Feb 18 17:26:32 localhost kernel: [<c0464b77>] get_page_from_freelist+0x24b/0x37a
Feb 18 17:26:32 localhost kernel: [<c05cfb8f>] ip_route_output_flow+0x14/0x1ef
Feb 18 17:26:32 localhost kernel: [<c05b0c83>] pskb_expand_head+0xde/0x144
Feb 18 17:26:32 localhost kernel: [<f8bbf6fe>] pppol2tp_xmit+0x365/0x4a2 [pppol2tp]
Feb 18 17:26:32 localhost kernel: [<f8bb730e>] ppp_channel_push+0x34/0x90 [ppp_generic]
Feb 18 17:26:32 localhost kernel: [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:32 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:32 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:32 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:32 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:32 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:32 localhost kernel:
Feb 18 17:26:32 localhost kernel: ... acquired at:
Feb 18 17:26:32 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:32 localhost kernel: [<f8bb56db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 17:26:32 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:32 localhost kernel: [<f8bb56db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 17:26:33 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:33 localhost kernel: [<f8bb56db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 17:26:33 localhost kernel: [<f8bb56db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 18 17:26:33 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:33 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 18 17:26:33 localhost kernel: [<f8bb63f1>] ppp_xmit_process+0x4f6/0x5a1 [ppp_generic]
Feb 18 17:26:33 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:33 localhost kernel: [<f8bb83b4>] ppp_write+0xc7/0xdb [ppp_generic]
Feb 18 17:26:33 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:33 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:33 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:33 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:33 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:33 localhost kernel:
Feb 18 17:26:33 localhost kernel: ... acquired at:
Feb 18 17:26:33 localhost kernel: [<c04495c0>] __lock_acquire+0xa05/0xbf1
Feb 18 17:26:33 localhost kernel: [<f8bb6bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 17:26:33 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:33 localhost kernel: [<f8bb6bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 17:26:33 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 18 17:26:33 localhost kernel: [<f8bb6bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 17:26:33 localhost kernel: [<f8bb6bb5>] ppp_ioctl+0x518/0xc06 [ppp_generic]
Feb 18 17:26:33 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 17:26:33 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 18 17:26:33 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:33 localhost kernel: [<c061b008>] __down+0x82/0xb8
Feb 18 17:26:33 localhost kernel: [<c0425474>] default_wake_function+0x0/0x8
Feb 18 17:26:33 localhost kernel: [<c061ae17>] __down_failed+0x7/0xc
Feb 18 17:26:33 localhost kernel: [<c0488854>] do_ioctl+0x4c/0x62
Feb 18 17:26:33 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 18 17:26:33 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 18 17:26:33 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:33 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:33 localhost kernel:
Feb 18 17:26:33 localhost kernel:
Feb 18 17:26:33 localhost kernel: the soft-irq-unsafe lock's dependencies:
Feb 18 17:26:33 localhost kernel: -> (&sk->sk_dst_lock){----} ops: 3988 {
Feb 18 17:26:33 localhost kernel: initial-use at:
Feb 18 17:26:33 localhost kernel: [<c0449059>] __lock_acquire+0x49e/0xbf1
Feb 18 17:26:33 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:33 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 17:26:33 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 17:26:33 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 17:26:34 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 17:26:34 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 17:26:34 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 17:26:34 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:34 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:34 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:34 localhost kernel: softirq-on-W at:
Feb 18 17:26:34 localhost kernel: [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 18 17:26:34 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:34 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 17:26:34 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 17:26:34 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 17:26:34 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 17:26:34 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 17:26:34 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 17:26:34 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:34 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:34 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:34 localhost kernel: hardirq-on-W at:
Feb 18 17:26:34 localhost kernel: [<c0449027>] __lock_acquire+0x46c/0xbf1
Feb 18 17:26:34 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:34 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115
Feb 18 17:26:34 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 18 17:26:34 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 18 17:26:34 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e
Feb 18 17:26:34 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f
Feb 18 17:26:34 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 18 17:26:34 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 18 17:26:34 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 18 17:26:35 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 18 17:26:35 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:35 localhost kernel: softirq-on-R at:
Feb 18 17:26:35 localhost kernel: [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 18 17:26:35 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:35 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel: [<c061b3a7>] _read_lock+0x29/0x34
Feb 18 17:26:35 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel: [<c05baadb>] neigh_lookup+0x94/0x9d
Feb 18 17:26:35 localhost kernel: [<c05f0609>] udp_sendmsg+0x240/0x59d
Feb 18 17:26:35 localhost kernel: [<c05cf322>] rt_intern_hash+0x310/0x322
Feb 18 17:26:35 localhost kernel: [<c05f5fc7>] inet_sendmsg+0x3b/0x45
Feb 18 17:26:35 localhost kernel: [<c05ab8cd>] sock_sendmsg+0xc9/0xe4
Feb 18 17:26:35 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:35 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:35 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:35 localhost kernel: [<c061b528>] _read_unlock_irq+0x20/0x23
Feb 18 17:26:35 localhost kernel: [<c061b528>] _read_unlock_irq+0x20/0x23
Feb 18 17:26:35 localhost kernel: [<c05ac29b>] sys_sendto+0x11b/0x13b
Feb 18 17:26:35 localhost kernel: [<c046b794>] __do_fault+0x23b/0x35d
Feb 18 17:26:35 localhost kernel: [<c046b874>] __do_fault+0x31b/0x35d
Feb 18 17:26:35 localhost kernel: [<c046d4ff>] handle_mm_fault+0x2a2/0x5f7
Feb 18 17:26:35 localhost kernel: [<c05ac2f2>] sys_send+0x37/0x3b
Feb 18 17:26:35 localhost kernel: [<c05acc35>] sys_socketcall+0x14a/0x261
Feb 18 17:26:35 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:35 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:35 localhost kernel: hardirq-on-R at:
Feb 18 17:26:35 localhost kernel: [<c048926a>] do_sys_poll+0x260/0x2e0
Feb 18 17:26:35 localhost kernel: [<c0449001>] __lock_acquire+0x446/0xbf1
Feb 18 17:26:35 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:35 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel: [<c061b3a7>] _read_lock+0x29/0x34
Feb 18 17:26:35 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel: [<c05ada91>] sk_dst_check+0x18/0x105
Feb 18 17:26:35 localhost kernel: [<c05baadb>] neigh_lookup+0x94/0x9d
Feb 18 17:26:35 localhost kernel: [<c05f0609>] udp_sendmsg+0x240/0x59d
Feb 18 17:26:35 localhost kernel: [<c05cf322>] rt_intern_hash+0x310/0x322
Feb 18 17:26:35 localhost kernel: [<c05f5fc7>] inet_sendmsg+0x3b/0x45
Feb 18 17:26:35 localhost kernel: [<c05ab8cd>] sock_sendmsg+0xc9/0xe4
Feb 18 17:26:35 localhost kernel: [<c043d5d5>] autoremove_wake_function+0x0/0x35
Feb 18 17:26:35 localhost kernel: [<c0449764>] __lock_acquire+0xba9/0xbf1
Feb 18 17:26:35 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 18 17:26:36 localhost kernel: [<c061b528>] _read_unlock_irq+0x20/0x23
Feb 18 17:26:36 localhost kernel: [<c061b528>] _read_unlock_irq+0x20/0x23
Feb 18 17:26:36 localhost kernel: [<c05ac29b>] sys_sendto+0x11b/0x13b
Feb 18 17:26:36 localhost kernel: [<c046b794>] __do_fault+0x23b/0x35d
Feb 18 17:26:36 localhost kernel: [<c046b874>] __do_fault+0x31b/0x35d
Feb 18 17:26:36 localhost kernel: [<c046d4ff>] handle_mm_fault+0x2a2/0x5f7
Feb 18 17:26:36 localhost kernel: [<c05ac2f2>] sys_send+0x37/0x3b
Feb 18 17:26:36 localhost kernel: [<c05acc35>] sys_socketcall+0x14a/0x261
Feb 18 17:26:36 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:36 localhost kernel: [<ffffffff>] 0xffffffff
Feb 18 17:26:36 localhost kernel: }
Feb 18 17:26:36 localhost kernel: ... key at: [<c0a3dc28>] __key.35996+0x0/0x8
Feb 18 17:26:36 localhost kernel:
Feb 18 17:26:36 localhost kernel: stack backtrace:
Feb 18 17:26:36 localhost kernel: Pid: 13863, comm: pppd Not tainted 2.6.24.2 #1
Feb 18 17:26:36 localhost kernel: [<c0448b1e>] check_usage+0x24e/0x258
Feb 18 17:26:36 localhost kernel: [<c0449555>] __lock_acquire+0x99a/0xbf1
Feb 18 17:26:36 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 18 17:26:36 localhost kernel: [<f8bbf73e>] pppol2tp_xmit+0x3a5/0x4a2 [pppol2tp]
Feb 18 17:26:36 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34
Feb 18 17:26:36 localhost kernel: [<f8bbf73e>] pppol2tp_xmit+0x3a5/0x4a2 [pppol2tp]
Feb 18 17:26:36 localhost kernel: [<f8bbf73e>] pppol2tp_xmit+0x3a5/0x4a2 [pppol2tp]
Feb 18 17:26:36 localhost kernel: [<f8bb730e>] ppp_channel_push+0x34/0x90 [ppp_generic]
Feb 18 17:26:36 localhost kernel: [<f8bb83bd>] ppp_write+0xd0/0xdb [ppp_generic]
Feb 18 17:26:36 localhost kernel: [<f8bb82ed>] ppp_write+0x0/0xdb [ppp_generic]
Feb 18 17:26:36 localhost kernel: [<c047e6d2>] vfs_write+0xa1/0x14d
Feb 18 17:26:36 localhost kernel: [<c047ecfd>] sys_write+0x41/0x67
Feb 18 17:26:36 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 18 17:26:36 localhost kernel: =======================On Mon, Feb 18, 2008 at 10:09:24PM +0000, James Chapman wrote:
...
Quoted 10 lines
Unfortunately the ISP's syslog stops. But I've been able to borrow two Quad Xeon boxes and have reproduced the problem.
Here's a new version of the patch. The patch avoids disabling irqs and fixes the
sk_dst_get()usage that DaveM mentioned. But even with this patch, lockdep still complains if hundreds of ppp sessions are inserted into a tunnel as rapidly as possible (lockdep trace is below). I can stop these errors by wrapping the call toppp_input()inpppol2tp_recv_dequeue_skb()with local_irq_save/restore. What is a better fix?
I send here my proposal: it's intended for testing and to check one of possible solutions here. IMHO your lockdep reports show there is no use to change anything around sk_dst_lock: it would need the global change of this lock to fix this problem. So the fix should be done around pch->upl lock and this means changing ppp_generic.
In the patch below I've used trylock in places which seem to allow for skipping some things (while config is changed only) or simply don't need this lock because there is no ppp struct. This could be modified to add some waiting loop if necessary. Another option is to change the write side of this lock: it looks like more vulnerable if something missed because there are more locks involved, but probably should be enough to solve this problem too.
I think pppol2tp need to be first checked only with hlist_lock bh patch, unless there were some lockdep reports on these other locks too. (BTW, I added ppp maintainer to CC - I hope we get Paul's opinion on this.)
Regards,
Jarek P.
(testing patch #1)
---
drivers/net/ppp_generic.c | 33 +++++++++++++++++++++++---------- 1 files changed, 23 insertions(+), 10 deletions(-)
drivers/net/ppp_generic.c +23 -10
@@ -1473,7 +1473,7 @@ void ppp_input(struct ppp_channel *chan, struct sk_buff *skb) { struct channel *pch = chan->ppp; -int proto; +int proto, locked;if (!pch || skb->len ==0 ) {kfree_skb (skb);
@@ -1481,8 +1481,13 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb) } proto =PPP_PROTO (skb); -read_lock_bh (&pch->upl); -if (!pch->ppp || proto >=0xc000 || proto ==PPP_CCPFRAG ) { +/* +* We use trylock to avoid dependency between soft-irq-safe upl lock +* and soft-irq-unsafe sk_dst_lock. +*/ +local_bh_disable (); + locked =read_trylock (&pch->upl); +if (!locked || !pch->ppp || proto >=0xc000 || proto ==PPP_CCPFRAG ) {/* put it on the channel queue */ skb_queue_tail (&pch->file.rq, skb);/* drop old frames if queue too long */
@@ -1493,7 +1498,10 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb) }else {ppp_do_recv (pch->ppp, skb, pch); } -read_unlock_bh (&pch->upl); + +if (locked) +read_unlock (&pch->upl); +local_bh_enable (); }/* Put a 0-length skb in the receive queue as an error indication */
@@ -1506,16 +1514,18 @@ ppp_input_error(struct ppp_channel *chan, int code)if (!pch)return ; -read_lock_bh (&pch->upl); -if (pch->ppp) { +/* a trylock comment in ppp_input() */ +local_bh_disable (); +if (read_trylock (&pch->upl) && pch->ppp) { skb =alloc_skb (0 ,GFP_ATOMIC );if (skb) { skb->len =0 ;/* probably unnecessary */ skb->cb[0 ] = code;ppp_do_recv (pch->ppp, skb, pch); } +read_unlock (&pch->upl); } -read_unlock_bh (&pch->upl); +local_bh_enable (); }/*
@@ -2044,10 +2054,13 @@ int ppp_unit_number(struct ppp_channel *chan)int unit =-1 ;if (pch) { -read_lock_bh (&pch->upl); -if (pch->ppp) +/* a trylock comment in ppp_input() */ +local_bh_disable (); +if (read_trylock (&pch->upl) && pch->ppp) { unit = pch->ppp->file.index; -read_unlock_bh (&pch->upl); +read_unlock (&pch->upl); + } +local_bh_enable (); }return unit; }
On Wed, Feb 20, 2008 at 12:06:40AM +0100, Jarek Poplawski wrote:
...
Quoted 1 line
(testing patch #1)
SORRY!!! ----> take 2 (unlocking fixed)
---
drivers/net/ppp_generic.c | 39 +++++++++++++++++++++++++++++---------- 1 files changed, 29 insertions(+), 10 deletions(-)
drivers/net/ppp_generic.c +29 -10
@@ -1473,7 +1473,7 @@ void ppp_input(struct ppp_channel *chan, struct sk_buff *skb) { struct channel *pch = chan->ppp; -int proto; +int proto, locked;if (!pch || skb->len ==0 ) {kfree_skb (skb);
@@ -1481,8 +1481,13 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb) } proto =PPP_PROTO (skb); -read_lock_bh (&pch->upl); -if (!pch->ppp || proto >=0xc000 || proto ==PPP_CCPFRAG ) { +/* +* We use trylock to avoid dependency between soft-irq-safe upl lock +* and soft-irq-unsafe sk_dst_lock. +*/ +local_bh_disable (); + locked =read_trylock (&pch->upl); +if (!locked || !pch->ppp || proto >=0xc000 || proto ==PPP_CCPFRAG ) {/* put it on the channel queue */ skb_queue_tail (&pch->file.rq, skb);/* drop old frames if queue too long */
@@ -1493,7 +1498,10 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb) }else {ppp_do_recv (pch->ppp, skb, pch); } -read_unlock_bh (&pch->upl); + +if (locked) +read_unlock (&pch->upl); +local_bh_enable (); }/* Put a 0-length skb in the receive queue as an error indication */
@@ -1502,12 +1510,14 @@ ppp_input_error(struct ppp_channel *chan, int code) {struct channel *pch = chan->ppp;struct sk_buff *skb; +int locked;if (!pch)return ; -read_lock_bh (&pch->upl); -if (pch->ppp) { +/* a trylock comment in ppp_input() */ +local_bh_disable (); +if ((locked =read_trylock (&pch->upl)) && pch->ppp) { skb =alloc_skb (0 ,GFP_ATOMIC );if (skb) { skb->len =0 ;/* probably unnecessary */
@@ -1515,7 +1525,10 @@ ppp_input_error(struct ppp_channel *chan, int code)ppp_do_recv (pch->ppp, skb, pch); } } -read_unlock_bh (&pch->upl); + +if (locked) +read_unlock (&pch->upl); +local_bh_enable (); }/*
@@ -2044,10 +2057,16 @@ int ppp_unit_number(struct ppp_channel *chan)int unit =-1 ;if (pch) { -read_lock_bh (&pch->upl); -if (pch->ppp) +int locked; + +/* a trylock comment in ppp_input() */ +local_bh_disable (); +if ((locked =read_trylock (&pch->upl)) && pch->ppp) unit = pch->ppp->file.index; -read_unlock_bh (&pch->upl); + +if (locked) +read_unlock (&pch->upl); +local_bh_enable (); }return unit; }
Quoted 13 lines · Jarek Poplawski
On Mon, Feb 18, 2008 at 10:09:24PM +0000, James Chapman wrote:
...Quoted 10 lines
Unfortunately the ISP's syslog stops. But I've been able to borrow two Quad Xeon boxes and have reproduced the problem.
Here's a new version of the patch. The patch avoids disabling irqs and fixes the
sk_dst_get()usage that DaveM mentioned. But even with this patch, lockdep still complains if hundreds of ppp sessions are inserted into a tunnel as rapidly as possible (lockdep trace is below). I can stop these errors by wrapping the call toppp_input()inpppol2tp_recv_dequeue_skb()with local_irq_save/restore. What is a better fix?
Quoted 5 lines
I send here my proposal: it's intended for testing and to check one of possible solutions here. IMHO your lockdep reports show there is no use to change anything around
sk_dst_lock: it would need the global change of this lock to fix this problem. So the fix should be done aroundpch->upllock and this means changingppp_generic.
Hmm, I need to study the lockdep report again. It seems I'm misreading the lockdep output. :(
Quoted 8 lines
In the patch below I've used trylock in places which seem to allow for skipping some things (while config is changed only) or simply don't need this lock because there is no ppp struct. This could be modified to add some waiting loop if necessary. Another option is to change the write side of this lock: it looks like more vulnerable if something missed because there are more locks involved, but probably should be enough to solve this problem too.
Quoted 4 lines
I think pppol2tp need to be first checked only with
hlist_lockbh patch, unless there were some lockdep reports on these other locks too. (BTW, I added ppp maintainer to CC - I hope we get Paul's opinion on this.)
I tried your ppp_generic patch with only the hlist_lock bh patch in pppol2tp and it seems to fix the ppp create/delete issue. However, when I added much more traffic into the test (flood pings over ppp interfaces while repeatedly creating/deleting the L2TP (PPP) sessions) I get a soft lockup detected in pppol2tp_xmit() after anything between 1 minute and an hour. :( I'm investigating that now.
Thanks for your help!
Quoted 87 lines
(testing patch #1)
---drivers/net/ppp_generic.c | 33 +++++++++++++++++++++++---------- 1 files changed, 23 insertions(+), 10 deletions(-)drivers/net/ppp_generic.c +23
-11@@ -1473,7 +1473,7 @@ void ppp_input(struct ppp_channel *chan, struct sk_buff *skb) { struct channel *pch = chan->ppp; -int proto; +int proto, locked;if (!pch || skb->len ==0 ) {kfree_skb (skb);@@ -1481,8 +1481,13 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb) } proto =PPP_PROTO (skb); -read_lock_bh (&pch->upl); -if (!pch->ppp || proto >=0xc000 || proto ==PPP_CCPFRAG ) { +/* +* We use trylock to avoid dependency between soft-irq-safe upl lock +* and soft-irq-unsafe sk_dst_lock. +*/ +local_bh_disable (); + locked =read_trylock (&pch->upl); +if (!locked || !pch->ppp || proto >=0xc000 || proto ==PPP_CCPFRAG ) {/* put it on the channel queue */ skb_queue_tail (&pch->file.rq, skb);/* drop old frames if queue too long */ @@ -1493,7 +1498,10 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb) }else {ppp_do_recv (pch->ppp, skb, pch); } -read_unlock_bh (&pch->upl); + +if (locked) +read_unlock (&pch->upl); +local_bh_enable (); }/* Put a 0-length skb in the receive queue as an error indication */ @@ -1506,16 +1514,18 @@ ppp_input_error(struct ppp_channel *chan, int code)if (!pch)return ; -read_lock_bh (&pch->upl); -if (pch->ppp) { +/* a trylock comment in ppp_input() */ +local_bh_disable (); +if (read_trylock (&pch->upl) && pch->ppp) { skb =alloc_skb (0 ,GFP_ATOMIC );if (skb) { skb->len =0 ;/* probably unnecessary */ skb->cb[0 ] = code;ppp_do_recv (pch->ppp, skb, pch); } +read_unlock (&pch->upl); } -read_unlock_bh (&pch->upl); +local_bh_enable (); }/* @@ -2044,10 +2054,13 @@ int ppp_unit_number(struct ppp_channel *chan)int unit =-1 ;if (pch) { -read_lock_bh (&pch->upl); -if (pch->ppp) +/* a trylock comment in ppp_input() */ +local_bh_disable (); +if (read_trylock (&pch->upl) && pch->ppp) { unit = pch->ppp->file.index; -read_unlock_bh (&pch->upl); +read_unlock (&pch->upl); + } +local_bh_enable (); }return unit; } --
Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
On Wed, Feb 20, 2008 at 04:02:52PM +0000, James Chapman wrote:
...
Quoted 8 lines
I tried your
ppp_genericpatch with only thehlist_lockbh patch in pppol2tp and it seems to fix the ppp create/delete issue. However, when I added much more traffic into the test (flood pings over ppp interfaces while repeatedly creating/deleting the L2TP (PPP) sessions) I get a soft lockup detected inpppol2tp_xmit()after anything between 1 minute and an hour. :( I'm investigating that now.Thanks for your help!
Not at all!
Quoted 2 lines
Quoted 2 lines
(testing patch #1)
But I hope you tested with the fixed (take 2) version of this patch...
Since it's quite experimental (testing) this patch could be wrong as it is, but I hope it should show the proper way to solve this problem. Probably you did some of these, but here are a few of my suggestions for testing this:
1) try my patch with your full bh locking changing patch;
2) add while loops to these trylocks on failure, with e.g. __delay(1); this should work like full locks again, but there should be no (this kind of) lockdep reports;
3) I send here another testing patch with this second way to do this:
on the write side, but it's even more "experimental" and only a proof of concept (should be applied on vanilla ppp_generic).
Regards,
Jarek P.
(testing patch #2)
---
drivers/net/ppp_generic.c | 20 +++++++++++++------- 1 files changed, 13 insertions(+), 7 deletions(-)
drivers/net/ppp_generic.c +13 -7
@@ -2606,11 +2606,16 @@ ppp_connect_channel(struct channel *pch, int unit) ppp =ppp_find_unit (unit);if (!ppp) goto out; -write_lock_bh (&pch->upl); + ret = -EINVAL ; -if (pch->ppp) - goto outl; +read_lock_bh (&pch->upl); +if (pch->ppp) { +read_unlock_bh (&pch->upl); + goto out; + } +read_unlock_bh (&pch->upl); +atomic_inc (&ppp->file.refcnt);ppp_lock (ppp);if (pch->file.hdrlen > ppp->file.hdrlen) ppp->file.hdrlen = pch->file.hdrlen;
@@ -2619,13 +2624,14 @@ ppp_connect_channel(struct channel *pch, int unit) ppp->dev->hard_header_len = hdrlen;list_add_tail (&pch->clist, &ppp->channels); ++ppp->n_channels; - pch->ppp = ppp; -atomic_inc (&ppp->file.refcnt);ppp_unlock (ppp); - ret =0 ; - outl: +/* avoid lock dependency with ppp_locks */ +write_lock_bh (&pch->upl); +BUG_ON (pch->ppp); + pch->ppp = ppp;write_unlock_bh (&pch->upl); + ret =0 ; out:mutex_unlock (&all_ppp_mutex);return ret;
Quoted 3 lines · Jarek Poplawski
Quoted 1 line
Quoted 1 line
(testing patch #1)
But I hope you tested with the fixed (take 2) version of this patch...
Yes I did. :)
But I just got another lockdep error (attached).
Quoted 9 lines
Since it's quite experimental (testing) this patch could be wrong as it is, but I hope it should show the proper way to solve this problem. Probably you did some of these, but here are a few of my suggestions for testing this:
1) try my patch with your full bh locking changing patch;
2) add while loops to these trylocks on failure, with e.g.__delay(1); this should work like full locks again, but there should be no (this kind of) lockdep reports;
Hmm, isn't this just bypassing the lockdep checks?
Quoted 3 lines
3) I send here another testing patch with this second way to do this:
on the write side, but it's even more "experimental" and only a proof of concept (should be applied on vanillappp_generic).
I'll look over it. I think I need to take a step back and look at what's happening in more detail though.
Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
Feb 20 22:11:41 localhost kernel: =================================
Feb 20 22:11:43 localhost kernel: [ INFO: inconsistent lock state ]
Feb 20 22:11:44 localhost kernel: 2.6.24.2 #1
Feb 20 22:11:44 localhost kernel: ---------------------------------
Feb 20 22:11:44 localhost kernel: inconsistent {softirq-on-W} -> {in-softirq-R} usage.
Feb 20 22:11:44 localhost kernel: pppd/3744 [HC0[0]:SC1[5]:HE1:SE0] takes:
Feb 20 22:11:44 localhost kernel: (&sk->sk_dst_lock){---?}, at: [<f8bac857>] pppol2tp_xmit+0x320/0x3d9 [pppol2tp]
Feb 20 22:11:45 localhost kernel: {softirq-on-W} state was registered at:
Feb 20 22:11:45 localhost kernel: [<c0449046>] __lock_acquire+0x48b/0xbf1
Feb 20 22:11:45 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 20 22:11:45 localhost kernel: [<c043083a>] local_bh_enable+0x10e/0x115
Feb 20 22:11:45 localhost kernel: [<c05da996>] inet_csk_get_port+0xc1/0x1cb
Feb 20 22:11:45 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 20 22:11:45 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 20 22:11:45 localhost kernel: [<c061b2c7>] _write_lock+0x29/0x34
Feb 20 22:11:45 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 20 22:11:45 localhost kernel: [<c05da85d>] inet_csk_listen_start+0x75/0xed
Feb 20 22:11:45 localhost kernel: [<c05f66da>] inet_listen+0x3b/0x5e
Feb 20 22:11:45 localhost kernel: [<c05ab68f>] sys_listen+0x43/0x5f
Feb 20 22:11:45 localhost kernel: [<c05acba8>] sys_socketcall+0xbd/0x261
Feb 20 22:11:45 localhost kernel: [<c0404ead>] sysenter_past_esp+0x9a/0xa5
Feb 20 22:11:45 localhost kernel: [<c0448787>] trace_hardirqs_on+0x122/0x14c
Feb 20 22:11:45 localhost kernel: [<c0404e72>] sysenter_past_esp+0x5f/0xa5
Feb 20 22:11:45 localhost kernel: [<ffffffff>] 0xffffffff
Feb 20 22:11:45 localhost kernel: irq event stamp: 41702
Feb 20 22:11:45 localhost kernel: hardirqs last enabled at (41702): [<c047a1bd>] kfree+0x9f/0xa6
Feb 20 22:11:45 localhost kernel: hardirqs last disabled at (41701): [<c047a135>] kfree+0x17/0xa6
Feb 20 22:11:45 localhost kernel: softirqs last enabled at (41630): [<c05ceed4>] rt_run_flush+0x64/0x8b
Feb 20 22:11:45 localhost kernel: softirqs last disabled at (41631): [<c040709f>] do_softirq+0x5e/0xc7
Feb 20 22:11:45 localhost kernel:
Feb 20 22:11:45 localhost kernel: other info that might help us debug this:
Feb 20 22:11:45 localhost kernel: 10 locks held by pppd/3744:
Feb 20 22:11:45 localhost kernel: #0: (rtnl_mutex){--..}, at: [<c05f53bd>] devinet_ioctl+0xf4/0x539
Feb 20 22:11:45 localhost kernel: #1: ((inetaddr_chain).rwsem){..--}, at: [<c04411c9>] __blocking_notifier_call_chain+0x22/0x51
Feb 20 22:11:45 localhost kernel: #2: (rcu_read_lock){..--}, at: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 20 22:11:45 localhost kernel: #3: (rcu_read_lock){..--}, at: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 20 22:11:45 localhost kernel: #4: (rcu_read_lock){..--}, at: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 20 22:11:45 localhost kernel: #5: (slock-AF_INET){-+..}, at: [<c05f2d21>] icmp_reply+0x52/0x1a4
Feb 20 22:11:45 localhost kernel: #6: (rcu_read_lock){..--}, at: [<c05b7e60>] dev_queue_xmit+0x125/0x2e9
Feb 20 22:11:45 localhost kernel: #7: (_xmit_PPP){-+..}, at: [<c05c4cb1>] __qdisc_run+0x5b/0x156
Feb 20 22:11:45 localhost kernel: #8: (&ppp->wlock){-+..}, at: [<f8b9ef10>] ppp_xmit_process+0x15/0x5a1 [ppp_generic]
Feb 20 22:11:45 localhost kernel: #9: (&pch->downl){-+..}, at: [<f8b9e6db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 20 22:11:45 localhost kernel:
Feb 20 22:11:45 localhost kernel: stack backtrace:
Feb 20 22:11:45 localhost kernel: Pid: 3744, comm: pppd Not tainted 2.6.24.2 #1
Feb 20 22:11:45 localhost kernel: [<c0447933>] print_usage_bug+0x139/0x143
Feb 20 22:11:45 localhost kernel: [<c04482d2>] mark_lock+0x1cb/0x454
Feb 20 22:11:45 localhost kernel: [<c04480a2>] find_usage_forwards+0x67/0x8b
Feb 20 22:11:45 localhost kernel: [<c0448fdf>] __lock_acquire+0x424/0xbf1
Feb 20 22:11:46 localhost kernel: [<c0448b8e>] check_noncircular+0x66/0x93
Feb 20 22:11:46 localhost kernel: [<c0448594>] mark_held_locks+0x39/0x53
Feb 20 22:11:46 localhost kernel: [<c047a1bd>] kfree+0x9f/0xa6
Feb 20 22:11:46 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 20 22:11:46 localhost kernel: [<c044981c>] lock_acquire+0x70/0x8a
Feb 20 22:11:46 localhost kernel: [<f8bac857>] pppol2tp_xmit+0x320/0x3d9 [pppol2tp]
Feb 20 22:11:46 localhost kernel: [<c061b3a7>] _read_lock+0x29/0x34
Feb 20 22:11:46 localhost kernel: [<f8bac857>] pppol2tp_xmit+0x320/0x3d9 [pppol2tp]
Feb 20 22:11:46 localhost kernel: [<f8bac857>] pppol2tp_xmit+0x320/0x3d9 [pppol2tp]
Feb 20 22:11:46 localhost kernel: [<f8b9e6db>] ppp_push+0x63/0x50d [ppp_generic]
Feb 20 22:11:46 localhost kernel: [<c061b293>] _spin_lock_bh+0x2e/0x39
Feb 20 22:11:46 localhost kernel: [<f8b9e6ee>] ppp_push+0x76/0x50d [ppp_generic]
Feb 20 22:11:46 localhost kernel: [<c061b622>] _spin_unlock_irqrestore+0x34/0x39
Feb 20 22:11:46 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 20 22:11:46 localhost kernel: [<f8b9f3f1>] ppp_xmit_process+0x4f6/0x5a1 [ppp_generic]
Feb 20 22:11:46 localhost kernel: [<c0448771>] trace_hardirqs_on+0x10c/0x14c
Feb 20 22:11:46 localhost kernel: [<f8b9f5d2>] ppp_start_xmit+0x136/0x167 [ppp_generic]
Feb 20 22:11:46 localhost kernel: [<c05b5d29>] dev_hard_start_xmit+0x245/0x29f
Feb 20 22:11:46 localhost kernel: [<c061b25a>] _spin_lock+0x29/0x34
Feb 20 22:11:46 localhost kernel: [<c05c4cc0>] __qdisc_run+0x6a/0x156
Feb 20 22:11:46 localhost kernel: [<c05b7ee0>] dev_queue_xmit+0x1a5/0x2e9
Feb 20 22:11:46 localhost kernel: [<c05b7e60>] dev_queue_xmit+0x125/0x2e9
Feb 20 22:11:46 localhost kernel: [<c05d62f5>] ip_finish_output+0x1f4/0x21e
Feb 20 22:11:46 localhost kernel: [<c05d4f78>] ip_push_pending_frames+0x2dc/0x33b
Feb 20 22:11:46 localhost kernel: [<c05f2de5>] icmp_reply+0x116/0x1a4
Feb 20 22:11:46 localhost kernel: [<c05f32b6>] icmp_echo+0x49/0x4d
Feb 20 22:11:46 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 20 22:11:46 localhost kernel: [<c05f2f67>] icmp_rcv+0xf4/0x116
Feb 20 22:11:46 localhost kernel: [<c05d2c4b>] ip_local_deliver_finish+0x13f/0x1f8
Feb 20 22:11:46 localhost kernel: [<c05d2b3a>] ip_local_deliver_finish+0x2e/0x1f8
Feb 20 22:11:46 localhost kernel: [<c05d2ad2>] ip_rcv_finish+0x2fe/0x338
Feb 20 22:11:46 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 20 22:11:46 localhost kernel: [<c05d2dc6>] ip_rcv+0x0/0x237
Feb 20 22:11:46 localhost kernel: [<c05b5681>] netif_receive_skb+0x373/0x3d4
Feb 20 22:11:46 localhost kernel: [<c05b5402>] netif_receive_skb+0xf4/0x3d4
Feb 20 22:11:46 localhost kernel: [<c05b7c53>] process_backlog+0x6c/0xb9
Feb 20 22:11:46 localhost kernel: [<c05b77f0>] net_rx_action+0xbc/0x1b3
Feb 20 22:11:46 localhost kernel: [<c05b7782>] net_rx_action+0x4e/0x1b3
Feb 20 22:11:46 localhost kernel: [<c0430a9a>] __do_softirq+0x69/0xde
Feb 20 22:11:46 localhost kernel: [<c040709f>] do_softirq+0x5e/0xc7
Feb 20 22:11:47 localhost kernel: [<c05ceed4>] rt_run_flush+0x64/0x8b
Feb 20 22:11:47 localhost kernel: [<c05ceed4>] rt_run_flush+0x64/0x8b
Feb 20 22:11:47 localhost kernel: [<c04308ee>] local_bh_enable_ip+0xad/0xd5
Feb 20 22:11:47 localhost kernel: [<c05ceed4>] rt_run_flush+0x64/0x8b
Feb 20 22:11:47 localhost kernel: [<c05fa7e7>] fib_disable_ip+0x1e/0x26
Feb 20 22:11:47 localhost kernel: [<c05faf99>] fib_inetaddr_event+0x19c/0x1b0
Feb 20 22:11:47 localhost kernel: [<c061d06c>] notifier_call_chain+0x2a/0x47
Feb 20 22:11:47 localhost kernel: [<c04411e5>] __blocking_notifier_call_chain+0x3e/0x51
Feb 20 22:11:47 localhost kernel: [<c044120f>] blocking_notifier_call_chain+0x17/0x1a
Feb 20 22:11:47 localhost kernel: [<c05f450b>] __inet_del_ifa+0x12f/0x1f0
Feb 20 22:11:47 localhost kernel: [<c05f45e3>] inet_del_ifa+0x17/0x1a
Feb 20 22:11:47 localhost kernel: [<c05f5716>] devinet_ioctl+0x44d/0x539
Feb 20 22:11:47 localhost kernel: [<c05b7188>] dev_ioctl+0x46f/0x5e0
Feb 20 22:11:47 localhost kernel: [<c05ab078>] sock_ioctl+0x1bb/0x1e0
Feb 20 22:11:47 localhost kernel: [<c05aaebd>] sock_ioctl+0x0/0x1e0
Feb 20 22:11:47 localhost kernel: [<c0488827>] do_ioctl+0x1f/0x62
Feb 20 22:11:47 localhost kernel: [<c0458df0>] audit_syscall_entry+0x10d/0x137
Feb 20 22:11:47 localhost kernel: [<c0488aa1>] vfs_ioctl+0x237/0x249
Feb 20 22:11:47 localhost kernel: [<c0488af8>] sys_ioctl+0x45/0x5d
Feb 20 22:11:47 localhost kernel: [<c0404efa>] syscall_call+0x7/0xb
Feb 20 22:11:47 localhost kernel: =======================Quoted 15 lines · James Chapman
Quoted 3 lines · Jarek Poplawski
Quoted 1 line
Quoted 1 line
(testing patch #1)
But I hope you tested with the fixed (take 2) version of this patch...
Yes I did. :)
But I just got another lockdep error (attached).
Quoted 5 lines
Since it's quite experimental (testing) this patch could be wrong as it is, but I hope it should show the proper way to solve this problem. Probably you did some of these, but here are a few of my suggestions for testing this:
Quoted 6 lines
Quoted 4 lines
1) try my patch with your full bh locking changing patch;
2) add while loops to these trylocks on failure, with e.g.__delay(1); this should work like full locks again, but there should be no (this kind of) lockdep reports;Hmm, isn't this just bypassing the lockdep checks?
Yes! But it's only for debugging: to find if this change in locking is to be blamed for these new lockups. It should effectively work just like without this patch, but without this lockdep warning. So, if after such change lockups still happen, then it would seem you didn't test this enough before. Otherwise the new patch is to blame and needs reworking.
Quoted 7 lines
Quoted 4 lines
3) I send here another testing patch with this second way to do this:
on the write side, but it's even more "experimental" and only a proof of concept (should be applied on vanillappp_generic).I'll look over it. I think I need to take a step back and look at what's happening in more detail though.
This is something completely new and changes all the picture: the xmit path wasn't expected (at least by me) to be called in softirq context at all, and there were no traces of this on previous reports. But, since lockdep always stops after the first warning, there could be even more surprises like this in the future. I'll check this report.
Regards,
Jarek P.
Quoted 18 lines · Jarek Poplawski
Quoted 17 lines · James Chapman
Quoted 2 lines · Jarek Poplawski
Quoted 1 line
Quoted 1 line
(testing patch #1)
But I hope you tested with the fixed (take 2) version of this patch...
Yes I did. :)
But I just got another lockdep error (attached).
Quoted 9 lines
Since it's quite experimental (testing) this patch could be wrong as it is, but I hope it should show the proper way to solve this problem. Probably you did some of these, but here are a few of my suggestions for testing this:
1) try my patch with your full bh locking changing patch;
2) add while loops to these trylocks on failure, with e.g.__delay(1); this should work like full locks again, but there should be no (this kind of) lockdep reports;
Quoted 8 lines
Quoted 1 line
Hmm, isn't this just bypassing the lockdep checks?
Yes! But it's only for debugging: to find if this change in locking is to be blamed for these new lockups. It should effectively work just like without this patch, but without this lockdep warning. So, if after such change lockups still happen, then it would seem you didn't test this enough before. Otherwise the new patch is to blame and needs reworking.
The lockups still happen, but I think they are now due to a different problem, as you say.
Some background on this issue might be useful to help get feedback from others on the list. This issue was first reported by an ISP who found random lockups if an L2TP tunnel carrying hundreds/thousands of L2TP sessions went down due to a network outage and then recovered itself. On recovery, all of the tunnel's sessions (PPP) are created rapidly. Sometimes the tunnel would recover just fine, but other times not. The ISP put some effort into reproducing the problem and found that repeatedly creating/deleting a tunnel with lots of L2TP sessions would cause the failure after a random time between a few minutes and several hours. The original lockdep trace came from the ISP. I initially couldn't reproduce the problem but I borrowed two equivalent quad-core systems and can now reproduce it. Subsequent lockdep traces have been from my testing.
The _bh locking fixes in pppol2tp combined with your ppp_generic change solved that problem. So I then added data traffic into the mix (since this will happen in a real network) and found that lockups still happen. But the lockdep trace in this case is different, as you noted.
Does PPPoE stress the PPP setup code as much as this scenario? I guess in theory it could if lots of PPPoE clients connected at the same time, but there is no aggregate tunnel like there is with L2TP to cause all sessions to connect simultaneously. Perhaps PPTP also suffers from these issues? Perhaps not because it tends to be used only in VPN setups where there is only 1 session per tunnel.
Quoted 10 lines
Quoted 4 lines
Quoted 3 lines
3) I send here another testing patch with this second way to do this:
on the write side, but it's even more "experimental" and only a proof of concept (should be applied on vanillappp_generic).I'll look over it. I think I need to take a step back and look at what's happening in more detail though.
This is something completely new and changes all the picture: the xmit path wasn't expected (at least by me) to be called in softirq context at all, and there were no traces of this on previous reports. But, since lockdep always stops after the first warning, there could be even more surprises like this in the future. I'll check this report.
Doesn't the TX softirq do transmits if they've been queued up?
Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
Quoted 1 line · James Chapman
Jarek Poplawski wrote:
...
Quoted 4 lines
The
_bhlocking fixes in pppol2tp combined with yourppp_genericchange solved that problem. So I then added data traffic into the mix (since this will happen in a real network) and found that lockups still happen. But the lockdep trace in this case is different, as you noted.
I'm not sure what do you mean by "solved that problem": lack of lockups or lack of this kind of lockdep reports. This lockdep report shows a real danger in this case, probably very little probable, unless a lot of tries. So if you think just this kind of lockup happend there, this is would be nice. But there could be something less nice too: we are "fighting" with lockdep in one place but the lockup happens somewhere else for some totally different reason...
Quoted 6 lines
Does PPPoE stress the PPP setup code as much as this scenario? I guess in theory it could if lots of PPPoE clients connected at the same time, but there is no aggregate tunnel like there is with L2TP to cause all sessions to connect simultaneously. Perhaps PPTP also suffers from these issues? Perhaps not because it tends to be used only in VPN setups where there is only 1 session per tunnel.
I don't know this code enough, but it seems it should be easier to maintain or debug if there are similar solutions where possible.
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
Quoted 3 lines
3) I send here another testing patch with this second way to do this:
on the write side, but it's even more "experimental" and only a proof of concept (should be applied on vanillappp_generic).I'll look over it. I think I need to take a step back and look at what's happening in more detail though.
Quoted 7 lines
Quoted 5 lines
This is something completely new and changes all the picture: the xmit path wasn't expected (at least by me) to be called in softirq context at all, and there were no traces of this on previous reports. But, since lockdep always stops after the first warning, there could be even more surprises like this in the future. I'll check this report.
Doesn't the TX softirq do transmits if they've been queued up?
I've probably too much looked at these reports, and should've expected this could happen. Probably queueing could be separated, but since there could be no queue at all and it's done like this, then this current proof of concept seems to be dead end, and we have to go back to fixing sk_dst_lock handling and my patches could be dumped...
So if I don't miss something again (and I need more time for this new report) you should try to fix the problem not reported originally by lockdep, but forseen by David(!): we need to avoid any path like:ppp_generic -> pppol2tp -> something, which could take sk_dst_lock while holding any ppp_generic writing lock: they all are softirq "safe" (i.e. endangered). David gave some example, but I'm not sure you did your patch like this (sk_dst_set()). Probably ip_queue_xmit() can't work with this too.
Another, probably simpler way would be to move almost all pppol2tp_xmit code to a workqueue: this should let to break most of dependencies with ppp_generic locks, but I don't know how much it would affect other things (e.g. performance). So you should really rethink these things.
Jarek P.
Jarek Poplawski wrote, On 02/21/2008 01:08 PM:
...
Quoted 1 line
Another, probably simpler way would be to move almost all
pppol2tp_xmit
...
Actually, the simplest off all seems to be now this old idea to maybe make sk_dst_lock globally softirq immune. At least I think it's worth of testing, to check for these other possible lockdep warnings. It should only need to change all write_ and read_lock(&sk->sk_dst_lock) in very few places:include/net/sock.h, include/net/ip6_route.h, and net/ipv6/ipv6_sockglue.c. This could be tested together with you full _bh locking patch (maybe except these other changes in pppol2tp_xmit).
Jarek P.
Quoted 6 lines · Jarek Poplawski
Jarek Poplawski wrote, On 02/21/2008 01:08 PM:
...Quoted 1 line
Another, probably simpler way would be to move almost all
pppol2tp_xmit...
Quoted 7 lines
Actually, the simplest off all seems to be now this old idea to maybe make
sk_dst_lockglobally softirq immune. At least I think it's worth of testing, to check for these other possible lockdep warnings. It should only need to change allwrite_andread_lock(&sk->sk_dst_lock)in very few places:include/net/sock.h,include/net/ip6_route.h, andnet/ipv6/ipv6_sockglue.c. This could be tested together with you full_bhlocking patch (maybe except these other changes inpppol2tp_xmit).
I did this and all lockdep errors have now gone. Tests ran all weekend. See attached patch.
Is this an acceptable solution? If so, I'll prepare and send official patches.
Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
include/net/ip6_route.h +2 -2
@@ -150,9 +150,9 @@ static inline void __ip6_dst_store(strucstatic inline void ip6_dst_store (struct sock *sk,struct dst_entry *dst,struct in6_addr *daddr,struct in6_addr *saddr) { -write_lock (&sk->sk_dst_lock); +write_lock_bh (&sk->sk_dst_lock);__ip6_dst_store (sk, dst, daddr, saddr); -write_unlock (&sk->sk_dst_lock); +write_unlock_bh (&sk->sk_dst_lock); }static inline int ipv6_unicast_destination (struct sk_buff *skb)
include/net/sock.h +6 -6
@@ -1058,11 +1058,11 @@ sk_dst_get(struct sock *sk) {struct dst_entry *dst; -read_lock (&sk->sk_dst_lock); +read_lock_bh (&sk->sk_dst_lock); dst = sk->sk_dst_cache;if (dst)dst_hold (dst); -read_unlock (&sk->sk_dst_lock); +read_unlock_bh (&sk->sk_dst_lock);return dst; }
@@ -1079,9 +1079,9 @@ __sk_dst_set(struct sock *sk, struct dststatic inline void sk_dst_set (struct sock *sk,struct dst_entry *dst) { -write_lock (&sk->sk_dst_lock); +write_lock_bh (&sk->sk_dst_lock);__sk_dst_set (sk, dst); -write_unlock (&sk->sk_dst_lock); +write_unlock_bh (&sk->sk_dst_lock); } static inline void
@@ -1097,9 +1097,9 @@ __sk_dst_reset(struct sock *sk)static inline void sk_dst_reset (struct sock *sk) { -write_lock (&sk->sk_dst_lock); +write_lock_bh (&sk->sk_dst_lock);__sk_dst_reset (sk); -write_unlock (&sk->sk_dst_lock); +write_unlock_bh (&sk->sk_dst_lock); }extern struct dst_entry *__sk_dst_check (struct sock *sk,u32 cookie);
net/ipv6/ipv6_sockglue.c +4 -4
@@ -440,9 +440,9 @@ static int do_ipv6_setsockopt(struct soc opt =xchg (&np->opt, opt);sk_dst_reset (sk); }else { -write_lock (&sk->sk_dst_lock); +write_lock_bh (&sk->sk_dst_lock); opt =xchg (&np->opt, opt); -write_unlock (&sk->sk_dst_lock); +write_unlock_bh (&sk->sk_dst_lock);sk_dst_reset (sk); } sticky_done:
@@ -504,9 +504,9 @@ update: opt =xchg (&np->opt, opt);sk_dst_reset (sk); }else { -write_lock (&sk->sk_dst_lock); +write_lock_bh (&sk->sk_dst_lock); opt =xchg (&np->opt, opt); -write_unlock (&sk->sk_dst_lock); +write_unlock_bh (&sk->sk_dst_lock);sk_dst_reset (sk); }
Quoted 15 lines · James Chapman
Quoted 6 lines · Jarek Poplawski
Jarek Poplawski wrote, On 02/21/2008 01:08 PM:
...Quoted 1 line
Another, probably simpler way would be to move almost all
pppol2tp_xmit...
Quoted 7 lines
Actually, the simplest off all seems to be now this old idea to maybe make
sk_dst_lockglobally softirq immune. At least I think it's worth of testing, to check for these other possible lockdep warnings. It should only need to change allwrite_andread_lock(&sk->sk_dst_lock)in very few places:include/net/sock.h,include/net/ip6_route.h, andnet/ipv6/ipv6_sockglue.c. This could be tested together with you full_bhlocking patch (maybe except these other changes inpppol2tp_xmit).
Quoted 5 lines
I did this and all lockdep errors have now gone. Tests ran all weekend. See attached patch.
Is this an acceptable solution? If so, I'll prepare and send official patches.
IMHO this should be acceptable because I can't see any reason for changing properly working code if there is so simple and not costly solution. But maybe David or somebody else finds some cons? Since this patch isn't very big I think you could try to send this officially and we will simply see...
Regards,
Jarek P.
Quoted 95 lines
Signature · 9 lines
-- James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development Index: linux-2.6.24.2/include/net/ip6_route.h ===================================================================include/net/ip6_route.h +2
-2@@ -150,9 +150,9 @@ static inline void __ip6_dst_store(strucstatic inline void ip6_dst_store (struct sock *sk,struct dst_entry *dst,struct in6_addr *daddr,struct in6_addr *saddr) { -write_lock (&sk->sk_dst_lock); +write_lock_bh (&sk->sk_dst_lock);__ip6_dst_store (sk, dst, daddr, saddr); -write_unlock (&sk->sk_dst_lock); +write_unlock_bh (&sk->sk_dst_lock); }static inline int ipv6_unicast_destination (struct sk_buff *skb)Index:
linux-2.6.24.2/include/net/sock.h
===================================================================include/net/sock.h +6
-6@@ -1058,11 +1058,11 @@ sk_dst_get(struct sock *sk) {struct dst_entry *dst; -read_lock (&sk->sk_dst_lock); +read_lock_bh (&sk->sk_dst_lock); dst = sk->sk_dst_cache;if (dst)dst_hold (dst); -read_unlock (&sk->sk_dst_lock); +read_unlock_bh (&sk->sk_dst_lock);return dst; }@@ -1079,9 +1079,9 @@ __sk_dst_set(struct sock *sk, struct dststatic inline void sk_dst_set (struct sock *sk,struct dst_entry *dst) { -write_lock (&sk->sk_dst_lock); +write_lock_bh (&sk->sk_dst_lock);__sk_dst_set (sk, dst); -write_unlock (&sk->sk_dst_lock); +write_unlock_bh (&sk->sk_dst_lock); } static inline void@@ -1097,9 +1097,9 @@ __sk_dst_reset(struct sock *sk)static inline void sk_dst_reset (struct sock *sk) { -write_lock (&sk->sk_dst_lock); +write_lock_bh (&sk->sk_dst_lock);__sk_dst_reset (sk); -write_unlock (&sk->sk_dst_lock); +write_unlock_bh (&sk->sk_dst_lock); }extern struct dst_entry *__sk_dst_check (struct sock *sk,u32 cookie);Index:
linux-2.6.24.2/net/ipv6/ipv6_sockglue.c
===================================================================net/ipv6/ipv6_sockglue.c +4
-4@@ -440,9 +440,9 @@ static int do_ipv6_setsockopt(struct soc opt =xchg (&np->opt, opt);sk_dst_reset (sk); }else { -write_lock (&sk->sk_dst_lock); +write_lock_bh (&sk->sk_dst_lock); opt =xchg (&np->opt, opt); -write_unlock (&sk->sk_dst_lock); +write_unlock_bh (&sk->sk_dst_lock);sk_dst_reset (sk); } sticky_done:@@ -504,9 +504,9 @@ update: opt =xchg (&np->opt, opt);sk_dst_reset (sk); }else { -write_lock (&sk->sk_dst_lock); +write_lock_bh (&sk->sk_dst_lock); opt =xchg (&np->opt, opt); -write_unlock (&sk->sk_dst_lock); +write_unlock_bh (&sk->sk_dst_lock);sk_dst_reset (sk); }
On Mon, Feb 25, 2008 at 01:05:08PM +0000, Jarek Poplawski wrote:
...
Quoted 7 lines · James Chapman
Quoted 2 lines
Is this an acceptable solution? If so, I'll prepare and send official patches.
IMHO this should be acceptable because I can't see any reason for changing properly working code if there is so simple and not costly solution. But maybe David or somebody else finds some cons? [...]
Hmm... Wait a minute! But on the other hand David has written about his cons here, and it looks reasonable: this place would be fixed, but some others can start reports like this. Maybe, it's better to analyze yet if it's really so hard to eliminate taking this lock on the xmit path?
Jarek P.
On Mon, Feb 25, 2008 at 01:39:48PM +0000, Jarek Poplawski wrote:
...
Quoted 3 lines
Maybe, it's better to
analyze yet if it's really so hard to eliminate taking this lock on the xmit path?
BTW, I'm not sure if it helps, but this matters only for the sockets which could be used (and locked) outside of pppol2tp code (so before pppol2tp code is called).
Jarek P.
Jarek Poplawski wrote, On 02/25/2008 02:39 PM:
...
Quoted 5 lines
Hmm... Wait a minute! But on the other hand David has written about his cons here, and it looks reasonable: this place would be fixed, but some others can start reports like this. Maybe, it's better to analyze yet if it's really so hard to eliminate taking this lock on the xmit path?
James, I wonder if you could try to test this patch below? ip_queue_xmit() seems to do proper things with __sk_dst_check(), and if some other functions don't behave similarly lockdep should tell. I think, you could test it with your "locks to _bh" patch (without pppol2tp_xmit() part), and maybe with my sock.c lockdep patch (it should help lockdep to see these locks a bit more distinctly).
Jarek P.
PS: Since ppp_generic isn't endangered for now I removed Paul from CC.
---
drivers/net/pppol2tp.c +1 -1
@@ -1058,7 +1058,7 @@ static int pppol2tp_xmit(struct ppp_channel *chan, struct sk_buff *skb)/* Get routing info from the tunnel socket */ dst_release (skb->dst); - skb->dst =sk_dst_get (sk_tun); + skb->dst =NULL ;skb_orphan (skb); skb->sk = sk_tun;
Quoted 8 lines · Jarek Poplawski
Jarek Poplawski wrote, On 02/25/2008 02:39 PM:
...Quoted 5 lines
Hmm... Wait a minute! But on the other hand David has written about his cons here, and it looks reasonable: this place would be fixed, but some others can start reports like this. Maybe, it's better to analyze yet if it's really so hard to eliminate taking this lock on the xmit path?
Quoted 6 lines
James, I wonder if you could try to test this patch below?
ip_queue_xmit()seems to do proper things with__sk_dst_check(), and if some other functions don't behave similarly lockdep should tell. I think, you could test it with your "locks to_bh" patch (withoutpppol2tp_xmit()part), and maybe with mysock.clockdep patch (it should help lockdep to see these locks a bit more distinctly).
I found the same thing and was running a variant of your patch last night; rather than set skb->dst to NULL though, I use __sk_dst_get() and let ip_queue_xmit() do the route lookup if it returns NULL. But this has the same symptoms as the code I tried a few days ago - no lockdep errors but a system lockup after up to several hours. Nothing is logged in the syslog.
Luckily, I'm in the lab where my two borrowed servers are today so I have access to their consoles. Hopefully I'll be able to find out why there are hanging. Btw, they don't hang if I disable irqs around the ppp_input() call.
Will update you later.
/james
Quoted 22 lines
PS: Since
ppp_genericisn't endangered for now I removed Paul from CC.---
drivers/net/pppol2tp.c +1
-2@@ -1058,7 +1058,7 @@ static int pppol2tp_xmit(struct ppp_channel *chan, struct sk_buff *skb)/* Get routing info from the tunnel socket */ dst_release (skb->dst); - skb->dst =sk_dst_get (sk_tun); + skb->dst =NULL ;skb_orphan (skb); skb->sk = sk_tun; --To unsubscribe from this list: send the line "unsubscribe netdev" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
Quoted 16 lines · James Chapman
Quoted 8 lines · Jarek Poplawski
Jarek Poplawski wrote, On 02/25/2008 02:39 PM:
...Quoted 5 lines
Hmm... Wait a minute! But on the other hand David has written about his cons here, and it looks reasonable: this place would be fixed, but some others can start reports like this. Maybe, it's better to analyze yet if it's really so hard to eliminate taking this lock on the xmit path?
Quoted 6 lines
James, I wonder if you could try to test this patch below?
ip_queue_xmit()seems to do proper things with__sk_dst_check(), and if some other functions don't behave similarly lockdep should tell. I think, you could test it with your "locks to_bh" patch (withoutpppol2tp_xmit()part), and maybe with mysock.clockdep patch (it should help lockdep to see these locks a bit more distinctly).
Quoted 6 lines
I found the same thing and was running a variant of your patch last night; rather than set
skb->dstto NULL though, I use__sk_dst_get()and letip_queue_xmit()do the route lookup if it returns NULL. But this has the same symptoms as the code I tried a few days ago - no lockdep errors but a system lockup after up to several hours. Nothing is logged in the syslog.
I guess you are going to try this together with this sk_dst_lock with bh patch too. If it's possible I'd suggest to try this skb->dst = NULL as well (__sk_dst_get instead of __sk_dst_check seems to be too racy).
Quoted 4 lines
Luckily, I'm in the lab where my two borrowed servers are today so I have access to their consoles. Hopefully I'll be able to find out why there are hanging. Btw, they don't hang if I disable irqs around the
ppp_input()call.
...and disabling bh instead isn't enough, BTW?
Quoted 1 line
Will update you later.
Thanks,
Jarek P.
Quoted 1 line · Jarek Poplawski
On Tue, Feb 26, 2008 at 12:14:26PM +0000, James Chapman wrote:
...
Quoted 4 lines
Quoted 2 lines
there are hanging. Btw, they don't hang if I disable irqs around the
ppp_input()call....and disabling bh instead isn't enough, BTW?
I guess not: they are mostly disabled by ppp_input() itself...
So, it looks like a network card could mess here?
Jarek P.
James Chapman wrote, On 02/26/2008 01:14 PM:
...
Quoted 4 lines
Luckily, I'm in the lab where my two borrowed servers are today so I have access to their consoles. Hopefully I'll be able to find out why there are hanging. Btw, they don't hang if I disable irqs around the
ppp_input()call.
Maybe you've found the same, or there is some other reason yet, but IMHO this locking break around ppp_input() is wrong. Probably there is needed more advanced solution, but this should fix the problem if it really exists (isn't there possible a race e.g. between receive from socket and from network card?).
Jarek P.
---
drivers/net/pppol2tp.c | 16 ++++++++-------- 1 files changed, 8 insertions(+), 8 deletions(-)
drivers/net/pppol2tp.c +8 -8
@@ -363,18 +363,17 @@ out:spin_unlock (&session->reorder_q.lock); } -/* Dequeue a single skb. +/* Requeue a single skb. */ -static void pppol2tp_recv_dequeue_skb (struct pppol2tp_session *session,struct sk_buff *skb) +static void pppol2tp_recv_requeue_skb (struct pppol2tp_session *session,struct sk_buff *skb) {struct pppol2tp_tunnel *tunnel = session->tunnel;int length =PPPOL2TP_SKB_CB (skb)->length;struct sock *session_sock =NULL ; -/* We're about to requeue the skb, so unlink it and return resources +/* We're about to requeue the skb, so return resources * to its current owner (a socket receive buffer). */ -skb_unlink (skb, &session->reorder_q);skb_orphan (skb); tunnel->stats.rx_packets++;
@@ -436,14 +435,14 @@ static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct sstatic void pppol2tp_recv_dequeue (struct pppol2tp_session *session) {struct sk_buff *skb; -struct sk_buff *tmp;/* If the pkt at the head of the queue has the nr that we * expect to send up next, dequeue it and any other * in-sequence packets behind it. */ +again:spin_lock (&session->reorder_q.lock); - skb_queue_walk_safe(&session->reorder_q, skb, tmp) { + skb_queue_walk(&session->reorder_q, skb) {if (time_after (jiffies,PPPOL2TP_SKB_CB (skb)->expires)) { session->stats.rx_seq_discards++; session->stats.rx_errors++;
@@ -469,9 +468,10 @@ static void pppol2tp_recv_dequeue(struct pppol2tp_session *session) goto out; } } +__skb_unlink (skb, &session->reorder_q);spin_unlock (&session->reorder_q.lock); -pppol2tp_recv_dequeue_skb (session, skb); -spin_lock (&session->reorder_q.lock); +pppol2tp_recv_requeue_skb (session, skb); + goto again; } out:
Quoted 9 lines · Jarek Poplawski
James Chapman wrote, On 02/26/2008 01:14 PM:
...Quoted 1 line
Luckily, I'm in the lab where my two borrowed servers are today so I have access to their consoles. Hopefully I'll be able to find out why there are hanging. Btw, they don't hang if I disable irqs around the
ppp_input()call.Maybe you've found the same, or there is some other reason yet, but IMHO this locking break around
ppp_input()is wrong. Probably there is needed more advanced solution, but this should fix the problem if it really exists (isn't there possible a race e.g. between receive from socket and from network card?).
I tried your patch but the result is the same.
Quoted 37 lines
Jarek P.
---drivers/net/pppol2tp.c | 16 ++++++++-------- 1 files changed, 8 insertions(+), 8 deletions(-)drivers/net/pppol2tp.c +3
-4@@ -363,18 +363,17 @@ out:spin_unlock (&session->reorder_q.lock); } -/* Dequeue a single skb. +/* Requeue a single skb. */ -static void pppol2tp_recv_dequeue_skb (struct pppol2tp_session *session,struct sk_buff *skb) +static void pppol2tp_recv_requeue_skb (struct pppol2tp_session *session,struct sk_buff *skb) {struct pppol2tp_tunnel *tunnel = session->tunnel;int length =PPPOL2TP_SKB_CB (skb)->length;struct sock *session_sock =NULL ; -/* We're about to requeue the skb, so unlink it and return resources +/* We're about to requeue the skb, so return resources * to its current owner (a socket receive buffer). */ -skb_unlink (skb, &session->reorder_q);skb_orphan (skb); tunnel->stats.rx_packets++;@@ -436,14 +435,14 @@ static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct sstatic void pppol2tp_recv_dequeue (struct pppol2tp_session *session) {struct sk_buff *skb;
drivers/net/pppol2tp.c static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct s · line 445
Quoted 10 lines
drivers/net/pppol2tp.c +2
-2-struct sk_buff *tmp;/* If the pkt at the head of the queue has the nr that we * expect to send up next, dequeue it and any other * in-sequence packets behind it. */ +again:spin_lock (&session->reorder_q.lock); - skb_queue_walk_safe(&session->reorder_q, skb, tmp) { + skb_queue_walk(&session->reorder_q, skb) {
I think this needs the _safe() call because the list may be modified in the loop body.
drivers/net/pppol2tp.c static void pppol2tp_recv_dequeue(struct pppol2tp_session *session) · line 474
Quoted 13 lines
drivers/net/pppol2tp.c +3
-2if (time_after (jiffies,PPPOL2TP_SKB_CB (skb)->expires)) { session->stats.rx_seq_discards++; session->stats.rx_errors++;@@ -469,9 +468,10 @@ static void pppol2tp_recv_dequeue(struct pppol2tp_session *session) goto out; } } +__skb_unlink (skb, &session->reorder_q);spin_unlock (&session->reorder_q.lock); -pppol2tp_recv_dequeue_skb (session, skb); -spin_lock (&session->reorder_q.lock); +pppol2tp_recv_requeue_skb (session, skb); + goto again;
I'd prefer to take the spinlock again after the recv_dequeue_skb() call to avoid the goto.
Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
Quoted 2 lines · James Chapman
Quoted 1 line · Jarek Poplawski
James Chapman wrote, On 02/26/2008 01:14 PM:
...
Quoted 41 lines
I tried your patch but the result is the same.
Quoted 38 lines
Jarek P.
---drivers/net/pppol2tp.c | 16 ++++++++-------- 1 files changed, 8 insertions(+), 8 deletions(-)drivers/net/pppol2tp.c +3
-3@@ -363,18 +363,17 @@ out:spin_unlock (&session->reorder_q.lock); } -/* Dequeue a single skb. +/* Requeue a single skb. */ -static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct sk_buff *skb) +static void pppol2tp_recv_requeue_skb(struct pppol2tp_session *session, struct sk_buff *skb) {struct pppol2tp_tunnel *tunnel = session->tunnel;int length =PPPOL2TP_SKB_CB (skb)->length;struct sock *session_sock =NULL ; -/* We're about to requeue the skb, so unlink it and return resources +/* We're about to requeue the skb, so return resources * to its current owner (a socket receive buffer). */ -skb_unlink (skb, &session->reorder_q);skb_orphan (skb); tunnel->stats.rx_packets++;@@ -436,14 +435,14 @@ static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct sstatic void pppol2tp_recv_dequeue (struct pppol2tp_session *session) {struct sk_buff *skb; -struct sk_buff *tmp;/* If the pkt at the head of the queue has the nr that we * expect to send up next, dequeue it and any other * in-sequence packets behind it.
Quoted 8 lines
drivers/net/pppol2tp.c static void pppol2tp_recv_dequeue_skb(struct pppol2tp_session *session, struct s · line 445
Quoted 5 lines
drivers/net/pppol2tp.c +2
-1*/ +again:spin_lock (&session->reorder_q.lock); - skb_queue_walk_safe(&session->reorder_q, skb, tmp) { + skb_queue_walk(&session->reorder_q, skb) {I think this needs the
_safe()call because the list may be modified in the loop body.
And you are very right.
Quoted 17 lines
Quoted 5 lines
drivers/net/pppol2tp.c +0
-0if (time_after (jiffies,PPPOL2TP_SKB_CB (skb)->expires)) { session->stats.rx_seq_discards++; session->stats.rx_errors++;drivers/net/pppol2tp.c static void pppol2tp_recv_dequeue(struct pppol2tp_session *session) · line 474
Quoted 9 lines
drivers/net/pppol2tp.c +3
-2@@ -469,9 +468,10 @@ static void pppol2tp_recv_dequeue(struct pppol2tp_session *session) goto out; } } +__skb_unlink (skb, &session->reorder_q);spin_unlock (&session->reorder_q.lock); -pppol2tp_recv_dequeue_skb (session, skb); -spin_lock (&session->reorder_q.lock); +pppol2tp_recv_requeue_skb (session, skb); + goto again;I'd prefer to take the spinlock again after the
recv_dequeue_skb()call to avoid the goto.
IMHO we have to do something like goto here because during the unlock there could be everything changed (including tmp) in this list, but maybe I miss something.
Anyway, this place seems to need more locking, e.g.: session->nr++ or some other session accesses in pppol2tp_recv_dequeue_skb() are done without locking. Maybe it would be interesting to try (in a test only) with calling this with reorder_q.lock held (so without unlocking inside this pppol2tp_recv_dequeue's loop at all: at least to see possible lockdep report. I can't see how this irq disabling could help for anything but some recursion during those misterious lockups.
BTW, since this bug is so hard to trigger, and it can take some time yet, I think it would be better to merge to David's 2.6.26 tree at least some parts of your "bh" patch and maybe other changes, which don't seem to make it worse at least, to try new things on some common base?
Regards,
Jarek P.
On Mon, Mar 03, 2008 at 08:22:04AM +0000, Jarek Poplawski wrote:
...
Quoted 4 lines
Maybe it would be interesting to try (in a test only) with calling this with reorder_q.lock held (so without unlocking inside this pppol2tp_recv_dequeue's loop at all: at least to see possible lockdep report.
...But we can't forget to change this skb_unlink() in pppol2tp_recv_dequeue_skb() then...
Jarek P.
PATCH 1/1 add missing sock_put() in pppol2tp_recv_dequeue()
In reply to James ChapmanHi,
Here is some tiny BTW fix (probably not connected with these lockups).
Regards,
Jarek P.
-------------->
Subject: [PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue()
Every skb removed from session->reorder_q needs sock_put().
---
drivers/net/pppol2tp.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-)
drivers/net/pppol2tp.c +1 -0
@@ -455,6 +455,7 @@ static void pppol2tp_recv_dequeue(struct pppol2tp_session *session) skb_queue_len(&session->reorder_q));__skb_unlink (skb, &session->reorder_q);kfree_skb (skb); +sock_put (session->sock);continue ; }
Re: [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue()
Quoted 6 lines · Jarek Poplawski
Subject: [PPPOL2TP] add missing
sock_put()inpppol2tp_recv_dequeue()Every skb removed from
session->reorder_qneedssock_put().Signed-off-by Jarek Poplawski <[email protected]>
Quoted 18 lines
---
drivers/net/pppol2tp.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-)drivers/net/pppol2tp.c +1
-0@@ -455,6 +455,7 @@ static void pppol2tp_recv_dequeue(struct pppol2tp_session *session) skb_queue_len(&session->reorder_q));__skb_unlink (skb, &session->reorder_q);kfree_skb (skb); +sock_put (session->sock);continue ; }
Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
David Miller
maintainerRe: [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_recv_dequeue()
From: James Chapman <[email protected]>
Date: Sun, 02 Mar 2008 20:31:17 +0000
Quoted 9 lines
Quoted 6 lines · Jarek Poplawski
Subject: [PPPOL2TP] add missing
sock_put()inpppol2tp_recv_dequeue()Every skb removed from
session->reorder_qneedssock_put().Signed-off-by Jarek Poplawski <[email protected]>Acked-by James Chapman <[email protected]>
Patch applied.
PATCH 1/1 add missing sock_put() in pppol2tp_tunnel_closeall()
In reply to James ChapmanAnd here is one more similar BTW fix.
Jarek P.
-------------->
Subject: [PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall()
Every skb removed from session->reorder_q needs sock_put().
---
drivers/net/pppol2tp.c | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-)
drivers/net/pppol2tp.c +6 -1
@@ -1110,6 +1110,8 @@ static void pppol2tp_tunnel_closeall(struct pppol2tp_tunnel *tunnel)for (hash =0 ; hash <PPPOL2TP_HASH_SIZE ; hash++) { again:hlist_for_each_safe (walk, tmp, &tunnel->session_hlist[hash]) { +struct sk_buff *skb; + session = hlist_entry(walk, struct pppol2tp_session, hlist); sk = session->sock;
@@ -1138,7 +1140,10 @@ again:/* Purge any queued data */ skb_queue_purge (&sk->sk_receive_queue);skb_queue_purge (&sk->sk_write_queue); -skb_queue_purge (&session->reorder_q); +while ((skb =skb_dequeue (&session->reorder_q))) { +kfree_skb (skb); +sock_put (sk); + }release_sock (sk);sock_put (sk);
Re: [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall()
Quoted 7 lines · Jarek Poplawski
Subject: [PPPOL2TP] add missing
sock_put()inpppol2tp_tunnel_closeall()Every skb removed from
session->reorder_qneedssock_put().Signed-off-by Jarek Poplawski <[email protected]>
Quoted 28 lines
drivers/net/pppol2tp.c | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-)drivers/net/pppol2tp.c +6
-1@@ -1110,6 +1110,8 @@ static void pppol2tp_tunnel_closeall(struct pppol2tp_tunnel *tunnel)for (hash =0 ; hash <PPPOL2TP_HASH_SIZE ; hash++) { again:hlist_for_each_safe (walk, tmp, &tunnel->session_hlist[hash]) { +struct sk_buff *skb; + session = hlist_entry(walk, struct pppol2tp_session, hlist); sk = session->sock;@@ -1138,7 +1140,10 @@ again:/* Purge any queued data */ skb_queue_purge (&sk->sk_receive_queue);skb_queue_purge (&sk->sk_write_queue); -skb_queue_purge (&session->reorder_q); +while ((skb =skb_dequeue (&session->reorder_q))) { +kfree_skb (skb); +sock_put (sk); + }release_sock (sk);sock_put (sk);
Signature · 4 lines
James Chapman Katalix Systems Ltd http://www.katalix.com Catalysts for your Embedded Linux software development
David Miller
maintainerRe: [PATCH][PPPOL2TP] add missing sock_put() in pppol2tp_tunnel_closeall()
From: James Chapman <[email protected]>
Date: Sun, 02 Mar 2008 20:32:41 +0000
Quoted 10 lines
Quoted 7 lines · Jarek Poplawski
Subject: [PPPOL2TP] add missing
sock_put()inpppol2tp_tunnel_closeall()Every skb removed from
session->reorder_qneedssock_put().Signed-off-by Jarek Poplawski <[email protected]>Acked-by James Chapman <[email protected]>
Also applied, thanks everyone.
PATCH 1/1 sock.c: sk_dst_lock lockdep keys and names per af_family
In reply to James ChapmanOn Thu, Feb 21, 2008 at 09:53:56AM +0000, James Chapman wrote:
...
Quoted 2 lines
The lockups still happen, but I think they are now due to a different problem, as you say.
...
I hope, this patch should help to remove some possibly false lockdep warnings during this current testing.
Regards,
Jarek P.
------------------->
Subject: [NET] sock.c: sk_dst_lock lockdep keys and names per af_family
Initialize sk_dst_lock lockdep keys and names per af_family.
Additionally some reorder is done in lockdep related code to keep it in one place and to use static key tables only when needed.
[not tested]
---
net/core/sock.c | 79 +++++++++++++++++++++++++++++++++++++++---------------- 1 files changed, 56 insertions(+), 23 deletions(-)
net/core/sock.c +56 -23
@@ -131,14 +131,17 @@#include <net/tcp.h> #endif +#ifdef CONFIG_DEBUG_LOCK_ALLOC +/* -* Each address family might have different locking rules, so we have -* one slock key per address family: +* Each address family might have different locking rules, so we have one: +* sk_lock, slock, sk_callback_lock and sk_dst_lock key per address family: */ static struct lock_class_key af_family_keys[AF_MAX ];static struct lock_class_key af_family_slock_keys[AF_MAX ]; +static struct lock_class_key af_family_callback_keys[AF_MAX ]; +static struct lock_class_key af_family_dst_keys[AF_MAX ]; -#ifdef CONFIG_DEBUG_LOCK_ALLOC /* * Make lock validator output more readable. (we pre-construct these * strings build-time, so that runtime initialization of socket
@@ -186,13 +189,52 @@ static const char *af_family_clock_key_strings[AF_MAX+1] = {"clock-AF_TIPC" ,"clock-AF_BLUETOOTH" ,"clock-AF_IUCV" ,"clock-AF_RXRPC" ,"clock-AF_MAX" }; -#endif +static const char *af_family_dst_key_strings[AF_MAX +1 ] = { +"sk_dst-AF_UNSPEC" ,"sk_dst-AF_UNIX" ,"sk_dst-AF_INET" , +"sk_dst-AF_AX25" ,"sk_dst-AF_IPX" ,"sk_dst-AF_APPLETALK" , +"sk_dst-AF_NETROM" ,"sk_dst-AF_BRIDGE" ,"sk_dst-AF_ATMPVC" , +"sk_dst-AF_X25" ,"sk_dst-AF_INET6" ,"sk_dst-AF_ROSE" , +"sk_dst-AF_DECnet" ,"sk_dst-AF_NETBEUI" ,"sk_dst-AF_SECURITY" , +"sk_dst-AF_KEY" ,"sk_dst-AF_NETLINK" ,"sk_dst-AF_PACKET" , +"sk_dst-AF_ASH" ,"sk_dst-AF_ECONET" ,"sk_dst-AF_ATMSVC" , +"sk_dst-21" ,"sk_dst-AF_SNA" ,"sk_dst-AF_IRDA" , +"sk_dst-AF_PPPOX" ,"sk_dst-AF_WANPIPE" ,"sk_dst-AF_LLC" , +"sk_dst-27" ,"sk_dst-28" ,"sk_dst-29" , +"sk_dst-AF_TIPC" ,"sk_dst-AF_BLUETOOTH" ,"sk_dst-AF_IUCV" , +"sk_dst-AF_RXRPC" ,"sk_dst-AF_MAX" +}; -/* -* sk_callback_lock locking rules are per-address-family, -* so split the lock classes by using a per-AF key: -*/ -static struct lock_class_key af_callback_keys[AF_MAX ]; + +static inline void sock_lock_init (struct sock *sk) +{ +sock_lock_init_class_and_name (sk, + af_family_slock_key_strings[sk->sk_family], + af_family_slock_keys + sk->sk_family, + af_family_key_strings[sk->sk_family], + af_family_keys + sk->sk_family); +} + +#define lockdep_set_sk_callback_lock (lock, family) \ + lockdep_set_class_and_name(lock, \ + af_family_callback_keys + (family), \ + af_family_clock_key_strings[(family)]) + +#define lockdep_set_sk_dst_lock (lock, family) \ + lockdep_set_class_and_name(lock, \ + af_family_dst_keys + (family), \ + af_family_dst_key_strings[(family)]) + +#else + +static inline void sock_lock_init (struct sock *sk) +{ +sock_lock_init_class_and_name (sk,0 ,0 ,0 ,0 ); +} + +#define lockdep_set_sk_callback_lock (lock, family) do {} while (0) +#define lockdep_set_sk_dst_lock (lock, family) do {} while (0) + +#endif /* CONFIG_DEBUG_LOCK_ALLOC */ /* Take into consideration the size of the struct sk_buff overhead in the * determination of these values, since that is non-constant across
@@ -866,14 +908,6 @@ lenout:* * (We also register the sk_lock with the lock validator.) */ -static inline void sock_lock_init (struct sock *sk) -{ -sock_lock_init_class_and_name (sk, - af_family_slock_key_strings[sk->sk_family], - af_family_slock_keys + sk->sk_family, - af_family_key_strings[sk->sk_family], - af_family_keys + sk->sk_family); -}static void sock_copy (struct sock *nsk,const struct sock *osk) {
@@ -1014,10 +1048,10 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority)#endif rwlock_init (&newsk->sk_dst_lock); +lockdep_set_sk_dst_lock (&newsk->sk_dst_lock, newsk->sk_family);rwlock_init (&newsk->sk_callback_lock); -lockdep_set_class_and_name (&newsk->sk_callback_lock, - af_callback_keys + newsk->sk_family, - af_family_clock_key_strings[newsk->sk_family]); +lockdep_set_sk_callback_lock (&newsk->sk_callback_lock, + newsk->sk_family); newsk->sk_dst_cache =NULL ; newsk->sk_wmem_queued =0 ;
@@ -1703,10 +1737,9 @@ void sock_init_data(struct socket *sock, struct sock *sk) sk->sk_sleep =NULL ;rwlock_init (&sk->sk_dst_lock); +lockdep_set_sk_dst_lock (&sk->sk_dst_lock, sk->sk_family);rwlock_init (&sk->sk_callback_lock); -lockdep_set_class_and_name (&sk->sk_callback_lock, - af_callback_keys + sk->sk_family, - af_family_clock_key_strings[sk->sk_family]); +lockdep_set_sk_callback_lock (&sk->sk_callback_lock, sk->sk_family); sk->sk_state_change = sock_def_wakeup; sk->sk_data_ready = sock_def_readable;
Quoted 19 lines · James Chapman
Quoted 18 lines · Jarek Poplawski
On Mon, Feb 11, 2008 at 10:19:35PM +0000, James Chapman wrote:
...Quoted 15 lines
Below is example output from lockdep. The oops is reproducible when creating/deleting lots of sessions while passing data. The lock is being acquired for read and write in softirq contexts.
Is there a better way to fix this?
=================================
[ INFO: inconsistent lock state ] 2.6.24-core2 #1
---------------------------------
inconsistent {in-softirq-R} -> {softirq-on-W} usage. openl2tpd/3215 [HC0[0]:SC0[0]:HE1:SE1] takes:
(&tunnel->hlist_lock){---?}, at: [<f8eea157>]pppol2tp_connect+0x517/0x6d0[pppol2tp] {in-softirq-R} state was registered at:
Quoted 8 lines
Quoted 5 lines
IMHO, according to this, disabling bh should be enough. And if it's like in this report: only
read_lockis taken from softirqs, then this should be necessary to change only allwrite_lockstowrite_lock_bh(of course unless somewhere bhs are disabled already). Unless I miss something?!I thought so too. I tried
_bhlocks first and the problem still occurred. Maybe I'll try it again in case I messed something up.
If there are any new lockdep warnings I'd be interested to have a look. (BTW, with irqs disabling such ISP would probably get considerable drop in performance?)
Regards,
Jarek P.