New: IPVS sync_backup oops
Subsystems (4)
A message that is not in the archive (sent off-list or not yet received): [email protected]/
Quoted 17 lines · On Fri, 25 Apr 2008 21:27:18 -0700 (PDT) [email protected] wrote:
http://bugzilla.kernel.org/show_bug.cgi?id=10556
Summary: IPVS sync_backup oops Product: Networking Version: 2.5 KernelVersion: 2.6.24.x Platform: All OS/Version: Linux Tree: Mainline Status: NEW Severity: high Priority: P1 Component: IPV4 AssignedTo: [email protected] ReportedBy: [email protected]
Quoted 2 lines
Latest working kernel version: 2.6.23.17
Earliest failing kernel version: 2.6.24
It's a regression.
Quoted 39 lines
Distribution: kernel.org
Hardware Environment: i386,P4
Software Environment: ipvsadm-1.24
Problem Description:
MATSER ipvs box will push connection table to BACKUP one using multicast, while the BACKUP ipvs box received the multicast message,
the BACKUP ipvs box will reportBUG()and crashSteps to reproduce:
+-------------+ |192.168.1.123| +------+------+ | --------------+-------+------------+-------- | VIP:192.168.1.219 | MASTER +-----+-----+ +-----+-----+ BACKUP eth1 |192.168.1.1| eth1 |192.168.1.2| eth0 |192.168.0.1| eth0 |192.168.0.2| +-----+-----+ +-----+-----+ |-> multicast sync ->| --------------+-------+------------+-------- | real-server +------------+ www service open |192.168.0.80| gateway 192.168.0.1 +------+-----+1)
MASTER ipvs box:
ipvsadm --start-daemon master --mcast-interface eth0 --syncid 80BACKUP ipvs box:
ipvsadm --start-daemon backup --mcast-interface eth0 --syncid 802)
Both MASTER and BACKUP box:
2.1) define service
ipvsadm -A -t 192.168.1.219:80 -s rr
2.2) add real-server
ipvsadm -a -t 192.168.1.219:80 -r 192.168.0.80:80 -m
Quoted 8 lines
# set real-server gateway = 192.168.0.13)
client(192.168.1.123) access http://192.168.1.219,
MATSER ipvs box will push connection table to BACKUP one using multicast, while the BACKUP ipvs box received the multicast message,
the BACKUP ipvs box reportBUG()and crash.
Also,
BUG: unable to handle kernel NULL pointer dereference at virtual address 00000014 printing eip: c030659e *pde = 00000000 Oops: 0000 [#1] SMP Modules linked in: xt_tcpudp iptable_mangle xt_MARK xt_multiport ip_tables
x_tables ip_vs_wrr ip_vs_wlc ip_vs_sh ip_vs_sed ip_vs_rr ip_vs_nq ip_vs_lc ip_vs_lblcr ip_vs_lblc ip_vs_ftp ip_vs_dh pcnet32 crc32 e1000 e100 mii
Pid: 3960, comm: ipvs_syncbackup Not tainted (2.6.24.4 #3)
EIP: 0060:[<c030659e>] EFLAGS: 00010246 CPU: 0
EIP is at sync_thread+0x919/0xa3c
EAX: 00000000 EBX: f6fe6800 ECX: e3ba2c00 EDX: 00000000
ESI: 00000000 EDI: 00000000 EBP: f7310034 ESP: f699bf54
DS: 007b ES: 007b FS: 00d8 GS: 0000 SS: 0068
Process ipvs_syncbackup (pid: 3960, ti=f699a000 task=f6995030 task.ti=f699a000)
Stack: 01000000 00000000 23035e0a 00000000 00001183 00000000 f699bfc0 f6d04860
00000002 00000034 f7310000 f78f1800 00000001 f731001c c0000000 00000000
00000010 f699bfc0 00000001 00000000 00000000 00000000 00000000 f6995030
Call Trace:
[<c011c53a>] default_wake_function+0x0/0x8
[<c0305c85>] sync_thread+0x0/0xa3c
[<c0104aa7>] kernel_thread_helper+0x7/0x10
=======================
Code: a1 20 64 3e c0 89 43 54 8b 54 24 34 0f b7 42 16 66 c1 c0 08 0f b7 c0 66
89 43 42 0f b6 42 01 e8 90 06 00 00 0f b7 53 42 0f b7 d2 <8b> 40 14 8b 04 90 89
43 38 89 d8 e8 8a 8e ff ff 8b 44 24 28 03
EIP: [<c030659e>] sync_thread+0x919/0xa3c SS:ESP 0068:f699bf54
Kernel panic - not syncing: Fatal exception in interruptWhich will be a bit hard to track down because it seems that pretty much the whole world got inlined into sync_thread().
Hi.
Quoted 9 lines · Andrew Morton
[<c011c53a>] default_wake_function+0x0/0x8 [<c0305c85>] sync_thread+0x0/0xa3c [<c0104aa7>] kernel_thread_helper+0x7/0x10 ======================= Code: a1 20 64 3e c0 89 43 54 8b 54 24 34 0f b7 42 16 66 c1 c0 08 0f b7 c0 66 89 43 42 0f b6 42 01 e8 90 06 00 00 0f b7 53 42 0f b7 d2 <8b> 40 14 8b 04 90 89 43 38 89 d8 e8 8a 8e ff ff 8b 44 24 28 03 EIP: [<c030659e>] sync_thread+0x919/0xa3c SS:ESP 0068:f699bf54 Kernel panic - not syncing: Fatal exception in interrupt
Any chance to run
gdb ./vmlinux
$ l *(sync_thread+0x919)
for that kernel, and if it was not compiled with debug info try to do that and get bug with it?
Quoted 2 lines
Which will be a bit hard to track down because it seems that pretty much the whole world got inlined into
sync_thread().
I'm not ipvs guru, but that will give us at least some hint...
Signature · 1 line
Evgeniy Polyakov
Julian Anastasov
maintainerHello,
Quoted 41 lines · Andrew Morton
Quoted 6 lines · On Fri, 25 Apr 2008 21:27:18 -0700 (PDT) [email protected] wrote:
http://bugzilla.kernel.org/show_bug.cgi?id=10556
ReportedBy: [email protected]
Latest working kernel version: 2.6.23.17
Earliest failing kernel version: 2.6.24Also,
BUG: unable to handle kernel NULL pointer dereference at virtual address 00000014 printing eip: c030659e *pde = 00000000 Oops: 0000 [#1] SMP Modules linked in: xt_tcpudp iptable_mangle xt_MARK xt_multiport ip_tables
x_tablesip_vs_wrrip_vs_wlcip_vs_ship_vs_sedip_vs_rrip_vs_nqip_vs_lcip_vs_lblcrip_vs_lblcip_vs_ftpip_vs_dhpcnet32 crc32 e1000 e100 miiPid: 3960, comm:
ipvs_syncbackupNot tainted (2.6.24.4 #3)
EIP: 0060:[<c030659e>] EFLAGS: 00010246 CPU: 0
EIP is atsync_thread+0x919/0xa3c
EAX: 00000000 EBX:f6fe6800ECX:e3ba2c00EDX: 00000000
ESI: 00000000 EDI: 00000000 EBP:f7310034ESP:f699bf54
DS: 007b ES: 007b FS: 00d8 GS: 0000 SS: 0068
Processipvs_syncbackup(pid: 3960, ti=f699a000 task=f6995030 task.ti=f699a000)Stack: 01000000 00000000 23035e0a 00000000 00001183 00000000 f699bfc0 f6d04860 00000002 00000034 f7310000 f78f1800 00000001 f731001c c0000000 00000000 00000010 f699bfc0 00000001 00000000 00000000 00000000 00000000 f6995030 Call Trace: [<c011c53a>] default_wake_function+0x0/0x8 [<c0305c85>] sync_thread+0x0/0xa3c [<c0104aa7>] kernel_thread_helper+0x7/0x10 ======================= Code: a1 20 64 3e c0 89 43 54 8b 54 24 34 0f b7 42 16 66 c1 c0 08 0f b7 c0 66 89 43 42 0f b6 42 01 e8 90 06 00 00 0f b7 53 42 0f b7 d2 <8b> 40 14 8b 04 90 89 43 38 89 d8 e8 8a 8e ff ff 8b 44 24 28 03 EIP: [<c030659e>] sync_thread+0x919/0xa3c SS:ESP 0068:f699bf54 Kernel panic - not syncing: Fatal exception in interrupt
Quoted 2 lines
Which will be a bit hard to track down because it seems that pretty much the whole world got inlined into
sync_thread().
I can not fully understand the above oops but hope following
fix can help (not tested). It is for 2.6.25. I can provide patch for 2.6.24 if needed (there are rejects):
Result from ip_vs_proto_get() should be checked because
protocol value can be invalid or unsupported in backup. Also, add checks to validate message limits and connection state.
---
include/net/ip_vs.h renamed from linux/include/net/ip_vs.h +2 -1
@@ -405,7 +405,8 @@ struct sk_buff;struct ip_vs_protocol {struct ip_vs_protocol *next;char *name; -__u16 protocol; +u16 protocol; +u16 num_states;int dont_defrag;atomic_t appcnt;/* counter of proto app incs */ int *timeout_table;/* protocol timeout table */
net/ipv4/ipvs/ip_vs_proto_ah.c renamed from linux/net/ipv4/ipvs/ip_vs_proto_ah.c +1 -0
@@ -160,6 +160,7 @@ static void ah_exit(struct ip_vs_protocostruct ip_vs_protocol ip_vs_protocol_ah = { .name ="AH" , .protocol =IPPROTO_AH , + .num_states =1 , .dont_defrag =1 , .init = ah_init, .exit = ah_exit,
net/ipv4/ipvs/ip_vs_proto_esp.c renamed from linux/net/ipv4/ipvs/ip_vs_proto_esp.c +1 -0
@@ -159,6 +159,7 @@ static void esp_exit(struct ip_vs_protocstruct ip_vs_protocol ip_vs_protocol_esp = { .name ="ESP" , .protocol =IPPROTO_ESP , + .num_states =1 , .dont_defrag =1 , .init = esp_init, .exit = esp_exit,
net/ipv4/ipvs/ip_vs_proto_tcp.c renamed from linux/net/ipv4/ipvs/ip_vs_proto_tcp.c +1 -0
@@ -594,6 +594,7 @@ static void ip_vs_tcp_exit(struct ip_vs_struct ip_vs_protocol ip_vs_protocol_tcp = { .name ="TCP" , .protocol =IPPROTO_TCP , + .num_states =IP_VS_TCP_S_LAST , .dont_defrag =0 , .appcnt =ATOMIC_INIT (0 ), .init = ip_vs_tcp_init,
net/ipv4/ipvs/ip_vs_proto_udp.c renamed from linux/net/ipv4/ipvs/ip_vs_proto_udp.c +1 -0
@@ -409,6 +409,7 @@ static void udp_exit(struct ip_vs_protocstruct ip_vs_protocol ip_vs_protocol_udp = { .name ="UDP" , .protocol =IPPROTO_UDP , + .num_states =IP_VS_UDP_S_LAST , .dont_defrag =0 , .init = udp_init, .exit = udp_exit,
net/ipv4/ipvs/ip_vs_sync.c renamed from linux/net/ipv4/ipvs/ip_vs_sync.c +38 -16
88 lines not on this page · raw message
Julian Anastasov
maintainerHello,
Quoted 9 lines
Quoted 5 lines · On Fri, 25 Apr 2008 21:27:18 -0700 (PDT) [email protected] wrote:
Quoted 4 lines
http://bugzilla.kernel.org/show_bug.cgi?id=10556
ReportedBy: [email protected]
I can not fully understand the above oops but hope following
fix can help (not tested). It is for 2.6.25. I can provide patch for 2.6.24 if needed (there are rejects):
As I said, there should be rejects if you apply patch against
2.6.24.4. I recommend to use patch against 2.6.25 because 2.6.25 has some fixes in ip_vs_sync. I suspect some hunks were not applied correctly, can you clarify what is the kernel version used now?
In case you are still using 2.6.24.4 I'm appending new
patch, this time against 2.6.24. Let me know if you see some rejects when applying patch (Hunk FAILED). Also, if TCP conns in backup are again NONE and conns are not removed I don't have any explanation, may be you can enable IPVS debugging? As for timeout, I have never used IPVS sync, it does not send timeout, I suspect it would be compatibility issue if timeout and nfmark are added to sync structure. 715827:52 should be jiffies 0xFFFFFFFF (time to expire), may be you can see them with cat /proc/net/ip_vs_conn
Result from ip_vs_proto_get() should be checked because
protocol value can be invalid or unsupported in backup. But for valid message we should not fail for templates which use IPPROTO_IP. Also, add checks to validate message limits and connection state. Show NONE for templates.
---
include/net/ip_vs.h renamed from linux/include/net/ip_vs.h +2 -1
@@ -403,7 +403,8 @@ struct sk_buff;struct ip_vs_protocol {struct ip_vs_protocol *next;char *name; -__u16 protocol; +u16 protocol; +u16 num_states;int dont_defrag;atomic_t appcnt;/* counter of proto app incs */ int *timeout_table;/* protocol timeout table */
net/ipv4/ipvs/ip_vs_proto.c renamed from linux/net/ipv4/ipvs/ip_vs_proto.c +1 -1
@@ -148,7 +148,7 @@ const char * ip_vs_state_name(__u16 protstruct ip_vs_protocol *pp =ip_vs_proto_get (proto);if (pp ==NULL || pp->state_name ==NULL ) -return "ERR!" ; +return (IPPROTO_IP == proto) ?"NONE" :"ERR!" ;return pp->state_name (state); }
net/ipv4/ipvs/ip_vs_proto_ah.c renamed from linux/net/ipv4/ipvs/ip_vs_proto_ah.c +1 -0
@@ -160,6 +160,7 @@ static void ah_exit(struct ip_vs_protocostruct ip_vs_protocol ip_vs_protocol_ah = { .name ="AH" , .protocol =IPPROTO_AH , + .num_states =1 , .dont_defrag =1 , .init = ah_init, .exit = ah_exit,
net/ipv4/ipvs/ip_vs_proto_esp.c renamed from linux/net/ipv4/ipvs/ip_vs_proto_esp.c +1 -0
@@ -159,6 +159,7 @@ static void esp_exit(struct ip_vs_protocstruct ip_vs_protocol ip_vs_protocol_esp = { .name ="ESP" , .protocol =IPPROTO_ESP , + .num_states =1 , .dont_defrag =1 , .init = esp_init, .exit = esp_exit,
net/ipv4/ipvs/ip_vs_proto_tcp.c renamed from linux/net/ipv4/ipvs/ip_vs_proto_tcp.c +1 -0
@@ -594,6 +594,7 @@ static void ip_vs_tcp_exit(struct ip_vs_struct ip_vs_protocol ip_vs_protocol_tcp = { .name ="TCP" , .protocol =IPPROTO_TCP , + .num_states =IP_VS_TCP_S_LAST , .dont_defrag =0 , .appcnt =ATOMIC_INIT (0 ), .init = ip_vs_tcp_init,
net/ipv4/ipvs/ip_vs_proto_udp.c renamed from linux/net/ipv4/ipvs/ip_vs_proto_udp.c +1 -0
@@ -409,6 +409,7 @@ static void udp_exit(struct ip_vs_protocstruct ip_vs_protocol ip_vs_protocol_udp = { .name ="UDP" , .protocol =IPPROTO_UDP , + .num_states =IP_VS_UDP_S_LAST , .dont_defrag =0 , .init = udp_init, .exit = udp_exit,
net/ipv4/ipvs/ip_vs_sync.c renamed from linux/net/ipv4/ipvs/ip_vs_sync.c +62 -21
118 lines not on this page · raw message
David Miller
maintainerFrom: Julian Anastasov <[email protected]>
Date: Sat, 26 Apr 2008 20:48:46 +0300 (EEST)
Quoted 5 lines
Result from
ip_vs_proto_get()should be checked because
protocol value can be invalid or unsupported in backup. Also, add checks to validate message limits and connection state.Signed-off-by Julian Anastasov <[email protected]>
This patch looks great, and fixes real bugs regardless of the effect upon the bug report in question.
I would really like to see this fix tested, because if it fixes the specific bug, all the better, and we should indicate this in the changelog message.
Quoted 7 lines · On Sun, 27 Apr 2008 16:40:16 -0700 (PDT) David Miller <[email protected]>…
From: Julian Anastasov <[email protected]>
Date: Sat, 26 Apr 2008 20:48:46 +0300 (EEST)Quoted 4 lines
Result from
ip_vs_proto_get()should be checked because
protocol value can be invalid or unsupported in backup. Also, add checks to validate message limits and connection state.
Quoted 8 lines
Quoted 1 line
Signed-off-by Julian Anastasov <[email protected]>This patch looks great, and fixes real bugs regardless of the effect upon the bug report in question.
I would really like to see this fix tested, because if it fixes the specific bug, all the better, and we should indicate this in the changelog message.
The reporter (L0op8ack <[email protected]>) has used the bugzilla web interface to confirm that this patch fixes 2.6.24.4.
David Miller
maintainerFrom: Andrew Morton <[email protected]>
Date: Tue, 6 May 2008 13:10:54 -0700
Quoted 15 lines
Quoted 12 lines · On Sun, 27 Apr 2008 16:40:16 -0700 (PDT) David Miller <[email protected]>…
From: Julian Anastasov <[email protected]>
Date: Sat, 26 Apr 2008 20:48:46 +0300 (EEST)Quoted 5 lines
Result from
ip_vs_proto_get()should be checked because
protocol value can be invalid or unsupported in backup. Also, add checks to validate message limits and connection state.Signed-off-by Julian Anastasov <[email protected]>This patch looks great, and fixes real bugs regardless of the effect upon the bug report in question.
Quoted 6 lines
Quoted 3 lines
I would really like to see this fix tested, because if it fixes the specific bug, all the better, and we should indicate this in the changelog message.
The reporter (L0op8ack <[email protected]>) has used the bugzilla web interface to confirm that this patch fixes 2.6.24.4.
Great, thanks for the update.
Julian Anastasov
maintainerHello,
Quoted 18 lines · Andrew Morton
Quoted 1 line
ReportedBy: [email protected]
Also,
BUG: unable to handle kernel NULL pointer dereference at virtual address 00000014 printing eip: c030659e *pde = 00000000 Oops: 0000 [#1] SMP Modules linked in: xt_tcpudp iptable_mangle xt_MARK xt_multiport ip_tables
x_tablesip_vs_wrrip_vs_wlcip_vs_ship_vs_sedip_vs_rrip_vs_nqip_vs_lcip_vs_lblcrip_vs_lblcip_vs_ftpip_vs_dhpcnet32 crc32 e1000 e100 miiPid: 3960, comm:
ipvs_syncbackupNot tainted (2.6.24.4 #3)
EIP: 0060:[<c030659e>] EFLAGS: 00010246 CPU: 0
EIP is atsync_thread+0x919/0xa3c
EAX: 00000000 EBX:f6fe6800ECX:e3ba2c00EDX: 00000000
ESI: 00000000 EDI: 00000000 EBP:f7310034ESP:f699bf54
DS: 007b ES: 007b FS: 00d8 GS: 0000 SS: 0068
Processipvs_syncbackup(pid: 3960, ti=f699a000 task=f6995030 task.ti=f699a000)
Quoted 13 lines
Stack: 01000000 00000000 23035e0a 00000000 00001183 00000000 f699bfc0 f6d04860
00000002 00000034 f7310000 f78f1800 00000001 f731001c c0000000 00000000
00000010 f699bfc0 00000001 00000000 00000000 00000000 00000000 f6995030
Call Trace:
[<c011c53a>] default_wake_function+0x0/0x8
[<c0305c85>] sync_thread+0x0/0xa3c
[<c0104aa7>] kernel_thread_helper+0x7/0x10
=======================
Code: a1 20 64 3e c0 89 43 54 8b 54 24 34 0f b7 42 16 66 c1 c0 08 0f b7 c0 66
89 43 42 0f b6 42 01 e8 90 06 00 00 0f b7 53 42 0f b7 d2 <8b> 40 14 8b 04 90 89
43 38 89 d8 e8 8a 8e ff ff 8b 44 24 28 03
EIP: [<c030659e>] sync_thread+0x919/0xa3c SS:ESP 0068:f699bf54
Kernel panic - not syncing: Fatal exception in interrupt Below is 2nd version that handles templates properly, which
should be the actual problem. Large timeouts in backup look like a problem in incorrect ipvsadm binary, timeouts are not converted from jiffies. Patch follows:
Result from ip_vs_proto_get() should be checked because
protocol value can be invalid or unsupported in backup. But for valid message we should not fail for templates which use IPPROTO_IP. Also, add checks to validate message limits and connection state. Show NONE for templates.
---
include/net/ip_vs.h renamed from linux/include/net/ip_vs.h +2 -1
@@ -405,7 +405,8 @@ struct sk_buff;struct ip_vs_protocol {struct ip_vs_protocol *next;char *name; -__u16 protocol; +u16 protocol; +u16 num_states;int dont_defrag;atomic_t appcnt;/* counter of proto app incs */ int *timeout_table;/* protocol timeout table */
net/ipv4/ipvs/ip_vs_proto.c renamed from linux/net/ipv4/ipvs/ip_vs_proto.c +1 -1
@@ -148,7 +148,7 @@ const char * ip_vs_state_name(__u16 protstruct ip_vs_protocol *pp =ip_vs_proto_get (proto);if (pp ==NULL || pp->state_name ==NULL ) -return "ERR!" ; +return (IPPROTO_IP == proto) ?"NONE" :"ERR!" ;return pp->state_name (state); }
net/ipv4/ipvs/ip_vs_proto_ah.c renamed from linux/net/ipv4/ipvs/ip_vs_proto_ah.c +1 -0
@@ -160,6 +160,7 @@ static void ah_exit(struct ip_vs_protocostruct ip_vs_protocol ip_vs_protocol_ah = { .name ="AH" , .protocol =IPPROTO_AH , + .num_states =1 , .dont_defrag =1 , .init = ah_init, .exit = ah_exit,
net/ipv4/ipvs/ip_vs_proto_esp.c renamed from linux/net/ipv4/ipvs/ip_vs_proto_esp.c +1 -0
@@ -159,6 +159,7 @@ static void esp_exit(struct ip_vs_protocstruct ip_vs_protocol ip_vs_protocol_esp = { .name ="ESP" , .protocol =IPPROTO_ESP , + .num_states =1 , .dont_defrag =1 , .init = esp_init, .exit = esp_exit,
net/ipv4/ipvs/ip_vs_proto_tcp.c renamed from linux/net/ipv4/ipvs/ip_vs_proto_tcp.c +1 -0
@@ -594,6 +594,7 @@ static void ip_vs_tcp_exit(struct ip_vs_struct ip_vs_protocol ip_vs_protocol_tcp = { .name ="TCP" , .protocol =IPPROTO_TCP , + .num_states =IP_VS_TCP_S_LAST , .dont_defrag =0 , .appcnt =ATOMIC_INIT (0 ), .init = ip_vs_tcp_init,
net/ipv4/ipvs/ip_vs_proto_udp.c renamed from linux/net/ipv4/ipvs/ip_vs_proto_udp.c +1 -0
@@ -409,6 +409,7 @@ static void udp_exit(struct ip_vs_protocstruct ip_vs_protocol ip_vs_protocol_udp = { .name ="UDP" , .protocol =IPPROTO_UDP , + .num_states =IP_VS_UDP_S_LAST , .dont_defrag =0 , .init = udp_init, .exit = udp_exit,
net/ipv4/ipvs/ip_vs_sync.c renamed from linux/net/ipv4/ipvs/ip_vs_sync.c +56 -16
106 lines not on this page · raw message