raw table and NOTRACK support

Hello,

This is a re-diffed patch to have raw table and NOTRACK support in 2.4.x kernel. I've kept the IPv6 part in this patch although it's untested and most probably it's never been used by anyone before.

Caveats: Currently we get an oops on SMP iif:

o NOTRACK rule loaded, active and used (refcnt>0)
o SMP kernel
o connection tracking is enabled
o a normal rule hitting the conntrack table during lookup
o iptables -X; iptables -F, rmmod <all netfilter related modules>

Earlier attempts to address this issue with Pablo Neira have resulted in a misplaced nf_reset(skb) patch, which I have removed again, because it broke masquerading (IIRC). I will enable KDB and report back once I get time some decent stack trace.

On normal reconfiguration all works perfectly, but to be safe we need to flush the conntrack table or already established connection without a filter rule would continue to be passed through the firewall. This will go into production on our packet filters, however with the note to our support team, that reconfiguration including connection tracking table flushing is forbidden under death penalty :).

Cheers,
Roberto Nibali, ratz

-------------------------------------------------------------
addr://Kasinostrasse 30, CH-5001 Aarau tel://++41 62 823 9355
http://www.terreactive.com             fax://++41 62 823 9356
-------------------------------------------------------------
terreActive AG                       Wir sichern Ihren Erfolg
-------------------------------------------------------------

diff -Nur linux-2.4.31-orig/Documentation/Configure.help linux-2.4.31-pab2/Documentation/Configure.help
@@ -3016,6 +3016,34 @@
   If you want to compile it as a module, say M here and read
   <file:Documentation/modules.txt>.  If unsure, say `N'.
 
+raw table support (required for NOTRACK/TRACE)
+CONFIG_IP_NF_RAW
+  This option adds a `raw' table to iptables. This table is the very
+  first in the netfilter framework and hooks in at the PREROUTING
+  and OUTPUT chains.
+
+  If you want to compile it as a module, say M here and read
+  <file:Documentation/modules.txt>.  If unsure, say `N'.
+
+NOTRACK target support
+CONFIG_IP_NF_TARGET_NOTRACK
+  The NOTRACK target allows a select rule to specify
+  which packets *not* to enter the conntrack/NAT
+  subsystem with all the consequences (no ICMP error tracking,
+  no protocol helpers for the selected packets).
+
+  If you want to compile it as a module, say M here and read
+  <file:Documentation/modules.txt>.  If unsure, say `N'.
+
+raw table support (required for TRACE)
+CONFIG_IP6_NF_RAW
+  This option adds a `raw' table to ip6tables. This table is the very
+  first in the netfilter framework and hooks in at the PREROUTING
+  and OUTPUT chains.
+
+  If you want to compile it as a module, say M here and read
+  <file:Documentation/modules.txt>.  If unsure, say `N'.
+
 Packet filtering
 CONFIG_IP_NF_FILTER
   Packet filtering defines a table `filter', which has a series of

diff -Nur linux-2.4.31-orig/include/linux/netfilter_ipv4/ip_conntrack.h linux-2.4.31-pab2/include/linux/netfilter_ipv4/ip_conntrack.h
@@ -254,6 +254,9 @@
 /* Call me when a conntrack is destroyed. */
 extern void (*ip_conntrack_destroyed)(struct ip_conntrack *conntrack);
 
+/* Fake conntrack entry for untracked connections */
+extern struct ip_conntrack ip_conntrack_untracked;
+
 /* Returns new sk_buff, or NULL */
 struct sk_buff *
 ip_ct_gather_frags(struct sk_buff *skb, u_int32_t user);

diff -Nur linux-2.4.31-orig/include/linux/netfilter_ipv4/ipt_conntrack.h linux-2.4.31-pab2/include/linux/netfilter_ipv4/ipt_conntrack.h
@@ -10,6 +10,7 @@
 
 #define IPT_CONNTRACK_STATE_SNAT (1 << (IP_CT_NUMBER + 1))
 #define IPT_CONNTRACK_STATE_DNAT (1 << (IP_CT_NUMBER + 2))
+#define IPT_CONNTRACK_STATE_UNTRACKED (1 << (IP_CT_NUMBER + 3))
 
 /* flags, invflags: */
 #define IPT_CONNTRACK_STATE	0x01

diff -Nur linux-2.4.31-orig/include/linux/netfilter_ipv4/ipt_state.h linux-2.4.31-pab2/include/linux/netfilter_ipv4/ipt_state.h
@@ -3,6 +3,7 @@
 
 #define IPT_STATE_BIT(ctinfo) (1 << ((ctinfo)%IP_CT_IS_REPLY+1))
 #define IPT_STATE_INVALID (1 << 0)
+#define IPT_STATE_UNTRACKED (1 << (IP_CT_NUMBER + 1))
 
 struct ipt_state_info
 {

diff -Nur linux-2.4.31-orig/include/linux/netfilter_ipv4.h linux-2.4.31-pab2/include/linux/netfilter_ipv4.h
@@ -51,6 +51,8 @@
 
 enum nf_ip_hook_priorities {
 	NF_IP_PRI_FIRST = INT_MIN,
+	NF_IP_PRI_CONNTRACK_DEFRAG = -400,
+	NF_IP_PRI_RAW = -300,
 	NF_IP_PRI_CONNTRACK = -200,
 	NF_IP_PRI_MANGLE = -150,
 	NF_IP_PRI_NAT_DST = -100,

diff -Nur linux-2.4.31-orig/net/ipv4/netfilter/Config.in linux-2.4.31-pab2/net/ipv4/netfilter/Config.in
@@ -107,6 +107,15 @@
   dep_tristate '  LOG target support' CONFIG_IP_NF_TARGET_LOG $CONFIG_IP_NF_IPTABLES
   dep_tristate '  ULOG target support' CONFIG_IP_NF_TARGET_ULOG $CONFIG_IP_NF_IPTABLES
   dep_tristate '  TCPMSS target support' CONFIG_IP_NF_TARGET_TCPMSS $CONFIG_IP_NF_IPTABLES
+  if [ "$CONFIG_EXPERIMENTAL" = "y" ]; then
+    tristate '  raw table support (required for NOTRACK/TRACE)' CONFIG_IP_NF_RAW $CONFIG_IP_NF_IPTABLES
+  fi
+  if [ "$CONFIG_IP_NF_RAW" != "n" ]; then
+    if [ "$CONFIG_IP_NF_CONNTRACK" != "n" ]; then
+      dep_tristate '    NOTRACK target support' CONFIG_IP_NF_TARGET_NOTRACK $CONFIG_IP_NF_RAW
+    fi
+  # Marker for TRACE target
+  fi
 fi
 
 tristate 'ARP tables support' CONFIG_IP_NF_ARPTABLES

diff -Nur linux-2.4.31-orig/net/ipv4/netfilter/Makefile linux-2.4.31-pab2/net/ipv4/netfilter/Makefile
@@ -65,6 +65,7 @@
 obj-$(CONFIG_IP_NF_FILTER) += iptable_filter.o
 obj-$(CONFIG_IP_NF_MANGLE) += iptable_mangle.o
 obj-$(CONFIG_IP_NF_NAT) += iptable_nat.o
+obj-$(CONFIG_IP_NF_RAW) += iptable_raw.o
 
 # matches
 obj-$(CONFIG_IP_NF_MATCH_HELPER) += ipt_helper.o

@@ -90,6 +91,7 @@
 obj-$(CONFIG_IP_NF_MATCH_CONNTRACK) += ipt_conntrack.o
 obj-$(CONFIG_IP_NF_MATCH_UNCLEAN) += ipt_unclean.o
 obj-$(CONFIG_IP_NF_MATCH_TCPMSS) += ipt_tcpmss.o
+obj-$(CONFIG_IP_NF_TARGET_NOTRACK) += ipt_NOTRACK.o
 
 # targets
 obj-$(CONFIG_IP_NF_TARGET_REJECT) += ipt_REJECT.o

diff -Nur linux-2.4.31-orig/net/ipv4/netfilter/ip_nat_core.c linux-2.4.31-pab2/net/ipv4/netfilter/ip_nat_core.c
@@ -1024,6 +1024,10 @@
 	IP_NF_ASSERT(ip_conntrack_destroyed == NULL);
 	ip_conntrack_destroyed = &ip_nat_cleanup_conntrack;
 
+	/* Initialize fake conntrack so that NAT will skip it */
+	ip_conntrack_untracked.nat.info.initialized |=
+		(1 << IP_NAT_MANIP_SRC) | (1 << IP_NAT_MANIP_DST);
+
 	return 0;
 }
 

diff -Nur linux-2.4.31-orig/net/ipv4/netfilter/ipt_conntrack.c linux-2.4.31-pab2/net/ipv4/netfilter/ipt_conntrack.c
@@ -27,7 +27,9 @@
 
 #define FWINV(bool,invflg) ((bool) ^ !!(sinfo->invflags & invflg))
 
-	if (ct)
+	if (skb->nfct == &ip_conntrack_untracked.infos[IP_CT_NEW])
+		statebit = IPT_CONNTRACK_STATE_UNTRACKED;
+	else if (ct)
 		statebit = IPT_CONNTRACK_STATE_BIT(ctinfo);
 	else
 		statebit = IPT_CONNTRACK_STATE_INVALID;

diff -Nur linux-2.4.31-orig/net/ipv4/netfilter/ipt_state.c linux-2.4.31-pab2/net/ipv4/netfilter/ipt_state.c
@@ -21,7 +21,9 @@
 	enum ip_conntrack_info ctinfo;
 	unsigned int statebit;
 
-	if (!ip_conntrack_get((struct sk_buff *)skb, &ctinfo))
+	if (skb->nfct == &ip_conntrack_untracked.infos[IP_CT_NEW])
+		statebit = IPT_STATE_UNTRACKED;
+	else if (!ip_conntrack_get((struct sk_buff *)skb, &ctinfo))
 		statebit = IPT_STATE_INVALID;
 	else
 		statebit = IPT_STATE_BIT(ctinfo);

diff -Nur linux-2.4.31-orig/net/ipv6/netfilter/Config.in linux-2.4.31-pab2/net/ipv6/netfilter/Config.in
@@ -75,4 +75,9 @@
   #dep_tristate '  LOG target support' CONFIG_IP6_NF_TARGET_LOG $CONFIG_IP6_NF_IPTABLES
 fi
 
+  if [ "$CONFIG_EXPERIMENTAL" = "y" ]; then
+    tristate '  raw table support (required for TRACE)' CONFIG_IP6_NF_RAW $CONFIG_IP6_NF_IPTABLES
+  fi
+  # Marker for TRACE target
+
 endmenu

diff -Nur linux-2.4.31-orig/net/ipv6/netfilter/Makefile linux-2.4.31-pab2/net/ipv6/netfilter/Makefile
@@ -30,6 +30,7 @@
 obj-$(CONFIG_IP6_NF_TARGET_MARK) += ip6t_MARK.o
 obj-$(CONFIG_IP6_NF_QUEUE) += ip6_queue.o
 obj-$(CONFIG_IP6_NF_TARGET_LOG) += ip6t_LOG.o
+obj-$(CONFIG_IP6_NF_RAW) += ip6table_raw.o
 obj-$(CONFIG_IP6_NF_MATCH_HL) += ip6t_hl.o
 
 include $(TOPDIR)/Rules.make

No oops with kdb, a busyloop instead now.

Hmm, with kdb I get following trace (check the bt at the end):

Entering kdb (current=0xc0494000, pid 0) on processor 0 due to cpu switch [0]kdb> cpu
Currently on cpu 0
Available cpus: 0, 1, 2, 3
[0]kdb> cpu 3

Entering kdb (current=0xf5c72000, pid 7064) on processor 3 due to cpu switch [3]kdb> ssb

0xf89a7232 get_next_corpse+0xb2:    mov    (%esi),%eax
0xf89a7234 get_next_corpse+0xb4:    inc    %eax
0xf89a7235 get_next_corpse+0xb5:    mov    %eax,%ecx
0xf89a7237 get_next_corpse+0xb7:    mov    %eax,(%esi)
0xf89a7239 get_next_corpse+0xb9:    cmp    0xf89aad04,%ecx
0xf89a723f get_next_corpse+0xbf:    jb     0xf89a71e0 get_next_corpse+0x60
[3]kdb>
0xf89a71e0 get_next_corpse+0x60:    movl   $0x0,0xfffffff0(%ebp)
0xf89a71e7 get_next_corpse+0x67:    mov    0xf89aae64,%eax
0xf89a71ec get_next_corpse+0x6c:    mov    (%eax,%ecx,8),%ebx
0xf89a71ef get_next_corpse+0x6f:    mov    (%ebx),%edx
0xf89a71f1 get_next_corpse+0x71:    prefetchnta (%edx)
0xf89a71f4 get_next_corpse+0x74:    lea    (%eax,%ecx,8),%eax
0xf89a71f7 get_next_corpse+0x77:    jmp    0xf89a7227 get_next_corpse+0xa7
[3]kdb>
0xf89a7227 get_next_corpse+0xa7:    cmp    %ebx,%eax
0xf89a7229 get_next_corpse+0xa9:    jne    0xf89a7200 get_next_corpse+0x80
[3]kdb>
0xf89a722b get_next_corpse+0xab:    mov    0xfffffff0(%ebp),%ecx
0xf89a722e get_next_corpse+0xae:    test   %ecx,%ecx
0xf89a7230 get_next_corpse+0xb0:    jne    0xf89a7276 get_next_corpse+0xf6
[3]kdb>
0xf89a7232 get_next_corpse+0xb2:    mov    (%esi),%eax
0xf89a7234 get_next_corpse+0xb4:    inc    %eax
0xf89a7235 get_next_corpse+0xb5:    mov    %eax,%ecx
0xf89a7237 get_next_corpse+0xb7:    mov    %eax,(%esi)
0xf89a7239 get_next_corpse+0xb9:    cmp    0xf89aad04,%ecx
0xf89a723f get_next_corpse+0xbf:    jb     0xf89a71e0 get_next_corpse+0x60

[3]kdb> ss
SS trap at 0xf89a7227 ([ip_conntrack]get_next_corpse+0xa7)
0xf89a7227 get_next_corpse+0xa7: cmp %ebx,%eax
[3]kdb> rd
eax = 0xf89d9f08 ebx = 0xf89d9f08 ecx = 0x00001fe1 edx = 0xf89d9f08 esi = 0xf5c73f20 edi = 0x00000000 esp = 0xf5c73ef4 eip = 0xf89a7227 ebp = 0xf5c73f0c xss = 0xc0350018 xcs = 0x00000010 eflags = 0x00000287 xds = 0x00000018 xes = 0x00000018 origeax = 0xffffffff &regs = 0xf5c73ec0 [3]kdb> ss
SS trap at 0xf89a7229 ([ip_conntrack]get_next_corpse+0xa9)
0xf89a7229 get_next_corpse+0xa9: jne 0xf89a7200 get_next_corpse+0x80
[3]kdb> rd
eax = 0xf89d9f08 ebx = 0xf89d9f08 ecx = 0x00001fe1 edx = 0xf89d9f08 esi = 0xf5c73f20 edi = 0x00000000 esp = 0xf5c73ef4 eip = 0xf89a7229 ebp = 0xf5c73f0c xss = 0xc0350018 xcs = 0x00000010 eflags = 0x00000246 xds = 0x00000018 xes = 0x00000018 origeax = 0xffffffff &regs = 0xf5c73ec0 [3]kdb> ss
SS trap at 0xf89a722b ([ip_conntrack]get_next_corpse+0xab)
0xf89a722b get_next_corpse+0xab: mov 0xfffffff0(%ebp),%ecx
[3]kdb> rd
eax = 0xf89d9f08 ebx = 0xf89d9f08 ecx = 0x00001fe1 edx = 0xf89d9f08 esi = 0xf5c73f20 edi = 0x00000000 esp = 0xf5c73ef4 eip = 0xf89a722b ebp = 0xf5c73f0c xss = 0xc0350018 xcs = 0x00000010 eflags = 0x00000246 xds = 0x00000018 xes = 0x00000018 origeax = 0xffffffff &regs = 0xf5c73ec0 [3]kdb> bt
Stack traceback for pid 7064
0xf5c72000 7064 7011 1 3 R 0xf5c722b0 *rmmod
EBP EIP Function (args)
0xf5c73f0c 0xf89a723f [ip_conntrack]get_next_corpse+0xbf (0xf89a7470, 0x0, 0xf5c73f20, 0x1fe2, 0xf5c72000)

                               ip_conntrack .text 0xf89a4060 0xf89a7180

0xf89a72d0
0xf5c73f30 0xf89a7303 [ip_conntrack]ip_ct_iterate_cleanup_Rsmp_4ff11842+0x33 (0xf89a7470, 0x0, 0x0)

                               ip_conntrack .text 0xf89a4060 0xf89a72d0
0xf89a7370
0xf5c73f44 0xf89a74f7 [ip_conntrack]ip_conntrack_cleanup+0x77
(0xf89a990f, 0xc2a7bd20, 0xc0471e20, 0xf89a4000)
                               ip_conntrack .text 0xf89a4060 0xf89a7480
0xf89a7550
0xf5c73f5c 0xf89a479f [ip_conntrack]init_or_cleanup+0x17f (0x0)
                               ip_conntrack .text 0xf89a4060 0xf89a4620
0xf89a4810
0xf5c73f68 0xf89a4a22 [ip_conntrack]fini+0x12 (0xf89a4000, 0xfffffff0,
0xf5d8b000, 0xf5c73f84, 0xf89a4000)
                               ip_conntrack .text 0xf89a4060 0xf89a4a10
0xf89a4a24
0xf5c73f8c 0xc0120641 free_module+0x111 (0xf89a4000, 0x0, 0x1000,
0xbfffde18, 0xf5c72000)
                               kernel .text 0xc0100000 0xc0120530 0xc0120660
0xf5c73fbc 0xc011f639 sys_delete_module+0x129 (0xbffffcd9, 0xbfffefd4,
0xbfffdf2c, 0x1, 0xbfffdf2c)
                               kernel .text 0xc0100000 0xc011f510 0xc011f940
           0xc010774f system_call+0x33
                               kernel .text 0xc0100000 0xc010771c 0xc0107754

[3]kdb>

Tell me if you need more info.

Cheers,
Roberto Nibali, ratz

-------------------------------------------------------------
addr://Kasinostrasse 30, CH-5001 Aarau tel://++41 62 823 9355
http://www.terreactive.com             fax://++41 62 823 9356
-------------------------------------------------------------
terreActive AG                       Wir sichern Ihren Erfolg
-------------------------------------------------------------

I don't get it. It's looping in:

void
ip_ct_iterate_cleanup(int (*iter)(struct ip_conntrack *i, void *), void *data)

{
        struct ip_conntrack_tuple_hash *h;
        unsigned int bucket = 0;

        while ((h = get_next_corpse(iter, data, &bucket)) != NULL) {
                /* Time to push up daises... */
                if (del_timer(&h->ctrack->timeout))
                        death_by_timeout((unsigned long)h->ctrack);
                /* ... else the timer will get him soon. */

                ip_conntrack_put(h->ctrack);
        }
}

which is called from:

void ip_conntrack_cleanup(void)

{
        ip_ct_attach = NULL;
        /* This makes sure all current packets have passed through
           netfilter framework.  Roll on, two-stage module
           delete... */
        br_write_lock_bh(BR_NETPROTO_LOCK);
        br_write_unlock_bh(BR_NETPROTO_LOCK);

i_see_dead_people:

        ip_ct_iterate_cleanup(kill_all, NULL);
        if (atomic_read(&ip_conntrack_count) != 0) {
                schedule();
                goto i_see_dead_people;
        }
        while (atomic_read(&ip_conntrack_untracked.ct_general.use) > 1)
                schedule();

        kmem_cache_destroy(ip_conntrack_cachep);
        vfree(ip_conntrack_hash);
        nf_unregister_sockopt(&so_getorigdst);
}

I don't see where ip_conntrack_untracked.ct_general.use is > 1, ever ...

I'm completely puzzled,
Roberto Nibali, ratz

-------------------------------------------------------------
addr://Kasinostrasse 30, CH-5001 Aarau tel://++41 62 823 9355
http://www.terreactive.com             fax://++41 62 823 9356
-------------------------------------------------------------
terreActive AG                       Wir sichern Ihren Erfolg
-------------------------------------------------------------

SS trap at 0xf89a7227 ([ip_conntrack]get_next_corpse+0xa7)
0xf89a7227 get_next_corpse+0xa7: cmp %ebx,%eax
[0]kdb> mm4 ip_conntrack_count 0
0xf89aae68 = 0x0
[0]kdb> go
lb-lb0-phys:~#

So forcing ip_conntrack_count to be 0 of course breaks the endless schedule(). And naturally after a fw reconfiguration we oops:

kernel BUG at slab.c:815!
invalid operand: 0000
ip_conntrack ipt_limit ip_vs_wlc ip_vs ipt_LOG iptable_raw iptable_mangle iptable_filter ip_tables

CPU:    0
EIP:    0010:[<c013bb32>]    Not tainted
EFLAGS: 00010246

EIP is at kmem_cache_create+0x262/0x3d0 [kernel]
eax: 00000000 ebx: f7ae6a98 ecx: f7ae6ba0 edx: f7295fc8
esi: f7ae6b99 edi: f89a9945 ebp: f5b1deac esp: f5b1de84
ds: 0018 es: 0018 ss: 0018
Process modprobe (pid: 7457, stackpage=f5b1d000)

Stack: f7ae6a98 00000160 00002000 f5b1de9c f7ae6ab8 ffffffe0 00000080
00000000
       00000000 00000060 f5b1ded0 f89a7660 f89a9938 00000160 00000020
00022000
       00000000 00000000 00000000 f5b1dee8 f89a4639 ffffffea 00000000
00000060
Call Trace:
 [<f89a7660>] ip_conntrack_init+0x110/0x298 [ip_conntrack]
 [<f89a9938>] .rodata.str1.1+0x198/0x2e0 [ip_conntrack]
 [<f89a4639>] init_or_cleanup+0x19/0x1f0 [ip_conntrack]
 [<f89a4a02>] init_module+0x12/0x20 [ip_conntrack]
 [<c011f40e>] sys_init_module+0x85e/0x8c0 [kernel]
 [<f89a4060>] kill_proto+0x0/0x20 [ip_conntrack]
 [<f89ad1cc>] E ip_conntrack_hash_Rsmp_386855a5+0x2368/0xfffffebc

[ip_conntrack]
[<f89aa168>]
__ksymtab_ip_conntrack_protocol_register_Rsmp_6e500e17+0x0/0x8 [ip_conntrack]

 [<f89a4060>] kill_proto+0x0/0x20 [ip_conntrack]
 [<c010774f>] system_call+0x33/0x38 [kernel]

Code: 0f 0b 2f 03 57 c8 37 c0 89 d0 8b 12 0f 18 02 3d 90 1b 47 c0

Entering kdb (current=0xf5b1c000, pid 7457) on processor 0 Oops: invalid operand
due to oops @ 0xc013bb32
eax = 0x00000000 ebx = 0xf7ae6a98 ecx = 0xf7ae6ba0 edx = 0xf7295fc8 esi = 0xf7ae6b99 edi = 0xf89a9945 esp = 0xf5b1de84 eip = 0xc013bb32 ebp = 0xf5b1deac xss = 0xc0350018 xcs = 0x00000010 eflags = 0x00010246 xds = 0xf7ae0018 xes = 0x00000018 origeax = 0xffffffff &regs = 0xf5b1de50 [0]kdb> bt
Stack traceback for pid 7457
0xf5b1c000 7457 7455 1 0 R 0xf5b1c2b0 *modprobe
EBP EIP Function (args)
0xf5b1deac 0xc013bb32 kmem_cache_create+0x262 (0xf89a9938, 0x160, 0x20, 0x22000, 0x0)

                               kernel .text 0xc0100000 0xc013b8d0 0xc013bca0
0xf5b1ded0 0xf89a7660 [ip_conntrack]ip_conntrack_init+0x110 (0xffffffea,
0x0, 0x60, 0xffffffea)
                               ip_conntrack .text 0xf89a4060 0xf89a7550
0xf89a77e8
0xf5b1dee8 0xf89a4639 [ip_conntrack]init_or_cleanup+0x19 (0x1)
                               ip_conntrack .text 0xf89a4060 0xf89a4620
0xf89a4810
0xf5b1def4 0xf89a4a02 [ip_conntrack]init_module+0x12 (0xf89a4060,
0x8096a20, 0x916c, 0xf89ad1cc, 0xf89aa168)
                               ip_conntrack .text 0xf89a4060 0xf89a49f0
0xf89a4a10
0xf5b1dfbc 0xc011f40e sys_init_module+0x85e (0x806ab70, 0x80969c0,
0x80969c0, 0x400191d8, 0xbfffb0fc)
                               kernel .text 0xc0100000 0xc011ebb0 0xc011f470
           0xc010774f system_call+0x33
                               kernel .text 0xc0100000 0xc010771c 0xc0107754

[0]kdb> go
Catastrophic error detected
kdb_continue_catastrophic=0, type go a second time if you really want to continue
[0]kdb> mm4 sysrq_enabled 1
0xc047bf20 = 0x1
[0]kdb> sr 7
<6>SysRq : Changing Loglevel
Loglevel set to 7
[0]kdb> sr s
SysRq : Emergency Sync
[0]kdb> sr u
SysRq : Emergency Remount R/O
[0]kdb> sr s
SysRq : Emergency Sync
[0]kdb> sr b
SysRq : Resetting

Damn! I wish I understood that conntrack stuff better ...

Cheers,
Roberto Nibali, ratz

-------------------------------------------------------------
addr://Kasinostrasse 30, CH-5001 Aarau tel://++41 62 823 9355
http://www.terreactive.com             fax://++41 62 823 9356
-------------------------------------------------------------
terreActive AG                       Wir sichern Ihren Erfolg
-------------------------------------------------------------

Ok, so NOTRACK registers itself into the conntrack table upon target entry using nf_conntrack_get((*pskb)->nfct). And each skb updates the nfct counter, but when deregistering the conntrack we still have references of the fake connection tracking entry of the NOTRACK hook. This was discussed already and a Patrick submitted a patchset:

http://www.kernel.org/git/?p=linux/kernel/git/torvalds/linux-2.6.git;a=commit;h=b31e5b1bb53b99dfd5e890aa07e943aff114ae1c

Patrick, in the thread leading to this patch we concluded that you would forward the nf_reset patch to Marcelo for 2.4.x inclusion. I only realised now that this did not happen and thus the following patch is needed for 2.4.x to have rmmod ip_conntrack working correctly when having either bridging or NOTRACK (both not in vanilla) loaded and used in the kernel:

@@ -167,6 +167,9 @@
        nf_debug_ip_finish_output2(skb);
 #endif /*CONFIG_NETFILTER_DEBUG*/
 
+       /* Drop conntrack reference when packet leaves IP */
+       nf_reset(skb);
+
        if (hh) {
                int hh_alen;
 

Is there a reason not to include this patch in 2.4.x?

Thanks and regards,
Roberto Nibali, ratz

-------------------------------------------------------------
addr://Kasinostrasse 30, CH-5001 Aarau tel://++41 62 823 9355
http://www.terreactive.com             fax://++41 62 823 9356
-------------------------------------------------------------
terreActive AG                       Wir sichern Ihren Erfolg
-------------------------------------------------------------

Yes, it turned out to break a lots of things on loopback. We put a different patch in 2.6, which dropped the reference at known points where the packet would be queued, except for the qdiscs. We can put the same patch in 2.4.

Hello Patrick,

Thanks for replying.

Although I don't see how in 2.4.x, I now vaguely remember the bug report.

That would be perfect, could you point me to the git reference to this patch, please?

Cheers,
Roberto Nibali, ratz

echo
'[q]sa[ln0=aln256%Pln256/snlbx]sb3135071790101768542287578439snlbxq' | dc

One of the things it broke was SO_ORIGINAL_DST support for transparent proxying, which also affects 2.4.

It was commit 84531c24f27b02daa8e54e2bb6dc74a730fdf0a5, titled "[NETFILTER]: Revert nf_reset change".

Ok.

Hmmm, so how about the following approach?

@@ -105,6 +105,7 @@
 void dst_release(struct dst_entry * dst)
 {
        if (dst) {
+               WARN_ON(atomic_read(&dst->__refcnt) < 1);
                smp_mb__before_atomic_dec();
                atomic_dec(&dst->__refcnt);
        }

diff -X dontdiff -Nur linux-2.4.32-orig/net/packet/af_packet.c linux-2.4.32-pab2/net/packet/af_packet.c
@@ -272,6 +272,11 @@
        if ((skb = skb_share_check(skb, GFP_ATOMIC)) == NULL)
                goto oom;
 
+       /* drop any routing info and conntrack reference */
+       dst_release(skb->dst);
+       skb->dst = NULL;
+       nf_reset(skb);
+
        spkt = (struct sockaddr_pkt*)skb->cb;
 
        skb_push(skb, skb->data-skb->mac.raw);

@@ -507,6 +512,12 @@
 
        skb_set_owner_r(skb, sk);
        skb->dev = NULL;
+
+       /* drop any routing info and conntrack reference */
+       dst_release(skb->dst);
+       skb->dst = NULL;
+       nf_reset(skb);
+
        spin_lock(&sk->receive_queue.lock);
        po->stats.tp_packets++;
        __skb_queue_tail(&sk->receive_queue, skb);

I'm compiling it now and will be running test, so long as the thing even boots ;).

I think the WARN_ON could be submitted to 2.4.x anyway since it helps finding other occurances of wrong refcnt decreasing. Why is the routing entry dropped in 2.6.x and not in 2.4.x? Maybe I should also cc netdev as well.

Thanks and best regards,
Roberto Nibali, ratz

-------------------------------------------------------------
addr://Kasinostrasse 30, CH-5001 Aarau tel://++41 62 823 9355
http://www.terreactive.com             fax://++41 62 823 9356
-------------------------------------------------------------
terreActive AG                       Wir sichern Ihren Erfolg
-------------------------------------------------------------

Hmm, somehow I haven't caught all possible skb releases, a conntrack flush takes 8m32s. :). I'll add another nf_reset in ipv4/ip_input.c for ip_call_ra_chain() ... this is getty really fishy.

-------------------------------------------------------------
addr://Kasinostrasse 30, CH-5001 Aarau tel://++41 62 823 9355
http://www.terreactive.com             fax://++41 62 823 9356
-------------------------------------------------------------
terreActive AG                       Wir sichern Ihren Erfolg
-------------------------------------------------------------