kernel BUG at net/core/skbuff.c:100

Hi all,

I hope, You can help me debug this issue. I’m running batman-adv 2014.3.0 included in 3.16.7 linux. Something provokes kernel panic on the VM. It might be a modified or regular packet. The crash occurs at “prime time” but the machine is not under heavy load.

I hope you can localize the error. I provide you with some crash dumps from a vmcore dump. Please tell me if you need more or there is something I should do to provide you with more information about this issue.

Thank you all for B.A.T.M.A.N.-adv!

Best regards

Philipp

crash 7.0.7

      KERNEL: /usr/src/linux-3.16.7-gentoo/vmlinux
    DUMPFILE: vmcore_20141118174414
        CPUS: 1
        DATE: Tue Nov 18 16:17:01 2014
      UPTIME: 16:30:32
LOAD AVERAGE: 0.19, 0.22, 0.25
       TASKS: 124
    NODENAME: wolke
     RELEASE: 3.16.7-gentoo
     VERSION: #1 SMP Mon Nov 17 03:44:22 CET 2014
     MACHINE: x86_64  (2593 Mhz)
      MEMORY: 511.6 MB
       PANIC: "kernel BUG at net/core/skbuff.c:100!"
         PID: 2012
     COMMAND: "fastd"
        TASK: ffff880019c411a0  [THREAD_INFO: ffff880019200000]
         CPU: 0
       STATE: TASK_RUNNING (PANIC)

crash> bt
PID: 2012 TASK: ffff880019c411a0 CPU: 0 COMMAND: "fastd"
#0 [ffff88001fc03980] machine_kexec at ffffffff8103a34e #1 [ffff88001fc039e0] crash_kexec at ffffffff810be503 #2 [ffff88001fc03ab0] oops_end at ffffffff81005fc8 #3 [ffff88001fc03ae0] die at ffffffff81006463 #4 [ffff88001fc03b10] do_trap at ffffffff81002e12 #5 [ffff88001fc03b70] do_error_trap at ffffffff8100316d #6 [ffff88001fc03c30] do_invalid_op at ffffffff8100394b #7 [ffff88001fc03c40] invalid_op at ffffffff817f385e

    [exception RIP: skb_panic+94]
    RIP: ffffffff817eb99d  RSP: ffff88001fc03cf8  RFLAGS: 00010296
    RAX: 000000000000008b  RBX: ffff8800100ccee0  RCX: 0000000000000092
    RDX: 0000000000000062  RSI: 0000000000000046  RDI: 0000000000000246
    RBP: ffff88001fc03d18   R8: 0000000000000000   R9: 0000000000000000
    R10: 00000000000001ed  R11: 0000000000000006  R12: 0000000000000564
    R13: ffff88001fc03da0  R14: ffff880013fd9100  R15: ffff880005948062
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018

#8 [ffff88001fc03d20] skb_put at ffffffff81611bb1
#9 [ffff88001fc03d30] batadv_frag_skb_buffer at ffffffffa002fdfa [batman_adv]
#10 [ffff88001fc03d90] batadv_recv_frag_packet at ffffffffa003a213 [batman_adv] #11 [ffff88001fc03dd0] batadv_batman_skb_recv at ffffffffa0033e95 [batman_adv] #12 [ffff88001fc03e10] __netif_receive_skb_core at ffffffff81621962 #13 [ffff88001fc03e80] __netif_receive_skb at ffffffff81621e91
#14 [ffff88001fc03ea0] process_backlog at ffffffff81621f7e
#15 [ffff88001fc03ef0] net_rx_action at ffffffff81622731
#16 [ffff88001fc03f50] __do_softirq at ffffffff81053ef8
#17 [ffff88001fc03fb0] do_softirq_own_stack at ffffffff817f3a5c --- <IRQ stack> ---
#18 [ffff880019203d10] do_softirq_own_stack at ffffffff817f3a5c

    [exception RIP: tun_get_user+1056]
    RIP: ffffffffa001d8f0  RSP: 0000000000000001  RFLAGS: 7fff00000586
    RAX: ffffffff816210b4  RBX: ffff880019203d58  RCX: ffff8800193dc780
    RDX: 0000000000000000  RSI: ffff8800193dc780  RDI: 0000000000000586
    RBP: ffffffff81620de4   R8: ffff880019203d88   R9: ffff8800193dc780
    R10: ffff8800193dc780  R11: ffffffff81054135  R12: ffff880019203d58
    R13: 0000000000000586  R14: ffff88001932b900  R15: 0000000000000000
    ORIG_RAX: ffff880019203e38  CS: 7fff018d3a90  SS: 0000

bt: WARNING: possibly bogus exception frame
#19 [ffff880019203e40] tun_chr_aio_write at ffffffffa001de0b [tun] #20 [ffff880019203e70] do_sync_write at ffffffff8115c665 #21 [ffff880019203f00] vfs_write at ffffffff8115d38a #22 [ffff880019203f40] sys_write at ffffffff8115d89a #23 [ffff880019203f80] system_call_fastpath at ffffffff817f1f29

    RIP: 00007fef80cbe37d  RSP: 00007fff018d3bd0  RFLAGS: 00000206
    RAX: 0000000000000001  RBX: ffffffff817f1f29  RCX: 00000000000000b4
    RDX: 0000000000000586  RSI: 00000000016e9ba0  RDI: 0000000000000009
    RBP: 0000000000000586   R8: 00007fef80ca7400   R9: 00007fff018d31d8
    R10: 00007fff018d391f  R11: 0000000000000293  R12: 00000000016e9358
    R13: 0000000000000001  R14: 00000000016e9b90  R15: 00000000016e13a0
    ORIG_RAX: 0000000000000001  CS: 0033  SS: 002b

crash> log
[…]

[59432.101578] skbuff: skb_over_panic: text:ffffffffa002fdfa len:1464 put:1380 head:ffff88000596f800 data:ffff88000596f862 tail:0x61a end:0x2c0 dev:fastd0
[59432.101901] ------------[ cut here ]------------
[59432.102014] kernel BUG at net/core/skbuff.c:100!
[59432.102125] invalid opcode: 0000 [#1] SMP 
[59432.102225] Modules linked in: xt_nat batman_adv libcrc32c tun iptable_nat nf_nat_ipv4 nf_nat ipip crc32c_intel
[59432.102490] CPU: 0 PID: 2012 Comm: fastd Not tainted 3.16.7-gentoo #1
[59432.102519] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2007
[59432.102519] task: ffff880019c411a0 ti: ffff880019200000 task.ti: ffff880019200000
[59432.102519] RIP: 0010:[<ffffffff817eb99d>]  [<ffffffff817eb99d>] skb_panic+0x5e/0x60
[59432.102519] RSP: 0018:ffff88001fc03cf8  EFLAGS: 00010296
[59432.102519] RAX: 000000000000008b RBX: ffff8800100ccee0 RCX: 0000000000000092
[59432.102519] RDX: 0000000000000062 RSI: 0000000000000046 RDI: 0000000000000246
[59432.102519] RBP: ffff88001fc03d18 R08: 0000000000000000 R09: 0000000000000000
[59432.102519] R10: 00000000000001ed R11: 0000000000000006 R12: 0000000000000564
[59432.102519] R13: ffff88001fc03da0 R14: ffff880013fd9100 R15: ffff880005948062
[59432.102519] FS:  00007fef8196a700(0000) GS:ffff88001fc00000(0000) knlGS:0000000000000000
[59432.102519] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[59432.102519] CR2: 00007f5c610ac000 CR3: 0000000019352000 CR4: 00000000000006f0
[59432.102519] Stack:
[59432.102519]  ffff88000596f862 000000000000061a 00000000000002c0 ffff8800193dc000
[59432.102519]  ffff88001fc03d28 ffffffff81611bb1 ffff88001fc03d88 ffffffffa002fdfa
[59432.102519]  ffff8800193aa168 ffff88000594804e ffff88001fc03d78 ffff8800100ccee0
[59432.102519] Call Trace:
[59432.102519]  <IRQ> 
[59432.102519] 
[59432.102519]  [<ffffffff81611bb1>] skb_put+0x41/0x50
[59432.102519]  [<ffffffffa002fdfa>] batadv_frag_skb_buffer+0x25a/0x410 [batman_adv]
[59432.102519]  [<ffffffffa003a213>] batadv_recv_frag_packet+0x183/0x200 [batman_adv]
[59432.102519]  [<ffffffffa0033e95>] batadv_batman_skb_recv+0xd5/0x110 [batman_adv]
[59432.102519]  [<ffffffff81621962>] __netif_receive_skb_core+0x222/0x730
[59432.102519]  [<ffffffff81621e91>] __netif_receive_skb+0x21/0x70
[59432.102519]  [<ffffffff81621f7e>] process_backlog+0x9e/0x170
[59432.102519]  [<ffffffff81622731>] net_rx_action+0x141/0x240
[59432.102519]  [<ffffffff81053ef8>] __do_softirq+0xe8/0x280
[59432.102519]  [<ffffffff817f3a5c>] do_softirq_own_stack+0x1c/0x30
[59432.102519]  <EOI> 
[59432.102519] 
[59432.102519]  [<ffffffff81054135>] do_softirq+0x55/0x60
[59432.102519]  [<ffffffff816210b4>] netif_rx_ni+0x34/0x70
[59432.102519]  [<ffffffffa001d8f0>] tun_get_user+0x420/0x840 [tun]
[59432.102519]  [<ffffffffa001de0b>] tun_chr_aio_write+0x7b/0xa0 [tun]
[59432.102519]  [<ffffffff8115c665>] do_sync_write+0x55/0x90
[59432.102519]  [<ffffffff8115d38a>] vfs_write+0xba/0x1f0
[59432.102519]  [<ffffffff8115d89a>] SyS_write+0x4a/0xa0
[59432.102519]  [<ffffffff817f1f29>] system_call_fastpath+0x16/0x1b
[59432.102519] Code: 00 00 48 89 44 24 10 8b 87 c0 00 00 00 48 89 44 24 08 48 8b 87 d0 00 00 00 48 c7 c7 30 67 a3 81 48 89 04 24 31 c0 e8 0d 8b ff ff <0f> 0b 55 48 89 f8 48 8b 57 30 48 89 e5 48 8b 0f 5d 80 e5 80 48 
[59432.102519] RIP  [<ffffffff817eb99d>] skb_panic+0x5e/0x60
[59432.102519]  RSP <ffff88001fc03cf8>

crash> ps

   PID    PPID  CPU       TASK        ST  %MEM     VSZ    RSS  COMM
      0      0   0  ffffffff81c16480  RU   0.0       0      0  [swapper/0]
      1      0   0  ffff88001f140000  IN   0.3    4232   1508  init
      2      0   0  ffff88001f1408d0  IN   0.0       0      0  [kthreadd]
      3      2   0  ffff88001f1411a0  IN   0.0       0      0  [ksoftirqd/0]
      5      2   0  ffff88001f142340  IN   0.0       0      0  [kworker/0:0H]
      7      2   0  ffff88001f1434e0  IN   0.0       0      0  [rcu_sched]
      8      2   0  ffff88001f143db0  IN   0.0       0      0  [rcu_bh]
      9      2   0  ffff88001f144680  IN   0.0       0      0  [migration/0]
     10      2   0  ffff88001f144f50  IN   0.0       0      0  [khelper]
     11      2   0  ffff88001f145820  IN   0.0       0      0  [kdevtmpfs]
     12      2   0  ffff88001f1460f0  IN   0.0       0      0  [netns]
     16      2   0  ffff88001f3408d0  IN   0.0       0      0  [writeback]
     18      2   0  ffff88001f341a70  IN   0.0       0      0  [crypto]
     20      2   0  ffff88001f342c10  IN   0.0       0      0  [bioset]
     22      2   0  ffff88001f343db0  IN   0.0       0      0  [kblockd]
     48      2   0  ffff88001f3469c0  IN   0.0       0      0  [ata_sff]
     51      2   0  ffff88001f345820  IN   0.0       0      0  [khubd]
     53      2   0  ffff88001f347290  IN   0.0       0      0  [md]
     56      2   0  ffff88001f340000  IN   0.0       0      0  [cfg80211]
     69      2   0  ffff88001f344680  IN   0.0       0      0  [rpciod]
     74      2   0  ffff88001f342340  IN   0.0       0      0  [kswapd0]
     78      2   0  ffff88001f147290  IN   0.0       0      0  [fsnotify_mark]
     80      2   0  ffff8800000808d0  IN   0.0       0      0  [nfsiod]
    117      2   0  ffff88001f3434e0  IN   0.0       0      0  [acpi_thermal_pm]
    734      2   0  ffff88001f2260f0  IN   0.0       0      0  [scsi_eh_0]
    735      2   0  ffff88001f2269c0  IN   0.0       0      0  [scsi_tmf_0]
    737      2   0  ffff88001f224f50  IN   0.0       0      0  [scsi_eh_1]
    739      2   0  ffff88001f223db0  IN   0.0       0      0  [scsi_tmf_1]
    774      2   0  ffff88001a3d7290  IN   0.0       0      0  [kpsmoused]
    781      2   0  ffff88001a3d4f50  IN   0.0       0      0  [ipv6_addrconf]
    785      2   0  ffff88001a3d60f0  IN   0.0       0      0  [deferwq]
    869      2   0  ffff88001f32cf50  IN   0.0       0      0  [kjournald]
   1053      1   0  ffff88001f329a70  IN   0.6   36576   2916  systemd-udevd
   1110      2   0  ffff88001f32e0f0  IN   0.0       0      0  [kworker/0:1H]
   1493      1   0  ffff880019c47290  IN   0.3    4240   1544  acpid
   1514      1   0  ffff880019c460f0  IN   0.4   17752   2316  crond
   1860      1   0  ffff88001f32ac10  IN   7.9  749452  41644  ntop
   1861      1   0  ffff88001f32d820  IN   7.9  749452  41644  ntop
   1862      1   0  ffff88001f3291a0  IN   7.9  749452  41644  ntop
   1863      1   0  ffff880019c43db0  IN   7.9  749452  41644  ntop
   1864      1   0  ffff880019c45820  IN   7.9  749452  41644  ntop
   1865      1   0  ffff880019c41a70  IN   7.9  749452  41644  ntop
   1866      1   0  ffff880019c408d0  IN   7.9  749452  41644  ntop
   1885      1   0  ffff880019c44f50  IN   7.9  749452  41644  ntop
   1886      1   0  ffff880019c434e0  IN   7.9  749452  41644  ntop
   1890      1   0  ffff880019c40000  IN   0.6   41536   3200  sshd
   1935      1   0  ffff88001a39d820  IN   0.4   14232   2072  agetty
   1939      1   0  ffff88001a398000  IN   7.9  749452  41644  ntop
   1963      2   0  ffff88001037c680  IN   0.0       0      0  [kworker/u2:1]
>  2012      1   0  ffff880019c411a0  RU   0.5  179496   2836  fastd
   2028      2   0  ffff88001f225820  IN   0.0       0      0  [bat_events]
   2043      1   0  ffff88001f224680  IN   0.5   11904   2664  alfred
   2044      1   0  ffff88001f227290  IN   0.8   51388   3992  sudo
   2045   2044   0  ffff88001f2211a0  IN   0.4    8684   1860  batadv-vis
   2046   2043   0  ffff88001f221a70  IN   0.8   51388   3980  sudo
   2047   2046   0  ffff88001f2208d0  IN   0.3    8484   1592  alfred
   2093      1   0  ffff88001a399a70  IN   2.4   43536  12344  dhcpd
   2133      1   0  ffff88001f222340  IN   0.3    7336   1692  vnstatd
   2177      1   0  ffff88001f32b4e0  IN   7.5  177112  39340  named
   2178      1   0  ffff88001f32e9c0  IN   7.5  177112  39340  named
   2179      1   0  ffff88001f32bdb0  IN   7.5  177112  39340  named
   2180      1   0  ffff88001f3288d0  IN   7.5  177112  39340  named
   2230      1   0  ffff88001f220000  IN   4.5  300988  23680  apache2
   2232   2230   0  ffff880019c42c10  IN   1.3  227520   6716  apache2
   2247   2230   0  ffff880019c44680  IN  11.9 1066408  62252  apache2
   2248   2230   0  ffff880019c469c0  IN  11.9 1066408  62524  apache2
   2251   2230   0  ffff88001f32a340  IN  11.9 1066408  62524  apache2
   2252   2230   0  ffff88001f344f50  IN  11.9 1066408  62524  apache2
   2253   2230   0  ffff88001f3411a0  IN  11.9 1066408  62524  apache2
   2254   2230   0  ffff88001a3991a0  IN  11.9 1066408  62524  apache2
   2255   2230   0  ffff88001a39b4e0  IN  11.9 1066408  62524  apache2
   2256   2230   0  ffff88001a2ed820  IN  11.9 1066408  62524  apache2
   2257   2230   0  ffff88001a2ef290  IN  11.9 1066408  62524  apache2
   2258   2230   0  ffff88001a2ecf50  IN  11.9 1066408  62524  apache2
   2259   2230   0  ffff88001a2e9a70  IN  11.9 1066408  62524  apache2
   2260   2230   0  ffff88001a2ec680  IN  11.9 1066408  62524  apache2
   2261   2230   0  ffff88001a2e91a0  IN  11.9 1066408  62524  apache2
   2262   2230   0  ffff88001a2ee0f0  IN  11.9 1066408  62524  apache2
   2263   2230   0  ffff88001a2e88d0  IN  11.9 1066408  62524  apache2
   2264   2230   0  ffff88001a2e8000  IN  11.9 1066408  62524  apache2
   2265   2230   0  ffff88001a2ea340  IN  11.9 1066408  62524  apache2
   2266   2230   0  ffff88001a2eac10  IN  11.9 1066408  62524  apache2
   2267   2230   0  ffff88001a2eb4e0  IN  11.9 1066408  62524  apache2
   2268   2230   0  ffff88001a2ebdb0  IN  11.9 1066408  62524  apache2
   2269   2230   0  ffff88001a3d2c10  IN  11.9 1066408  62524  apache2
   2270   2230   0  ffff88001a3d08d0  IN  11.9 1066408  62524  apache2
   2271   2230   0  ffff88001a3d2340  IN  11.9 1066408  62524  apache2
   2272   2230   0  ffff88001a3d1a70  IN  11.9 1066408  62524  apache2
   2273   2230   0  ffff88001a3d11a0  IN  11.9 1066408  62252  apache2
   2274   2230   0  ffff88001a3d0000  IN  11.9 1066408  62252  apache2
   2275   2230   0  ffff88001a3d3db0  IN  11.9 1066408  62252  apache2
   2276   2230   0  ffff88001a3d5820  IN  11.9 1066408  62252  apache2
   2277   2230   0  ffff88001a3d69c0  IN  11.9 1066408  62252  apache2
   2278   2230   0  ffff880000081a70  IN  11.9 1066408  62252  apache2
   2279   2230   0  ffff880000082340  IN  11.9 1066408  62252  apache2
   2280   2230   0  ffff880000082c10  IN  11.9 1066408  62252  apache2
   2281   2230   0  ffff8800000834e0  IN  11.9 1066408  62252  apache2
   2282   2230   0  ffff880000083db0  IN  11.9 1066408  62252  apache2
   2283   2230   0  ffff880000084680  IN  11.9 1066408  62252  apache2
   2284   2230   0  ffff880000084f50  IN  11.9 1066408  62252  apache2
   2285   2230   0  ffff880000085820  IN  11.9 1066408  62252  apache2
   2286   2230   0  ffff8800000860f0  IN  11.9 1066408  62252  apache2
   2287   2230   0  ffff880000087290  IN  11.9 1066408  62524  apache2
   2288   2230   0  ffff8800000811a0  IN  11.9 1066408  62524  apache2
   2289   2230   0  ffff880000080000  IN  11.9 1066408  62524  apache2
   2290   2230   0  ffff880013f18000  IN  11.9 1066408  62524  apache2
   2291   2230   0  ffff8800000869c0  IN  11.9 1066408  62252  apache2
   2292   2230   0  ffff880013f188d0  IN  11.9 1066408  62252  apache2
   2293   2230   0  ffff880013f191a0  IN  11.9 1066408  62252  apache2
   2294   2230   0  ffff880013f19a70  IN  11.9 1066408  62252  apache2
   2295   2230   0  ffff880013f1a340  IN  11.9 1066408  62252  apache2
   2296   2230   0  ffff880013f1ac10  IN  11.9 1066408  62252  apache2
   2297   2230   0  ffff880013f1b4e0  IN  11.9 1066408  62252  apache2
   2298   2230   0  ffff880013f1bdb0  IN  11.9 1066408  62252  apache2
   2299   2230   0  ffff880013f1c680  IN  11.9 1066408  62252  apache2
   2300   2230   0  ffff880013f1cf50  IN  11.9 1066408  62252  apache2
   2301   2230   0  ffff880013f1d820  IN  11.9 1066408  62252  apache2
   2302   2230   0  ffff880013f1e0f0  IN  11.9 1066408  62252  apache2
   2318      1   0  ffff880013ff88d0  IN   0.5   11904   2680  runmap
   4410      2   0  ffff88001a39f290  IN   0.0       0      0  [kworker/0:2]
   5335      2   0  ffff88001037f290  IN   0.0       0      0  [kworker/u2:0]
   6096      2   0  ffff88001037e0f0  IN   0.0       0      0  [kworker/0:1]
   7792      2   0  ffff88001a39bdb0  IN   0.0       0      0  [kworker/0:0]
   8412   2318   0  ffff88001037ac10  IN   0.2    4220   1308  sleep

crash> files
PID: 2012 TASK: ffff880019c411a0 CPU: 0 COMMAND: "fastd"
ROOT: / CWD: /
FD FILE DENTRY INODE TYPE PATH

  0 ffff88001f35f100 ffff88001a776c00 ffff880019a53440 CHR  /dev/pts/0
  1 ffff88001f35f100 ffff88001a776c00 ffff880019a53440 CHR  /dev/pts/0
  2 ffff88001f35f100 ffff88001a776c00 ffff880019a53440 CHR  /dev/pts/0
  3 ffff88001935f300 ffff88001a7076c0 ffff88001f14fce0 CHR  /dev/urandom

5 ffff88001935f700 ffff880019a96600 ffff88001a77dbb0 SOCK UNIX 6 ffff88001935f900 ffff880019a96900 ffff88001a77d930 SOCK UNIX 7 ffff88001935f400 ffff880019a94f00 ffff88001a74fd40 UNKN [eventpoll] 8 ffff88001935f100 ffff880019a94e40 ffff88001a77d6b0 SOCK UDP 9 ffff880019376200 ffff88001a7ed0c0 ffff88001a3fba48 CHR /dev/net/tun

# batctl -v
batctl gentoo-2014.3.0 [batman-adv: 2014.3.0]

# fastd -v
fastd v16

# batctl if
fastd0: active

Hi Philipp,

Thanks for you report. The bug is probably triggered by some bogus data in an incoming packet. I have created a small debug patch that will detect if this is the case, and print some debug info if so.

Is it possible for you to checkout the source, add the patch, and compile the module?

Thanks,
Martin

Kind Regards,
Martin Hundebøll
Frederiks Allé 99A, 1.th
8000 Aarhus C

+45 61 65 54 61
[email protected]

index 362e91a..f733b99 100644
@@ -249,6 +249,24 @@ batadv_frag_merge_packets(struct hlist_head *chain, struct sk_buff *skb)
 	skb_out = entry->skb;
 	kfree(entry);
 
+	if (size < skb->len) {
+		printk(KERN_DEBUG "batadv_frag_merge_packets: skb total size too small: size=%i, skb->len=%u\n",
+		       size, skb->len);
+		print_hex_dump_bytes("", DUMP_PREFIX_NONE, skb->data - ETH_HLEN,
+				     ETH_HLEN + hdr_size + 16);
+
+		goto free;
+	}
+
+	if (size < skb_out->len) {
+		printk(KERN_DEBUG "batadv_frag_merge_packets: skb_out total size too small: size=%i, skb_out->len=%u\n",
+		       size, skb_out->len);
+		print_hex_dump_bytes("", DUMP_PREFIX_NONE, skb_out->data - ETH_HLEN,
+				     ETH_HLEN + hdr_size + 16);
+
+		goto free;
+	}
+
 	/* Make room for the rest of the fragments. */
 	if (pskb_expand_head(skb_out, 0, size - skb->len, GFP_ATOMIC) < 0) {
 		kfree_skb(skb_out);

Hi Martin,

Thank you for your response. I'm glad to help making Batman-adv better.

Batman-adv ran with network_coding enabled while kernel panic. This was a misconfiguration because our nodes doesn't have nc compiled in their Batman kernel module. The VM is in production. I deactivated nc after someone told me I do not need nc. But I think my community forgive me another gateway failure for research sake.

Am Donnerstag, den 20.11.2014, 09:32 +0100 schrieb Martin Hundebøll:

Thank you for your work. I didn't find your Patch on
http://git.open-mesh.org/batman-adv.git

I can not analyse the packages because the gateway is part of an ISP infrastructure and there is data privacy. But if you're capable to fish only the bogus data package during kernel panic with your patch there shouldn't be any problems, I think.

Yes, I can checkout, patch and compile. The kernel is compiled with CONFIG_DEBUG_INFO=y
# CONFIG_DEBUG_INFO_REDUCED is not set
# CONFIG_ENABLE_WARN_DEPRECATED is not set

Batman-adv is compiled as module. Is there a reboot of the VM needed if I patch the source, compile, replace, depmod and reload the Batman module?

Please send me the patch and tell me the additional make parameters to compile the module with debug symbols. Is it something like make \
CONFIG_BATMAN_ADV_DEBUG=y \
CONFIG_BATMAN_ADV_BLA=y \
CONFIG_BATMAN_ADV_DAT=y \
CONFIG_BATMAN_ADV_NC=y
?
If I patch the (batman) kernel sources directly then a simply make in kernel directory should be enough, I presume. I also presume vmimage will be updated. Or should I rebuild the kernel from scratch?

I hope, this bug doesn't occur through the gentoo patches. But some similar freezes happened on Arch Linux with 3.14.23_ARCH and 3.17.1-ARCH with nc enabled. Unfortunately I can not analyse this bug on the Arch VMs because I'm not in total control of their VM terminal.

Best regards

Philipp

________________________
Freifunk Rheinland e. V.
– Funkzelle Wuppertal –

Hi Philipp,

Yeah, most people compile out network coding. Has the bug disappeared after disabling NC ?

It was attached to my previous mail :)

My debug patch should only print the header of the packet causing the panic, so no problems with privacy here. (But you should probably check the output before mailing it to a public list...)

A simple rmmod/insmod should be enough. (Including the following configuration, which is reset with rmmod.)

Running make (as you write it above) in the module directory should do the trick. (Given you have the needed kernel header files installed) E.g. something like this:

git clone --branch v2014.3.0 git://git.open-mesh.org/batman-adv.git
cd batman-adv
git apply frag_debug_size.patch
make \
CONFIG_BATMAN_ADV_DEBUG=y \
CONFIG_BATMAN_ADV_BLA=y \
CONFIG_BATMAN_ADV_DAT=y \
CONFIG_BATMAN_ADV_NC=y

sudo rmmod batman_adv
sudo insmod batman-adv.ko
sudo batctl if add fastd0

And then your usual IP configuration on bat0 etc.

I am running with NC on my machines in the lab and haven't seen this frag-issue before. I have seen a similar issue (wrong size value in the header) in another context though, but this wasn't due to either network coding or fragmentation.

Would you mind sending me your fastd config (without the key), so that I can try to reproduce this in my VMs?

Thanks,
Martin

Hi Martin

/usr/src/linux/net/batman-adv/fragmentation.c is patched. I'm sorry I oversaw your attachment. the new module is running, the size differs

# lsmod
[ … ]
batman_adv            147774  0 # old
batman_adv            148030  0 # new
[ … ]

Batman-adv runs with

# batctl if
fastd0: active

# batctl it
5000

# batctl ap
disabled

# batctl bl
enabled
# batctl dat
enabled

# batctl ag
enabled

# batctl b
disabled

# batctl f
enabled

# batctl nc
enabled

# batctl mark
0x00000000/0x00000000

# batctl mm
enabled

batctl ll
Error - can't open file '/sys/class/net/bat0/mesh/log_level': No such file or directory [ … ]

batctl gw
server (announced bw: 100.0/100.0 MBit)

this are also the options while kernel panic.

Am Donnerstag, den 20.11.2014, 11:27 +0100 schrieb Martin Hundebøll:

[ … ]

I can't tell for sure. nc is disabled for 20 hours. The Bug appeared from 1 minute to 72 hours. It depends on our users. To reproduce the bug nc is enabled again.

I'm so sorry ;-) my fault

OK, thanks for that

[ … ]

Well, the lab is peaceful but in the free wild there are evil data packages.

Not at all. Here is the censored /etc/fastd/fastd.conf

#---8<---8<---8<---8<---8<---8<----
bind <my_publicIP>:<my_fastdPORT>;
include "secret.conf";
include peers from "peers/wupper";
include peers from "testpeers/wupper";
include peers from "servers/wupper";
interface "fastd0";
log level warn;
method "salsa2012+gmac";
#### doesn't have anything to do with the bug, also seen with fastd v14 #### not used yet but with the new firmware:
method "salsa2012+umac";
mtu 1426;

on up "
ip link set address <MAC_ADDRESS> dev $INTERFACE
ip link set up dev $INTERFACE
modprobe batman-adv
batctl if add fastd0
batctl it 5000
batctl bl enable
batctl gw client
### gw will be changed later to server 100000/100000 ip link set up dev bat0
ip addr add 10.3.<IP>/16 broadcast 10.3.255.255 dev bat0 ip addr add 10.3.<anotherIP>/16 broadcast 10.3.255.255 dev bat0 ip addr add fda0:747e:ab29:e1ba:<IPv6_IP>/64 dev bat0 ip route add 10.3.0.0/16 dev bat0 proto kernel scope link src
10.3.<wrong_IP*)>
alfred -i bat0 -m > /dev/null 2>&1 &
batadv-vis -i bat0 -s > /dev/null 2>&1 &
";
#---8<---8<---8<---8<---8<---8<----EOF

*) now I see there is a different IP. This IP does not belong to this machine, and during kernel panic and now to no machine in the Batman cloud.

wolke linux # /etc/init.d/fastd start fastd ... RTNETLINK answers: Invalid argument
#### now I know why ;-) but to reproduce the bug I don't change it

then this commands are executed:
#---8<---8<---8<---8<---8<---8<----
ip tunnel add tun-ffw-w07 mode ipip remote <remoteIP> local <myIP> ip addr add <some_ISP_IP>/31 dev tun-ffw-w07
ip tunnel change tun-ffw-w07 ttl 64
ip link set mtu 1400 dev tun-ffw-w07
ip link set dev tun-ffw-w07 up

ip rule add from <some_ISP_IP>/31 table 16
ip rule add iif bat0 table 16
ip rule add from all to <some_ISP_IP_for_this_machine> lookup 16

ip route add default via <some_ISP_IP_on_the_other_side> \
dev tun-ffw-w07 table 16
ip route add <some_ISP_IP>/31 dev tun-ffw-w07 table 16

# bat doesn't need any address, but the error occurs also with scope
# link
ip addr flush dev fastd0

iptables -t nat \
	-A POSTROUTING \
	-o tun-ffw-w07 ! -s <some_ISP_IP>/31 \
	-j SNAT --to <some_ISP_IP_for_this_machine>
iptables -A FORWARD -p tcp \
	--tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu

# yes, I know … but some services in the net do not like IMCP # http://lartc.org/howto/lartc.cookbook.mtu-mss.html

sysctl -w net.ipv4.ip_forward=1
sysctl -w net.ipv4.conf.default.rp_filter=0
sysctl -w net.ipv4.conf.all.rp_filter=0

/etc/local.d/kdump.start
/etc/init.d/dhcpd restart
/etc/init.d/vnstatd restart
/etc/init.d/named restart
/etc/init.d/apache2 restart
batctl gw server 100000/100000
#---8<---8<---8<---8<---8<---8<----EOF

Now we have to wait till “prime time” or weekend. I always hoped:
“please don't crush” but now it's different ;-) I hope after that you can reproduce the bug and fix it.

Best regards

Philipp
________________________
Freifunk Rheinland e. V.
– Funkzelle Wuppertal –

Hi Phi

Cool!

[...]

[...]

Thanks for the config. A little more commands than I expected, but I will take a look within the following days.

Please note that the server shouldn't crash with the patch applied, but simply print some info to dmesg. So please please check dmesg every now and then. You can use grep:

dmesg | grep batadv_frag_merge_packets

Thanks,
Martin

Hi Martin

I've bad news. I ran the VM with crashkernel=32M this wasn't enough. I couldn't save the vmcore. Now I'll use crashkernel=64M.

I don't think the attached crash report has something to do with our bug. This is the only information I could retrieve.

I'll report again when dmsg tells something; or with a crash dump.

Best regards

Philipp
________________________
Freifunk Rheinland e. V.
– Funkzelle Wuppertal –

Hi Martin, hi all!

This bug has not been recorded with your patch. There are no info in kernel ring buffer about it. I'd like to run the VM with nc disabled for a week and see if the bug happens again. I'm open for further patching and testing resolving this bug and glad to help. I can not give you the vmcore dump but you can tell me some commands for crash or we can meet in the IRC next week if you like to crawl live inside.

Best regards

Philipp

________________________
Freifunk Rheinland e. V.
– Funkzelle Wuppertal –

SYSTEM MAP: /boot/System.map
DEBUG KERNEL: /usr/src/linux-3.16.7-gentoo/vmlinux (3.16.7-gentoo)

    DUMPFILE: vmcore_20141122201714
        CPUS: 1
        DATE: Sat Nov 22 17:52:11 2014
      UPTIME: 1 days, 08:38:59
LOAD AVERAGE: 0.23, 0.18, 0.15
       TASKS: 125
    NODENAME: wolke
     RELEASE: 3.16.7-gentoo
     VERSION: #1 SMP Mon Nov 17 03:44:22 CET 2014
     MACHINE: x86_64  (2593 Mhz)
      MEMORY: 511.6 MB
       PANIC: "kernel BUG at net/core/skbuff.c:100!"
         PID: 2041
     COMMAND: "fastd"
        TASK: ffff88001a3a7290  [THREAD_INFO: ffff8800192b0000]
         CPU: 0
       STATE: TASK_RUNNING (PANIC)

crash> bt
PID: 2041 TASK: ffff88001a3a7290 CPU: 0 COMMAND: "fastd"
#0 [ffff88001fc03980] machine_kexec at ffffffff8103a34e #1 [ffff88001fc039e0] crash_kexec at ffffffff810be503 #2 [ffff88001fc03ab0] oops_end at ffffffff81005fc8 #3 [ffff88001fc03ae0] die at ffffffff81006463 #4 [ffff88001fc03b10] do_trap at ffffffff81002e12 #5 [ffff88001fc03b70] do_error_trap at ffffffff8100316d #6 [ffff88001fc03c30] do_invalid_op at ffffffff8100394b #7 [ffff88001fc03c40] invalid_op at ffffffff817f385e

    [exception RIP: skb_panic+94]
    RIP: ffffffff817eb99d  RSP: ffff88001fc03cf8  RFLAGS: 00010296
    RAX: 000000000000008b  RBX: ffff8800191f8980  RCX: 0000000000000092
    RDX: 000000000000002c  RSI: 0000000000000046  RDI: 0000000000000246
    RBP: ffff88001fc03d18   R8: 0000000000000000   R9: 0000000000000000
    R10: 00000000000001a8  R11: 0000000000000006  R12: 0000000000000564
    R13: ffff88001fc03da0  R14: ffff880019cb0800  R15: ffff880019001862
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018

#8 [ffff88001fc03d20] skb_put at ffffffff81611bb1
#9 [ffff88001fc03d30] batadv_frag_skb_buffer at ffffffffa001be12 [batman_adv]
#10 [ffff88001fc03d90] batadv_recv_frag_packet at ffffffffa0026273 [batman_adv] #11 [ffff88001fc03dd0] batadv_batman_skb_recv at ffffffffa001fef5 [batman_adv] #12 [ffff88001fc03e10] __netif_receive_skb_core at ffffffff81621962 #13 [ffff88001fc03e80] __netif_receive_skb at ffffffff81621e91
#14 [ffff88001fc03ea0] process_backlog at ffffffff81621f7e
#15 [ffff88001fc03ef0] net_rx_action at ffffffff81622731
#16 [ffff88001fc03f50] __do_softirq at ffffffff81053ef8
#17 [ffff88001fc03fb0] do_softirq_own_stack at ffffffff817f3a5c --- <IRQ stack> ---
#18 [ffff8800192b3d10] do_softirq_own_stack at ffffffff817f3a5c

    [exception RIP: tun_get_user+1056]
    RIP: ffffffffa00098f0  RSP: 0000000000000001  RFLAGS: 7fff00000586
    RAX: ffffffff816210b4  RBX: ffff8800192b3d58  RCX: ffff880019358780
    RDX: 0000000000000000  RSI: ffff880019358780  RDI: 0000000000000586
    RBP: ffffffff81620de4   R8: ffff8800192b3d88   R9: ffff880019358780
    R10: ffff880019358780  R11: ffffffff81054135  R12: ffff8800192b3d58
    R13: 0000000000000586  R14: ffff880013816900  R15: 0000000000000000
    ORIG_RAX: ffff8800192b3e38  CS: 7fffd0cc11e0  SS: 0000

bt: WARNING: possibly bogus exception frame
#19 [ffff8800192b3e40] tun_chr_aio_write at ffffffffa0009e0b [tun] #20 [ffff8800192b3e70] do_sync_write at ffffffff8115c665 #21 [ffff8800192b3f00] vfs_write at ffffffff8115d38a #22 [ffff8800192b3f40] sys_write at ffffffff8115d89a #23 [ffff8800192b3f80] system_call_fastpath at ffffffff817f1f29

    RIP: 00007f773e3f637d  RSP: 00007fffd0cc0f78  RFLAGS: 00010246
    RAX: 0000000000000001  RBX: ffffffff817f1f29  RCX: fffffffffffffffe
    RDX: 0000000000000586  RSI: 00000000008b5370  RDI: 0000000000000009
    RBP: 0000000000000586   R8: 00007f773e3df400   R9: 00007fffd0cc0928
    R10: 00007fffd0cc106f  R11: 0000000000000293  R12: 00000000008b4d78
    R13: 0000000000000001  R14: 00000000008b5360  R15: 00000000008a66a0
    ORIG_RAX: 0000000000000001  CS: 0033  SS: 002b

crash> log
[ … ]

[   82.041157] random: nonblocking pool is initialized
[  879.805754] tun: Universal TUN/TAP device driver, 1.6
[  879.805758] tun: (C) 1999-2004 Max Krasnyansky <[email protected]>
[  881.827196] batman_adv: B.A.T.M.A.N. advanced 2014.3.0 (compatibility version 15) loaded
[  882.061188] batman_adv: bat0: Adding interface: fastd0
[  882.061193] batman_adv: bat0: The MTU of interface fastd0 is too small (1426) to handle the transport of batman-adv packets. Packets going over this interface will be fragmented on layer2 which could impact the performance. Setting the MTU to 1560 would solve the problem.
[  882.061197] batman_adv: bat0: Interface activated: fastd0
[  882.062273] batman_adv: bat0: orig_interval: Changing from: 1000 to: 5000
[  882.063700] batman_adv: bat0: bridge_loop_avoidance: Changing from: disabled to: enabled
[  882.064445] batman_adv: bat0: Changing gw mode from: off to: client
[  901.324201] ipip: IPv4 over IPv4 tunneling driver
[  981.754520] batman_adv: bat0: Changing gw mode from: client to: server
[  981.754539] batman_adv: bat0: Changing gateway bandwidth from: '10.0/2.0 MBit' to: '100.0/100.0 MBit'
[ 4489.358966] nf_conntrack: automatic helper assignment is deprecated and it will be removed soon. Use the iptables CT target to attach helpers instead.
[65478.516847] rsync (19667) used greatest stack depth: 11608 bytes left
[106847.227697] UDP: bad checksum. From <some_IP_in_the_net>:X to <some_ISP_IP_for_this_machine>:X ulen 21
[117539.423168] skbuff: skb_over_panic: text:ffffffffa001be12 len:1445 put:1380 head:ffff88000fc85800 data:ffff88000fc85862 tail:0x607 end:0x2c0 dev:fastd0
[117539.423502] ------------[ cut here ]------------
[117539.423601] kernel BUG at net/core/skbuff.c:100!
[117539.423695] invalid opcode: 0000 [#1] SMP 
[117539.423796] Modules linked in: xt_nat iptable_nat nf_nat_ipv4 nf_nat ipip batman_adv libcrc32c tun crc32c_intel
[117539.424076] CPU: 0 PID: 2041 Comm: fastd Not tainted 3.16.7-gentoo #1
[117539.424107] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2007
[117539.424107] task: ffff88001a3a7290 ti: ffff8800192b0000 task.ti: ffff8800192b0000
[117539.424107] RIP: 0010:[<ffffffff817eb99d>]  [<ffffffff817eb99d>] skb_panic+0x5e/0x60
[117539.424107] RSP: 0018:ffff88001fc03cf8  EFLAGS: 00010296
[117539.424107] RAX: 000000000000008b RBX: ffff8800191f8980 RCX: 0000000000000092
[117539.424107] RDX: 000000000000002c RSI: 0000000000000046 RDI: 0000000000000246
[117539.424107] RBP: ffff88001fc03d18 R08: 0000000000000000 R09: 0000000000000000
[117539.424107] R10: 00000000000001a8 R11: 0000000000000006 R12: 0000000000000564
[117539.424107] R13: ffff88001fc03da0 R14: ffff880019cb0800 R15: ffff880019001862
[117539.424107] FS:  00007f773f0a2700(0000) GS:ffff88001fc00000(0000) knlGS:0000000000000000
[117539.424107] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[117539.424107] CR2: 00007f8cc7ada430 CR3: 000000001921c000 CR4: 00000000000006f0
[117539.424107] Stack:
[117539.424107]  ffff88000fc85862 0000000000000607 00000000000002c0 ffff880019358000
[117539.424107]  ffff88001fc03d28 ffffffff81611bb1 ffff88001fc03d88 ffffffffa001be12
[117539.424107]  ffff880019c07d28 ffff88001900184e ffff88001fc03d78 ffff8800191f8980
[117539.424107] Call Trace:
[117539.424107]  <IRQ> 
[117539.424107] 
[117539.424107]  [<ffffffff81611bb1>] skb_put+0x41/0x50
[117539.424107]  [<ffffffffa001be12>] batadv_frag_skb_buffer+0x272/0x470 [batman_adv]
[117539.424107]  [<ffffffffa0026273>] batadv_recv_frag_packet+0x183/0x200 [batman_adv]
[117539.424107]  [<ffffffffa001fef5>] batadv_batman_skb_recv+0xd5/0x110 [batman_adv]
[117539.424107]  [<ffffffff81621962>] __netif_receive_skb_core+0x222/0x730
[117539.424107]  [<ffffffff81621e91>] __netif_receive_skb+0x21/0x70
[117539.424107]  [<ffffffff81621f7e>] process_backlog+0x9e/0x170
[117539.424107]  [<ffffffff81622731>] net_rx_action+0x141/0x240
[117539.424107]  [<ffffffff81053ef8>] __do_softirq+0xe8/0x280
[117539.424107]  [<ffffffff817f3a5c>] do_softirq_own_stack+0x1c/0x30
[117539.424107]  <EOI> 
[117539.424107] 
[117539.424107]  [<ffffffff81054135>] do_softirq+0x55/0x60
[117539.424107]  [<ffffffff816210b4>] netif_rx_ni+0x34/0x70
[117539.424107]  [<ffffffffa00098f0>] tun_get_user+0x420/0x840 [tun]
[117539.424107]  [<ffffffffa0009e0b>] tun_chr_aio_write+0x7b/0xa0 [tun]
[117539.424107]  [<ffffffff8115c665>] do_sync_write+0x55/0x90
[117539.424107]  [<ffffffff8115d38a>] vfs_write+0xba/0x1f0
[117539.424107]  [<ffffffff8115d89a>] SyS_write+0x4a/0xa0
[117539.424107]  [<ffffffff817f1f29>] system_call_fastpath+0x16/0x1b
[117539.424107] Code: 00 00 48 89 44 24 10 8b 87 c0 00 00 00 48 89 44 24 08 48 8b 87 d0 00 00 00 48 c7 c7 30 67 a3 81 48 89 04 24 31 c0 e8 0d 8b ff ff <0f> 0b 55 48 89 f8 48 8b 57 30 48 89 e5 48 8b 0f 5d 80 e5 80 48 
[117539.424107] RIP  [<ffffffff817eb99d>] skb_panic+0x5e/0x60
[117539.424107]  RSP <ffff88001fc03cf8>

Hi Philiipp,

Can you help me do a quick sum-up?

1) At first it crashed with regular intervals (0 - 72 hours) with the backtrace you posted initially.
2) Then you disabled NC. Did it stop crashing at that point?
3) Then we enabled NC and added my patch, and it still does not crash?

I remeber you said it crashed with the distro-provided batman-adv module. Did you ensure to use the same version when running with my patch?

I haven't had time to dig into the reproduction of the crash, but I think I will do regardless.

Thanks,
Martin

Kind Regards,
Martin Hundebøll
Frederiks Allé 99A, 1.th
8000 Aarhus C

+45 61 65 54 61
[email protected]

Hi Martin

Am Montag, den 24.11.2014, 09:24 +0100 schrieb Martin Hundebøll:

At the beginning of the month some bug occurred on our two gateways with Arch Linux. MM and NC enabled. First bat-adv 2014.2.0 later bat-adv provided with the kernel. Kernel 3.14.23-ARCH later 3.17.1-ARCH. The bug can not be analysed on those VMs because I don't have access to their consoles.

I took a Gentoo Linux VM (bat-adv provided inside 3.16.6) with access to the console to ensure this bug is not Arch related. The bug occurs after three days. After kernel panic it can't be scrolled inside the console, so a complete trace-back is impossible. The end of the trace-back was the same I reported. I recompiled the Kernel (3.16.7) with debug symbols on and changed to it after one more crash.

No, with irregular intervals (0 - 72 hours). I think it has nothing to do with the time. With the Arch VMs I tried out this: one machine gw server the other gw client. After first VM's crash I immediately switched the other to gw server. After no time also this machine crashed. I think it has to be a bogus user packet.

I don't know which user sends bogus packages and I also can not ask our users what they are doing to crash our gateways.

I also don't know if the crash on Arch VM is the same on the Gentoo VM, with the back-trace I reported, but I assume.

NC has been disabled for 20 h before I patched the kernel, so it can't be told for sure that disabling stops the crashes.

After patching NC was enabled again to reproduce the bug. The VM crashed after 27 h. I could not retrieve the trace-back because I set the 'crashkernel' option to low. The next crash happened after 32:38:59. There has not been any batadv_frag_merge_packets in kernel ring buffer.

Yes. I patched /usr/src/linux/net/batman-adv/fragmentation.c I use batman-adv provided with the Kernel to reproduce all the steps. make modules recompiled only the batman-adv module, which I reloaded.

Please tell me, if you need some more information.

The VM's uptime is now 39 h. It survives Saturday evening and Sunday without a crash. I think the bug is NC related, but lets wait some more days until next Monday to tell for sure. In this time the users might do what they did in the past and trigger the bug.

Thank you for your time and making B.A.T.M.A.N.-adv better.

Best regards

Philipp

________________________
Freifunk Rheinland e. V.
– Funkzelle Wuppertal –

Hi Martin! Hi all!

Here we go again: same bug with disabled NC.

Now I run the VM with NC and MM disabled until next Monday.

Is this bug Batman-adv or kernel related?

Best regards

Philipp

________________________
Freifunk Rheinland e. V.
– Funkzelle Wuppertal –

SYSTEM MAP: /boot/System.map
DEBUG KERNEL: /usr/src/linux-3.16.7-gentoo/vmlinux (3.16.7-gentoo)

    DUMPFILE: vmcore_20141124124133
        CPUS: 1
        DATE: Mon Nov 24 12:35:05 2014
      UPTIME: 1 days, 16:17:07
LOAD AVERAGE: 0.15, 0.21, 0.20
       TASKS: 144
    NODENAME: wolke
     RELEASE: 3.16.7-gentoo
     VERSION: #1 SMP Mon Nov 17 03:44:22 CET 2014
     MACHINE: x86_64  (2593 Mhz)
      MEMORY: 511.6 MB
       PANIC: "kernel BUG at net/core/skbuff.c:100!"
         PID: 2008
     COMMAND: "fastd"
        TASK: ffff88001f3dbdb0  [THREAD_INFO: ffff880019214000]
         CPU: 0
       STATE: TASK_RUNNING (PANIC)

crash> bt
PID: 2008 TASK: ffff88001f3dbdb0 CPU: 0 COMMAND: "fastd"
#0 [ffff88001fc03980] machine_kexec at ffffffff8103a34e #1 [ffff88001fc039e0] crash_kexec at ffffffff810be503 #2 [ffff88001fc03ab0] oops_end at ffffffff81005fc8 #3 [ffff88001fc03ae0] die at ffffffff81006463 #4 [ffff88001fc03b10] do_trap at ffffffff81002e12 #5 [ffff88001fc03b70] do_error_trap at ffffffff8100316d #6 [ffff88001fc03c30] do_invalid_op at ffffffff8100394b #7 [ffff88001fc03c40] invalid_op at ffffffff817f385e

    [exception RIP: skb_panic+94]
    RIP: ffffffff817eb99d  RSP: ffff88001fc03cf8  RFLAGS: 00010296
    RAX: 000000000000008b  RBX: ffff880000011a40  RCX: 0000000000000092
    RDX: 0000000000000083  RSI: 0000000000000046  RDI: 0000000000000246
    RBP: ffff88001fc03d18   R8: 0000000000000000   R9: 0000000000000000
    R10: 00000000000001af  R11: 0000000000000006  R12: 0000000000000564
    R13: ffff88001fc03da0  R14: ffff88001f19d600  R15: ffff880013d55062
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018

#8 [ffff88001fc03d20] skb_put at ffffffff81611bb1
#9 [ffff88001fc03d30] batadv_frag_skb_buffer at ffffffffa001be12 [batman_adv]
#10 [ffff88001fc03d90] batadv_recv_frag_packet at ffffffffa0026273 [batman_adv] #11 [ffff88001fc03dd0] batadv_batman_skb_recv at ffffffffa001fef5 [batman_adv] #12 [ffff88001fc03e10] __netif_receive_skb_core at ffffffff81621962 #13 [ffff88001fc03e80] __netif_receive_skb at ffffffff81621e91
#14 [ffff88001fc03ea0] process_backlog at ffffffff81621f7e
#15 [ffff88001fc03ef0] net_rx_action at ffffffff81622731
#16 [ffff88001fc03f50] __do_softirq at ffffffff81053ef8
#17 [ffff88001fc03fb0] do_softirq_own_stack at ffffffff817f3a5c --- <IRQ stack> ---
#18 [ffff880019217d10] do_softirq_own_stack at ffffffff817f3a5c

    [exception RIP: tun_get_user+1056]
    RIP: ffffffffa00098f0  RSP: 0000000000000001  RFLAGS: 7fff00000586
    RAX: ffffffff816210b4  RBX: ffff880019217d58  RCX: ffff88001932c780
    RDX: 0000000000000000  RSI: ffff88001932c780  RDI: 0000000000000586
    RBP: ffffffff81620de4   R8: ffff880019217d88   R9: ffff88001932c780
    R10: ffff88001932c780  R11: ffffffff81054135  R12: ffff880019217d58
    R13: 0000000000000586  R14: ffff88001f19d800  R15: 0000000000000000
    ORIG_RAX: ffff880019217e38  CS: 7fff3207c250  SS: 0000

bt: WARNING: possibly bogus exception frame
#19 [ffff880019217e40] tun_chr_aio_write at ffffffffa0009e0b [tun] #20 [ffff880019217e70] do_sync_write at ffffffff8115c665 #21 [ffff880019217f00] vfs_write at ffffffff8115d38a #22 [ffff880019217f40] sys_write at ffffffff8115d89a #23 [ffff880019217f80] system_call_fastpath at ffffffff817f1f29

    RIP: 00007ffe0dbc637d  RSP: 00007fff3207c4d8  RFLAGS: 00010202
    RAX: 0000000000000001  RBX: ffffffff817f1f29  RCX: 0000000000000084
    RDX: 0000000000000586  RSI: 00000000007c0f00  RDI: 0000000000000009
    RBP: 0000000000000586   R8: 00007ffe0dbaf400   R9: 00007fff3207b998
    R10: 00007fff3207c0df  R11: 0000000000000293  R12: 00000000007c8278
    R13: 0000000000000001  R14: 00000000007c0ef0  R15: 00000000007b96a0
    ORIG_RAX: 0000000000000001  CS: 0033  SS: 002b

crash> log
[ … ]

[   85.585384] random: nonblocking pool is initialized
[  344.074524] tun: Universal TUN/TAP device driver, 1.6
[  344.074529] tun: (C) 1999-2004 Max Krasnyansky <[email protected]>
[  345.437185] batman_adv: B.A.T.M.A.N. advanced 2014.3.0 (compatibility version 15) loaded
[  345.586455] batman_adv: bat0: Adding interface: fastd0
[  345.586459] batman_adv: bat0: The MTU of interface fastd0 is too small (1426) to handle the transport of batman-adv packets. Packets going over this interface will be fragmented on layer2 which could impact the performance. Setting the MTU to 1560 would solve the problem.
[  345.586463] batman_adv: bat0: Interface activated: fastd0
[  345.587575] batman_adv: bat0: orig_interval: Changing from: 1000 to: 5000
[  345.589193] batman_adv: bat0: bridge_loop_avoidance: Changing from: disabled to: enabled
[  345.590126] batman_adv: bat0: Changing gw mode from: off to: client
[  385.143545] ipip: IPv4 over IPv4 tunneling driver
[  387.153331] dhcpd (2100) used greatest stack depth: 11944 bytes left
[  420.953976] batman_adv: bat0: Changing gw mode from: client to: server
[  420.954124] batman_adv: bat0: Changing gateway bandwidth from: '10.0/2.0 MBit' to: '100.0/100.0 MBit'
[  442.064172] batman_adv: bat0: network_coding: Changing from: enabled to: disabled
[  737.948219] nf_conntrack: automatic helper assignment is deprecated and it will be removed soon. Use the iptables CT target to attach helpers instead.
[ 6441.957012] dig (23207) used greatest stack depth: 11832 bytes left
[24933.266728] UDP: bad checksum. From _._._._:137 to _._._._ ulen 58
[24933.267969] UDP: bad checksum. From _._._._:137 to _._._._ ulen 58
[79218.832519] UDP: short packet: From _._._._:0 6117/1480 to _._._._:0
[88652.433174] UDP: short packet: From _._._._:0 6117/1480 to _._._._:0
[145027.089070] skbuff: skb_over_panic: text:ffffffffa001be12 len:1448 put:1380 head:ffff88000e4c0c00 data:ffff88000e4c0c62 tail:0x60a end:0x2c0 dev:fastd0
[145027.089388] ------------[ cut here ]------------
[145027.089492] kernel BUG at net/core/skbuff.c:100!
[145027.089590] invalid opcode: 0000 [#1] SMP 
[145027.089691] Modules linked in: xt_nat iptable_nat nf_nat_ipv4 nf_nat ipip batman_adv libcrc32c tun crc32c_intel
[145027.089977] CPU: 0 PID: 2008 Comm: fastd Not tainted 3.16.7-gentoo #1
[145027.090003] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2007
[145027.090003] task: ffff88001f3dbdb0 ti: ffff880019214000 task.ti: ffff880019214000
[145027.090003] RIP: 0010:[<ffffffff817eb99d>]  [<ffffffff817eb99d>] skb_panic+0x5e/0x60
[145027.090003] RSP: 0018:ffff88001fc03cf8  EFLAGS: 00010296
[145027.090003] RAX: 000000000000008b RBX: ffff880000011a40 RCX: 0000000000000092
[145027.090003] RDX: 0000000000000083 RSI: 0000000000000046 RDI: 0000000000000246
[145027.090003] RBP: ffff88001fc03d18 R08: 0000000000000000 R09: 0000000000000000
[145027.090003] R10: 00000000000001af R11: 0000000000000006 R12: 0000000000000564
[145027.090003] R13: ffff88001fc03da0 R14: ffff88001f19d600 R15: ffff880013d55062
[145027.090003] FS:  00007ffe0e872700(0000) GS:ffff88001fc00000(0000) knlGS:0000000000000000
[145027.090003] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[145027.090003] CR2: 00007f410b4a24b0 CR3: 0000000019282000 CR4: 00000000000006f0
[145027.090003] Stack:
[145027.090003]  ffff88000e4c0c62 000000000000060a 00000000000002c0 ffff88001932c000
[145027.090003]  ffff88001fc03d28 ffffffff81611bb1 ffff88001fc03d88 ffffffffa001be12
[145027.090003]  ffff880019cf59c8 ffff880013d5504e ffff88001fc03d78 ffff880000011a40
[145027.090003] Call Trace:
[145027.090003]  <IRQ> 
[145027.090003] 
[145027.090003]  [<ffffffff81611bb1>] skb_put+0x41/0x50
[145027.090003]  [<ffffffffa001be12>] batadv_frag_skb_buffer+0x272/0x470 [batman_adv]
[145027.090003]  [<ffffffffa0026273>] batadv_recv_frag_packet+0x183/0x200 [batman_adv]
[145027.090003]  [<ffffffffa001fef5>] batadv_batman_skb_recv+0xd5/0x110 [batman_adv]
[145027.090003]  [<ffffffff81621962>] __netif_receive_skb_core+0x222/0x730
[145027.090003]  [<ffffffff81621e91>] __netif_receive_skb+0x21/0x70
[145027.090003]  [<ffffffff81621f7e>] process_backlog+0x9e/0x170
[145027.090003]  [<ffffffff81622731>] net_rx_action+0x141/0x240
[145027.090003]  [<ffffffff81053ef8>] __do_softirq+0xe8/0x280
[145027.090003]  [<ffffffff817f3a5c>] do_softirq_own_stack+0x1c/0x30
[145027.090003]  <EOI> 
[145027.090003] 
[145027.090003]  [<ffffffff81054135>] do_softirq+0x55/0x60
[145027.090003]  [<ffffffff816210b4>] netif_rx_ni+0x34/0x70
[145027.090003]  [<ffffffffa00098f0>] tun_get_user+0x420/0x840 [tun]
[145027.090003]  [<ffffffffa0009e0b>] tun_chr_aio_write+0x7b/0xa0 [tun]
[145027.090003]  [<ffffffff8115c665>] do_sync_write+0x55/0x90
[145027.090003]  [<ffffffff8115d38a>] vfs_write+0xba/0x1f0
[145027.090003]  [<ffffffff8115d89a>] SyS_write+0x4a/0xa0
[145027.090003]  [<ffffffff817f1f29>] system_call_fastpath+0x16/0x1b
[145027.090003] Code: 00 00 48 89 44 24 10 8b 87 c0 00 00 00 48 89 44 24 08 48 8b 87 d0 00 00 00 48 c7 c7 30 67 a3 81 48 89 04 24 31 c0 e8 0d 8b ff ff <0f> 0b 55 48 89 f8 48 8b 57 30 48 89 e5 48 8b 0f 5d 80 e5 80 48 
[145027.090003] RIP  [<ffffffff817eb99d>] skb_panic+0x5e/0x60
[145027.090003]  RSP <ffff88001fc03cf8>

Hello again

Now it becomes a running gag, ;-) but I really try to provide you with some information about this bug.

Now we needn't to wait until next Monday. The bug appears also with NC and MM disabled.

Do you need this back-traces? Can you see something inside besides magic numbers? Somehow it should be possible to extract the bogus packages from vmcore with crash or some other utility to reproduce the bug.

Please tell me what to do.

Best regards

Philipp

________________________
Freifunk Rheinland e. V.
– Funkzelle Wuppertal –

SYSTEM MAP: /boot/System.map
DEBUG KERNEL: /usr/src/linux-3.16.7-gentoo/vmlinux (3.16.7-gentoo)

    DUMPFILE: vmcore_20141124212740
        CPUS: 1
        DATE: Mon Nov 24 21:01:11 2014
      UPTIME: 08:13:54
LOAD AVERAGE: 0.27, 0.28, 0.23
       TASKS: 125
    NODENAME: wolke
     RELEASE: 3.16.7-gentoo
     VERSION: #1 SMP Mon Nov 17 03:44:22 CET 2014
     MACHINE: x86_64  (2593 Mhz)
      MEMORY: 511.6 MB
       PANIC: "kernel BUG at net/core/skbuff.c:100!"
         PID: 1993
     COMMAND: "fastd"
        TASK: ffff88001f3369c0  [THREAD_INFO: ffff880019ff0000]
         CPU: 0
       STATE: TASK_RUNNING (PANIC)

crash> bt
PID: 1993 TASK: ffff88001f3369c0 CPU: 0 COMMAND: "fastd"
#0 [ffff88001fc03980] machine_kexec at ffffffff8103a34e #1 [ffff88001fc039e0] crash_kexec at ffffffff810be503 #2 [ffff88001fc03ab0] oops_end at ffffffff81005fc8 #3 [ffff88001fc03ae0] die at ffffffff81006463 #4 [ffff88001fc03b10] do_trap at ffffffff81002e12 #5 [ffff88001fc03b70] do_error_trap at ffffffff8100316d #6 [ffff88001fc03c30] do_invalid_op at ffffffff8100394b #7 [ffff88001fc03c40] invalid_op at ffffffff817f385e

    [exception RIP: skb_panic+94]
    RIP: ffffffff817eb99d  RSP: ffff88001fc03cf8  RFLAGS: 00010296
    RAX: 000000000000008b  RBX: ffff88000f9a4e80  RCX: 0000000000000092
    RDX: 000000000000001f  RSI: 0000000000000046  RDI: 0000000000000246
    RBP: ffff88001fc03d18   R8: 0000000000000000   R9: 0000000000000000
    R10: 00000000000001b0  R11: 0000000000000006  R12: 0000000000000564
    R13: ffff88001fc03da0  R14: ffff880019f69000  R15: ffff880007130062
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018

#8 [ffff88001fc03d20] skb_put at ffffffff81611bb1
#9 [ffff88001fc03d30] batadv_frag_skb_buffer at ffffffffa001be12 [batman_adv]
#10 [ffff88001fc03d90] batadv_recv_frag_packet at ffffffffa0026273 [batman_adv] #11 [ffff88001fc03dd0] batadv_batman_skb_recv at ffffffffa001fef5 [batman_adv] #12 [ffff88001fc03e10] __netif_receive_skb_core at ffffffff81621962 #13 [ffff88001fc03e80] __netif_receive_skb at ffffffff81621e91
#14 [ffff88001fc03ea0] process_backlog at ffffffff81621f7e
#15 [ffff88001fc03ef0] net_rx_action at ffffffff81622731
#16 [ffff88001fc03f50] __do_softirq at ffffffff81053ef8
#17 [ffff88001fc03fb0] do_softirq_own_stack at ffffffff817f3a5c --- <IRQ stack> ---
#18 [ffff880019ff3d10] do_softirq_own_stack at ffffffff817f3a5c

    [exception RIP: tun_get_user+1056]
    RIP: ffffffffa00098f0  RSP: 0000000000000001  RFLAGS: 7fff00000586
    RAX: ffffffff816210b4  RBX: ffff880019ff3d58  RCX: ffff880017710780
    RDX: 0000000000000000  RSI: ffff880017710780  RDI: 0000000000000586
    RBP: ffffffff81620de4   R8: ffff880019ff3d88   R9: ffff880017710780
    R10: ffff880017710780  R11: ffffffff81054135  R12: ffff880019ff3d58
    R13: 0000000000000586  R14: ffff880019f69e00  R15: 0000000000000000
    ORIG_RAX: ffff880019ff3e38  CS: 7ffffd27ec80  SS: 0000

bt: WARNING: possibly bogus exception frame
#19 [ffff880019ff3e40] tun_chr_aio_write at ffffffffa0009e0b [tun] #20 [ffff880019ff3e70] do_sync_write at ffffffff8115c665 #21 [ffff880019ff3f00] vfs_write at ffffffff8115d38a #22 [ffff880019ff3f40] sys_write at ffffffff8115d89a #23 [ffff880019ff3f80] system_call_fastpath at ffffffff817f1f29

    RIP: 00007f24ad2c037d  RSP: 00007ffffd27ea18  RFLAGS: 00010246
    RAX: 0000000000000001  RBX: ffffffff817f1f29  RCX: 0000000000000003
    RDX: 0000000000000586  RSI: 00000000008c9ff0  RDI: 0000000000000009
    RBP: 0000000000000586   R8: 00007f24ad2a9400   R9: 00007ffffd27e3c8
    R10: 00007ffffd27eb0f  R11: 0000000000000293  R12: 00000000008c99f8
    R13: 0000000000000001  R14: 00000000008c9fe0  R15: 00000000008b86e0
    ORIG_RAX: 0000000000000001  CS: 0033  SS: 002b
crash> log
[          … ]
[   46.421671] Adding 1571836k swap on /dev/vda2.  Priority:-1 extents:1 across:1571836k 
[   61.663553] crond (1525) used greatest stack depth: 11944 bytes left
[   94.611478] device eth0 entered promiscuous mode
[   95.050264] random: nonblocking pool is initialized
[   97.767718] sshd (1896) used greatest stack depth: 11848 bytes left
[  155.371394] tun: Universal TUN/TAP device driver, 1.6
[  155.371402] tun: (C) 1999-2004 Max Krasnyansky <[email protected]>
[  160.085539] batman_adv: B.A.T.M.A.N. advanced 2014.3.0 (compatibility version 15) loaded
[  161.520244] batman_adv: bat0: Adding interface: fastd0
[  161.520248] batman_adv: bat0: The MTU of interface fastd0 is too small (1426) to handle the transport of batman-adv packets. Packets going over this interface will be fragmented on layer2 which could impact the performance. Setting the MTU to 1560 would solve the problem.
[  161.520251] batman_adv: bat0: Interface activated: fastd0
[  161.521317] batman_adv: bat0: orig_interval: Changing from: 1000 to: 5000
[  161.522871] batman_adv: bat0: bridge_loop_avoidance: Changing from: disabled to: enabled
[  161.523606] batman_adv: bat0: Changing gw mode from: off to: client
[  163.534130] ipip: IPv4 over IPv4 tunneling driver
[  240.083770] batman_adv: bat0: Changing gw mode from: client to: server
[  240.083793] batman_adv: bat0: Changing gateway bandwidth from: '10.0/2.0 MBit' to: '100.0/100.0 MBit'
[  240.089611] batman_adv: bat0: network_coding: Changing from: enabled to: disabled
[  632.809027] nf_conntrack: automatic helper assignment is deprecated and it will be removed soon. Use the iptables CT target to attach helpers instead.
[  878.974740] batman_adv: bat0: multicast_mode: Changing from: enabled to: disabled
[ 7311.887428] named (9682) used greatest stack depth: 11720 bytes left
[25080.989287] UDP: bad checksum. From _._._._:26085 to _._._._:25901 ulen 76
[25081.134015] UDP: bad checksum. From _._._._:26085 to _._._._:25901 ulen 75
[25125.828688] UDP: bad checksum. From _._._._:26085 to _._._._:31502 ulen 79
[25225.232627] UDP: bad checksum. From _._._._:26085 to _._._._:25901 ulen 75
[25323.805796] UDP: bad checksum. From _._._._:26085 to _._._._:25901 ulen 76
[25355.026288] UDP: bad checksum. From _._._._:26085 to _._._._:25901 ulen 76
[29634.048268] skbuff: skb_over_panic: text:ffffffffa001be12 len:1459 put:1380 head:ffff8800049c4000 data:ffff8800049c4062 tail:0x615 end:0x2c0 dev:fastd0
[29634.048580] ------------[ cut here ]------------
[29634.048686] kernel BUG at net/core/skbuff.c:100!
[29634.048781] invalid opcode: 0000 [#1] SMP 
[29634.048881] Modules linked in: xt_nat iptable_nat nf_nat_ipv4 nf_nat ipip batman_adv libcrc32c tun crc32c_intel
[29634.049175] CPU: 0 PID: 1993 Comm: fastd Not tainted 3.16.7-gentoo #1
[29634.049198] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2007
[29634.049198] task: ffff88001f3369c0 ti: ffff880019ff0000 task.ti: ffff880019ff0000
[29634.049198] RIP: 0010:[<ffffffff817eb99d>]  [<ffffffff817eb99d>] skb_panic+0x5e/0x60
[29634.049198] RSP: 0018:ffff88001fc03cf8  EFLAGS: 00010296
[29634.049198] RAX: 000000000000008b RBX: ffff88000f9a4e80 RCX: 0000000000000092
[29634.049198] RDX: 000000000000001f RSI: 0000000000000046 RDI: 0000000000000246
[29634.049198] RBP: ffff88001fc03d18 R08: 0000000000000000 R09: 0000000000000000
[29634.049198] R10: 00000000000001b0 R11: 0000000000000006 R12: 0000000000000564
[29634.049198] R13: ffff88001fc03da0 R14: ffff880019f69000 R15: ffff880007130062
[29634.049198] FS:  00007f24adf6c700(0000) GS:ffff88001fc00000(0000) knlGS:0000000000000000
[29634.049198] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[29634.049198] CR2: 00007f9b1be31000 CR3: 00000000175ff000 CR4: 00000000000006f0
[29634.049198] Stack:
[29634.049198]  ffff8800049c4062 0000000000000615 00000000000002c0 ffff880017710000
[29634.049198]  ffff88001fc03d28 ffffffff81611bb1 ffff88001fc03d88 ffffffffa001be12
[29634.049198]  ffff8800176f21c8 ffff88000713004e ffff88001fc03d78 ffff88000f9a4e80
[29634.049198] Call Trace:
[29634.049198]  <IRQ> 
[29634.049198] 
[29634.049198]  [<ffffffff81611bb1>] skb_put+0x41/0x50
[29634.049198]  [<ffffffffa001be12>] batadv_frag_skb_buffer+0x272/0x470 [batman_adv]
[29634.049198]  [<ffffffffa0026273>] batadv_recv_frag_packet+0x183/0x200 [batman_adv]
[29634.049198]  [<ffffffffa001fef5>] batadv_batman_skb_recv+0xd5/0x110 [batman_adv]
[29634.049198]  [<ffffffff81621962>] __netif_receive_skb_core+0x222/0x730
[29634.049198]  [<ffffffff81621e91>] __netif_receive_skb+0x21/0x70
[29634.049198]  [<ffffffff81621f7e>] process_backlog+0x9e/0x170
[29634.049198]  [<ffffffff81622731>] net_rx_action+0x141/0x240
[29634.049198]  [<ffffffff81053ef8>] __do_softirq+0xe8/0x280
[29634.049198]  [<ffffffff817f3a5c>] do_softirq_own_stack+0x1c/0x30
[29634.049198]  <EOI> 
[29634.049198] 
[29634.049198]  [<ffffffff81054135>] do_softirq+0x55/0x60
[29634.049198]  [<ffffffff816210b4>] netif_rx_ni+0x34/0x70
[29634.049198]  [<ffffffffa00098f0>] tun_get_user+0x420/0x840 [tun]
[29634.049198]  [<ffffffffa0009e0b>] tun_chr_aio_write+0x7b/0xa0 [tun]
[29634.049198]  [<ffffffff8115c665>] do_sync_write+0x55/0x90
[29634.049198]  [<ffffffff8115d38a>] vfs_write+0xba/0x1f0
[29634.049198]  [<ffffffff8115d89a>] SyS_write+0x4a/0xa0
[29634.049198]  [<ffffffff817f1f29>] system_call_fastpath+0x16/0x1b
[29634.049198] Code: 00 00 48 89 44 24 10 8b 87 c0 00 00 00 48 89 44 24 08 48 8b 87 d0 00 00 00 48 c7 c7 30 67 a3 81 48 89 04 24 31 c0 e8 0d 8b ff ff <0f> 0b 55 48 89 f8 48 8b 57 30 48 89 e5 48 8b 0f 5d 80 e5 80 48 
[29634.049198] RIP  [<ffffffff817eb99d>] skb_panic+0x5e/0x60
[29634.049198]  RSP <ffff88001fc03cf8>

Hi all,

it's me again. There is another back-trace.

I can't help myself. Now I switch to 3.17.3 with your patch:

# cd net/batman-adv/ 
# patch < /usr/src/frag_debug_size.patch 
patching file fragmentation.c
Hunk #1 succeeded at 250 (offset 1 line).
# cd /usr/src/linux
# make
  CHK     include/config/kernel.release
  CHK     include/generated/uapi/linux/version.h
  CHK     include/generated/utsrelease.h
  CALL    scripts/checksyscalls.sh
  CHK     include/generated/compile.h
  CC [M]  net/batman-adv/fragmentation.o
  LD [M]  net/batman-adv/batman-adv.o
Kernel: arch/x86/boot/bzImage is ready  (#1)
  Building modules, stage 2.
  MODPOST 29 modules
  CC      net/batman-adv/batman-adv.mod.o
  LD [M]  net/batman-adv/batman-adv.ko
# make modules_install
[ … ]

3.17.4 compiles now for the next crash.

Best regards

Philipp

________________________
Freifunk Rheinland e. V.
– Funkzelle Wuppertal –

SYSTEM MAP: /boot/System.map
DEBUG KERNEL: /usr/src/linux-3.16.7-gentoo/vmlinux (3.16.7-gentoo)

    DUMPFILE: vmcore_20141124222013
        CPUS: 1
        DATE: Mon Nov 24 22:15:06 2014
      UPTIME: 00:46:49
LOAD AVERAGE: 0.14, 0.16, 0.14
       TASKS: 137
    NODENAME: wolke
     RELEASE: 3.16.7-gentoo
     VERSION: #1 SMP Mon Nov 17 03:44:22 CET 2014
     MACHINE: x86_64  (2593 Mhz)
      MEMORY: 511.6 MB
       PANIC: "kernel BUG at net/core/skbuff.c:100!"
         PID: 1971
     COMMAND: "fastd"
        TASK: ffff88001a31e0f0  [THREAD_INFO: ffff880019c5c000]
         CPU: 0
       STATE: TASK_RUNNING (PANIC)

crash> bt
PID: 1971 TASK: ffff88001a31e0f0 CPU: 0 COMMAND: "fastd"
#0 [ffff88001fc03980] machine_kexec at ffffffff8103a34e #1 [ffff88001fc039e0] crash_kexec at ffffffff810be503 #2 [ffff88001fc03ab0] oops_end at ffffffff81005fc8 #3 [ffff88001fc03ae0] die at ffffffff81006463 #4 [ffff88001fc03b10] do_trap at ffffffff81002e12 #5 [ffff88001fc03b70] do_error_trap at ffffffff8100316d #6 [ffff88001fc03c30] do_invalid_op at ffffffff8100394b #7 [ffff88001fc03c40] invalid_op at ffffffff817f385e

    [exception RIP: skb_panic+94]
    RIP: ffffffff817eb99d  RSP: ffff88001fc03cf8  RFLAGS: 00010296
    RAX: 000000000000008b  RBX: ffff8800057c9860  RCX: 0000000000000092
    RDX: 000000000000003c  RSI: 0000000000000046  RDI: 0000000000000246
    RBP: ffff88001fc03d18   R8: 0000000000000000   R9: 0000000000000000
    R10: 00000000000001a5  R11: 0000000000000006  R12: 0000000000000564
    R13: ffff88001fc03da0  R14: ffff8800191def00  R15: ffff880019200062
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018

#8 [ffff88001fc03d20] skb_put at ffffffff81611bb1
#9 [ffff88001fc03d30] batadv_frag_skb_buffer at ffffffffa001be12 [batman_adv]
#10 [ffff88001fc03d90] batadv_recv_frag_packet at ffffffffa0026273 [batman_adv] #11 [ffff88001fc03dd0] batadv_batman_skb_recv at ffffffffa001fef5 [batman_adv] #12 [ffff88001fc03e10] __netif_receive_skb_core at ffffffff81621962 #13 [ffff88001fc03e80] __netif_receive_skb at ffffffff81621e91
#14 [ffff88001fc03ea0] process_backlog at ffffffff81621f7e
#15 [ffff88001fc03ef0] net_rx_action at ffffffff81622731
#16 [ffff88001fc03f50] __do_softirq at ffffffff81053ef8
#17 [ffff88001fc03fb0] do_softirq_own_stack at ffffffff817f3a5c --- <IRQ stack> ---
#18 [ffff880019c5fd10] do_softirq_own_stack at ffffffff817f3a5c

    [exception RIP: tun_get_user+1056]
    RIP: ffffffffa00098f0  RSP: 0000000000000001  RFLAGS: 7fff00000586
    RAX: ffffffff816210b4  RBX: ffff880019c5fd58  RCX: ffff8800192f0780
    RDX: 0000000000000000  RSI: ffff8800192f0780  RDI: 0000000000000586
    RBP: ffffffff81620de4   R8: ffff880019c5fd88   R9: ffff8800192f0780
    R10: ffff8800192f0780  R11: ffffffff81054135  R12: ffff880019c5fd58
    R13: 0000000000000586  R14: ffff8800191de000  R15: 0000000000000000
    ORIG_RAX: ffff880019c5fe38  CS: 7fff4026c120  SS: 0000

bt: WARNING: possibly bogus exception frame
#19 [ffff880019c5fe40] tun_chr_aio_write at ffffffffa0009e0b [tun] #20 [ffff880019c5fe70] do_sync_write at ffffffff8115c665 #21 [ffff880019c5ff00] vfs_write at ffffffff8115d38a #22 [ffff880019c5ff40] sys_write at ffffffff8115d89a #23 [ffff880019c5ff80] system_call_fastpath at ffffffff817f1f29

    RIP: 00007fcaf8b9537d  RSP: 00007fff4026c3a8  RFLAGS: 00010202
    RAX: 0000000000000001  RBX: ffffffff817f1f29  RCX: 0000000000000084
    RDX: 0000000000000586  RSI: 00000000013e2f00  RDI: 0000000000000009
    RBP: 0000000000000586   R8: 00007fcaf8b7e400   R9: 00007fff4026b868
    R10: 00007fff4026bfaf  R11: 0000000000000293  R12: 00000000013d64d8
    R13: 0000000000000001  R14: 00000000013e2ef0  R15: 00000000013c46e0
    ORIG_RAX: 0000000000000001  CS: 0033  SS: 002b

crash> log

[   81.953301] tun: Universal TUN/TAP device driver, 1.6
[   81.953305] tun: (C) 1999-2004 Max Krasnyansky <[email protected]>
[   83.183042] batman_adv: B.A.T.M.A.N. advanced 2014.3.0 (compatibility version 15) loaded
[   83.444516] batman_adv: bat0: Adding interface: fastd0
[   83.444520] batman_adv: bat0: The MTU of interface fastd0 is too small (1426) to handle the transport of batman-adv packets. Packets going over this interface will be fragmented on layer2 which could impact the performance. Setting the MTU to 1560 would solve the problem.
[   83.444524] batman_adv: bat0: Interface activated: fastd0
[   83.445515] batman_adv: bat0: orig_interval: Changing from: 1000 to: 5000
[   83.446957] batman_adv: bat0: bridge_loop_avoidance: Changing from: disabled to: enabled
[   83.447682] batman_adv: bat0: Changing gw mode from: off to: client
[  132.723904] ipip: IPv4 over IPv4 tunneling driver
[  188.931093] batman_adv: bat0: Changing gw mode from: client to: server
[  188.931117] batman_adv: bat0: Changing gateway bandwidth from: '10.0/2.0 MBit' to: '100.0/100.0 MBit'
[  317.574742] nf_conntrack: automatic helper assignment is deprecated and it will be removed soon. Use the iptables CT target to attach helpers instead.
[ 2809.763665] skbuff: skb_over_panic: text:ffffffffa001be12 len:1472 put:1380 head:ffff88000719b400 data:ffff88000719b462 tail:0x622 end:0x2c0 dev:fastd0
[ 2809.763978] ------------[ cut here ]------------
[ 2809.764097] kernel BUG at net/core/skbuff.c:100!
[ 2809.764195] invalid opcode: 0000 [#1] SMP 
[ 2809.764297] Modules linked in: xt_nat iptable_nat nf_nat_ipv4 nf_nat ipip batman_adv libcrc32c tun crc32c_intel
[ 2809.764604] CPU: 0 PID: 1971 Comm: fastd Not tainted 3.16.7-gentoo #1
[ 2809.764604] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2007
[ 2809.764604] task: ffff88001a31e0f0 ti: ffff880019c5c000 task.ti: ffff880019c5c000
[ 2809.764604] RIP: 0010:[<ffffffff817eb99d>]  [<ffffffff817eb99d>] skb_panic+0x5e/0x60
[ 2809.764604] RSP: 0018:ffff88001fc03cf8  EFLAGS: 00010296
[ 2809.764604] RAX: 000000000000008b RBX: ffff8800057c9860 RCX: 0000000000000092
[ 2809.764604] RDX: 000000000000003c RSI: 0000000000000046 RDI: 0000000000000246
[ 2809.764604] RBP: ffff88001fc03d18 R08: 0000000000000000 R09: 0000000000000000
[ 2809.764604] R10: 00000000000001a5 R11: 0000000000000006 R12: 0000000000000564
[ 2809.764604] R13: ffff88001fc03da0 R14: ffff8800191def00 R15: ffff880019200062
[ 2809.764604] FS:  00007fcaf9841700(0000) GS:ffff88001fc00000(0000) knlGS:0000000000000000
[ 2809.764604] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 2809.764604] CR2: 00007f0953452000 CR3: 00000000191da000 CR4: 00000000000006f0
[ 2809.764604] Stack:
[ 2809.764604]  ffff88000719b462 0000000000000622 00000000000002c0 ffff8800192f0000
[ 2809.764604]  ffff88001fc03d28 ffffffff81611bb1 ffff88001fc03d88 ffffffffa001be12
[ 2809.764604]  ffff88001f33d168 ffff88001920004e ffff88001fc03d78 ffff8800057c9860
[ 2809.764604] Call Trace:
[ 2809.764604]  <IRQ> 
[ 2809.764604] 
[ 2809.764604]  [<ffffffff81611bb1>] skb_put+0x41/0x50
[ 2809.764604]  [<ffffffffa001be12>] batadv_frag_skb_buffer+0x272/0x470 [batman_adv]
[ 2809.764604]  [<ffffffffa0026273>] batadv_recv_frag_packet+0x183/0x200 [batman_adv]
[ 2809.764604]  [<ffffffffa001fef5>] batadv_batman_skb_recv+0xd5/0x110 [batman_adv]
[ 2809.764604]  [<ffffffff81621962>] __netif_receive_skb_core+0x222/0x730
[ 2809.764604]  [<ffffffff81621e91>] __netif_receive_skb+0x21/0x70
[ 2809.764604]  [<ffffffff81621f7e>] process_backlog+0x9e/0x170
[ 2809.764604]  [<ffffffff81622731>] net_rx_action+0x141/0x240
[ 2809.764604]  [<ffffffff81053ef8>] __do_softirq+0xe8/0x280
[ 2809.764604]  [<ffffffff817f3a5c>] do_softirq_own_stack+0x1c/0x30
[ 2809.764604]  <EOI> 
[ 2809.764604] 
[ 2809.764604]  [<ffffffff81054135>] do_softirq+0x55/0x60
[ 2809.764604]  [<ffffffff816210b4>] netif_rx_ni+0x34/0x70
[ 2809.764604]  [<ffffffffa00098f0>] tun_get_user+0x420/0x840 [tun]
[ 2809.764604]  [<ffffffffa0009e0b>] tun_chr_aio_write+0x7b/0xa0 [tun]
[ 2809.764604]  [<ffffffff8115c665>] do_sync_write+0x55/0x90
[ 2809.764604]  [<ffffffff8115d38a>] vfs_write+0xba/0x1f0
[ 2809.764604]  [<ffffffff8115d89a>] SyS_write+0x4a/0xa0
[ 2809.764604]  [<ffffffff817f1f29>] system_call_fastpath+0x16/0x1b
[ 2809.764604] Code: 00 00 48 89 44 24 10 8b 87 c0 00 00 00 48 89 44 24 08 48 8b 87 d0 00 00 00 48 c7 c7 30 67 a3 81 48 89 04 24 31 c0 e8 0d 8b ff ff <0f> 0b 55 48 89 f8 48 8b 57 30 48 89 e5 48 8b 0f 5d 80 e5 80 48 
[ 2809.764604] RIP  [<ffffffff817eb99d>] skb_panic+0x5e/0x60
[ 2809.764604]  RSP <ffff88001fc03cf8>

Hi Martin, hi all

The next crash with 3.17.3 – I’m kind of speechless. The heavy Load is due to compiling with nice -n 19.

I set the gw mode to client. I need sleep. 3.17.4 compiles …

Kind regards

Philipp

________________________
Freifunk Rheinland e. V.
– Funkzelle Wuppertal –

SYSTEM MAP: /boot/System.map
DEBUG KERNEL: /usr/src/linux-3.17.3-gentoo/vmlinux (3.17.3-gentoo)

    DUMPFILE: vmcore_20141125003014
        CPUS: 1
        DATE: Thu Jan  1 01:00:00 1970
      UPTIME: 00:39:55
LOAD AVERAGE: 1.59, 1.56, 1.23
       TASKS: 152
    NODENAME: wolke
     RELEASE: 3.17.3-gentoo
     VERSION: #1 SMP Wed Nov 19 15:42:49 CET 2014
     MACHINE: x86_64  (2593 Mhz)
      MEMORY: 511.6 MB
       PANIC: "kernel BUG at net/core/skbuff.c:100!"
         PID: 2036
     COMMAND: "fastd"
        TASK: ffff880019cef290  [THREAD_INFO: ffff88001768c000]
         CPU: 0
       STATE: TASK_RUNNING (PANIC)

crash> bt
PID: 2036 TASK: ffff880019cef290 CPU: 0 COMMAND: "fastd"
#0 [ffff88001fc03980] machine_kexec at ffffffff8103aace #1 [ffff88001fc039e0] crash_kexec at ffffffff810bfa43 #2 [ffff88001fc03ab0] oops_end at ffffffff810060f8 #3 [ffff88001fc03ae0] die at ffffffff81006593 #4 [ffff88001fc03b10] do_trap at ffffffff81002ef2 #5 [ffff88001fc03b70] do_error_trap at ffffffff8100305d #6 [ffff88001fc03c30] do_invalid_op at ffffffff81003a7b #7 [ffff88001fc03c40] invalid_op at ffffffff8180e79e

    [exception RIP: skb_panic+94]
    RIP: ffffffff81806a38  RSP: ffff88001fc03cf8  RFLAGS: 00010296
    RAX: 000000000000008b  RBX: ffff88001a2bbd20  RCX: 0000000000000092
    RDX: 00000000000000c2  RSI: 0000000000000246  RDI: 0000000000000246
    RBP: ffff88001fc03d18   R8: 0000000000000000   R9: 0000000000000000
    R10: ffffffff81a6b240  R11: 0000000000000000  R12: 0000000000000564
    R13: ffff88001fc03da0  R14: ffff88000f877000  R15: ffff880002c56862
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018

#8 [ffff88001fc03d20] skb_put at ffffffff8162c921
#9 [ffff88001fc03d30] batadv_frag_skb_buffer at ffffffffa001be32 [batman_adv]
#10 [ffff88001fc03d90] batadv_recv_frag_packet at ffffffffa0026283 [batman_adv] #11 [ffff88001fc03dd0] batadv_batman_skb_recv at ffffffffa001ff15 [batman_adv] #12 [ffff88001fc03e10] __netif_receive_skb_core at ffffffff8163c752 #13 [ffff88001fc03e80] __netif_receive_skb at ffffffff8163cc91
#14 [ffff88001fc03ea0] process_backlog at ffffffff8163cd7e
#15 [ffff88001fc03ef0] net_rx_action at ffffffff8163d531
#16 [ffff88001fc03f50] __do_softirq at ffffffff81052e28
#17 [ffff88001fc03fb0] do_softirq_own_stack at ffffffff8180e99c --- <IRQ stack> ---
#18 [ffff88001768fd10] do_softirq_own_stack at ffffffff8180e99c

    [exception RIP: tun_get_user+1043]
    RIP: ffffffffa00098e3  RSP: 0000000000000001  RFLAGS: ff880000000586
    RAX: ffffffff8163bca4  RBX: ffff88001768fd58  RCX: ffff880017638780
    RDX: 0000000000000000  RSI: ffff880017638780  RDI: 0000000000000586
    RBP: ffffffff8163b9d4   R8: ffff88001768fd88   R9: ffff880017638780
    R10: ffff880017638780  R11: ffffffff81053065  R12: ffff88001768fd58
    R13: 0000000000000586  R14: ffff88001770f700  R15: 0000000000000000
    ORIG_RAX: ffff88001768fe38  CS: ffffea00005dbef0  SS: 0000

bt: WARNING: possibly bogus exception frame
#19 [ffff88001768fe40] tun_chr_aio_write at ffffffffa0009e0b [tun] #20 [ffff88001768fe70] do_sync_write at ffffffff8115fdb5 #21 [ffff88001768ff00] vfs_write at ffffffff81160ada #22 [ffff88001768ff40] sys_write at ffffffff81160fea #23 [ffff88001768ff80] system_call_fastpath at ffffffff8180ce69

    RIP: 00007f0c818c037d  RSP: 00007fff6da33dc0  RFLAGS: 00010206
    RAX: 0000000000000001  RBX: ffffffff8180ce69  RCX: 0000000000000000
    RDX: 0000000000000586  RSI: 0000000000c74300  RDI: 0000000000000009
    RBP: 0000000000000586   R8: 00007f0c818a9400   R9: 00007fff6da34508
    R10: 00007fff6da34c4f  R11: 0000000000000293  R12: 0000000000c7dee8
    R13: 0000000000000001  R14: 0000000000c742f0  R15: 0000000000c616e0
    ORIG_RAX: 0000000000000001  CS: 0033  SS: 002b
crash> log
[    0.000000] Initializing cgroup subsys cpuset
[    0.000000] Initializing cgroup subsys cpu
[    0.000000] Initializing cgroup subsys cpuacct
[    0.000000] Linux version 3.17.3-gentoo (root@wolke) (gcc version 4.7.3 (Gentoo 4.7.3-r1 p1.4, pie-0.5.5) ) #1 SMP Wed Nov 19 15:42:49 CET 2014
[    0.000000] Command line: root=/dev/vda1 raid=noautodetect crashkernel=64M
[    0.000000] e820: BIOS-provided physical RAM map:
[    0.000000] BIOS-e820: [mem 0x0000000000000000-0x000000000009dbff] usable
[    0.000000] BIOS-e820: [mem 0x000000000009dc00-0x000000000009ffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000000f0000-0x00000000000fffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000000100000-0x000000001fffdfff] usable
[    0.000000] BIOS-e820: [mem 0x000000001fffe000-0x000000001fffffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000feffc000-0x00000000feffffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fffc0000-0x00000000ffffffff] reserved
[    0.000000] NX (Execute Disable) protection: active
[    0.000000] SMBIOS 2.4 present.
[    0.000000] DMI: Bochs Bochs, BIOS Bochs 01/01/2007
[    0.000000] Hypervisor detected: KVM
[    0.000000] e820: update [mem 0x00000000-0x00000fff] usable ==> reserved
[    0.000000] e820: remove [mem 0x000a0000-0x000fffff] usable
[    0.000000] e820: last_pfn = 0x1fffe max_arch_pfn = 0x400000000
[    0.000000] MTRR default type: write-back
[    0.000000] MTRR fixed ranges enabled:
[    0.000000]   00000-9FFFF write-back
[    0.000000]   A0000-BFFFF uncachable
[    0.000000]   C0000-FFFFF write-protect
[    0.000000] MTRR variable ranges enabled:
[    0.000000]   0 base 00E0000000 mask FFE0000000 uncachable
[    0.000000]   1 disabled
[    0.000000]   2 disabled
[    0.000000]   3 disabled
[    0.000000]   4 disabled
[    0.000000]   5 disabled
[    0.000000]   6 disabled
[    0.000000]   7 disabled
[    0.000000] x86 PAT enabled: cpu 0, old 0x70406, new 0x7010600070106
[    0.000000] found SMP MP-table at [mem 0x000fdaf0-0x000fdaff] mapped at [ffff8800000fdaf0]
[    0.000000] Scanning 1 areas for low memory corruption
[    0.000000] Base memory trampoline at [ffff880000097000] 97000 size 24576
[    0.000000] init_memory_mapping: [mem 0x00000000-0x000fffff]
[    0.000000]  [mem 0x00000000-0x000fffff] page 4k
[    0.000000] BRK [0x020d7000, 0x020d7fff] PGTABLE
[    0.000000] BRK [0x020d8000, 0x020d8fff] PGTABLE
[    0.000000] BRK [0x020d9000, 0x020d9fff] PGTABLE
[    0.000000] init_memory_mapping: [mem 0x1fc00000-0x1fdfffff]
[    0.000000]  [mem 0x1fc00000-0x1fdfffff] page 2M
[    0.000000] init_memory_mapping: [mem 0x1c000000-0x1fbfffff]
[    0.000000]  [mem 0x1c000000-0x1fbfffff] page 2M
[    0.000000] init_memory_mapping: [mem 0x00100000-0x1bffffff]
[    0.000000]  [mem 0x00100000-0x001fffff] page 4k
[    0.000000]  [mem 0x00200000-0x1bffffff] page 2M
[    0.000000] init_memory_mapping: [mem 0x1fe00000-0x1fffdfff]
[    0.000000]  [mem 0x1fe00000-0x1fffdfff] page 4k
[    0.000000] BRK [0x020da000, 0x020dafff] PGTABLE
[    0.000000] ACPI: Early table checksum verification disabled
[    0.000000] ACPI: RSDP 0x00000000000FD990 000014 (v00 BOCHS )
[    0.000000] ACPI: RSDT 0x000000001FFFE5B0 000038 (v01 BOCHS  BXPCRSDT 00000001 BXPC 00000001)
[    0.000000] ACPI: FACP 0x000000001FFFFF80 000074 (v01 BOCHS  BXPCFACP 00000001 BXPC 00000001)
[    0.000000] ACPI: DSDT 0x000000001FFFE5F0 001121 (v01 BXPC   BXDSDT   00000001 INTL 20100528)
[    0.000000] ACPI: FACS 0x000000001FFFFF40 000040
[    0.000000] ACPI: SSDT 0x000000001FFFFEA0 00009E (v01 BOCHS  BXPCSSDT 00000001 BXPC 00000001)
[    0.000000] ACPI: APIC 0x000000001FFFFDB0 000078 (v01 BOCHS  BXPCAPIC 00000001 BXPC 00000001)
[    0.000000] ACPI: HPET 0x000000001FFFFD70 000038 (v01 BOCHS  BXPCHPET 00000001 BXPC 00000001)
[    0.000000] ACPI: SSDT 0x000000001FFFF720 000644 (v01 BXPC   BXSSDTPC 00000001 INTL 20100528)
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] No NUMA configuration found
[    0.000000] Faking a node at [mem 0x0000000000000000-0x000000001fffdfff]
[    0.000000] Initmem setup node 0 [mem 0x00000000-0x1fffdfff]
[    0.000000]   NODE_DATA [mem 0x1fffa000-0x1fffdfff]
[    0.000000] Reserving 64MB of memory at 432MB for crashkernel (System RAM: 511MB)
[    0.000000] kvm-clock: Using msrs 4b564d01 and 4b564d00
[    0.000000] kvm-clock: cpu 0, msr 0:1fff9001, primary cpu clock
[    0.000000]  [ffffea0000000000-ffffea00007fffff] PMD -> [ffff88001a800000-ffff88001affffff] on node 0
[    0.000000] Zone ranges:
[    0.000000]   DMA      [mem 0x00001000-0x00ffffff]
[    0.000000]   DMA32    [mem 0x01000000-0xffffffff]
[    0.000000]   Normal   empty
[    0.000000] Movable zone start for each node
[    0.000000] Early memory node ranges
[    0.000000]   node   0: [mem 0x00001000-0x0009cfff]
[    0.000000]   node   0: [mem 0x00100000-0x1fffdfff]
[    0.000000] On node 0 totalpages: 130970
[    0.000000]   DMA zone: 64 pages used for memmap
[    0.000000]   DMA zone: 21 pages reserved
[    0.000000]   DMA zone: 3996 pages, LIFO batch:0
[    0.000000]   DMA32 zone: 1984 pages used for memmap
[    0.000000]   DMA32 zone: 126974 pages, LIFO batch:31
[    0.000000] ACPI: PM-Timer IO Port: 0xb008
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] ACPI: LAPIC (acpi_id[0x00] lapic_id[0x00] enabled)
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xff] dfl dfl lint[0x1])
[    0.000000] ACPI: IOAPIC (id[0x01] address[0xfec00000] gsi_base[0])
[    0.000000] IOAPIC[0]: apic_id 1, version 17, address 0xfec00000, GSI 0-23
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 5 global_irq 5 high level)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 10 global_irq 10 high level)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 11 global_irq 11 high level)
[    0.000000] ACPI: IRQ0 used by override.
[    0.000000] ACPI: IRQ5 used by override.
[    0.000000] ACPI: IRQ9 used by override.
[    0.000000] ACPI: IRQ10 used by override.
[    0.000000] ACPI: IRQ11 used by override.
[    0.000000] Using ACPI (MADT) for SMP configuration information
[    0.000000] ACPI: HPET id: 0x8086a201 base: 0xfed00000
[    0.000000] smpboot: Allowing 1 CPUs, 0 hotplug CPUs
[    0.000000] PM: Registered nosave memory: [mem 0x0009d000-0x0009dfff]
[    0.000000] PM: Registered nosave memory: [mem 0x0009e000-0x0009ffff]
[    0.000000] PM: Registered nosave memory: [mem 0x000a0000-0x000effff]
[    0.000000] PM: Registered nosave memory: [mem 0x000f0000-0x000fffff]
[    0.000000] e820: [mem 0x20000000-0xfeffbfff] available for PCI devices
[    0.000000] Booting paravirtualized kernel on KVM
[    0.000000] setup_percpu: NR_CPUS:64 nr_cpumask_bits:64 nr_cpu_ids:1 nr_node_ids:1
[    0.000000] PERCPU: Embedded 27 pages/cpu @ffff88001fc00000 s79744 r8192 d22656 u2097152
[    0.000000] pcpu-alloc: s79744 r8192 d22656 u2097152 alloc=1*2097152
[    0.000000] pcpu-alloc: [0] 0 
[    0.000000] KVM setup async PF for cpu 0
[    0.000000] kvm-stealtime: cpu 0, msr 1fc0cf80
[    0.000000] Built 1 zonelists in Node order, mobility grouping on.  Total pages: 128901
[    0.000000] Policy zone: DMA32
[    0.000000] Kernel command line: root=/dev/vda1 raid=noautodetect crashkernel=64M
[    0.000000] PID hash table entries: 2048 (order: 2, 16384 bytes)
[    0.000000] Calgary: detecting Calgary via BIOS EBDA area
[    0.000000] Calgary: Unable to locate Rio Grande table in EBDA - bailing!
[    0.000000] Memory: 432620K/523880K available (8263K kernel code, 863K rwdata, 2684K rodata, 1108K init, 896K bss, 91260K reserved)
[    0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=1, Nodes=1
[    0.000000] Hierarchical RCU implementation.
[    0.000000]  RCU restricting CPUs from NR_CPUS=64 to nr_cpu_ids=1.
[    0.000000] RCU: Adjusting geometry for rcu_fanout_leaf=16, nr_cpu_ids=1
[    0.000000] NR_IRQS:4352 nr_irqs:256 0
[    0.000000] Console: colour VGA+ 80x25
[    0.000000] console [tty0] enabled
[    0.000000] hpet clockevent registered
[    0.000000] tsc: Detected 2593.748 MHz processor
[    0.002000] Calibrating delay loop (skipped) preset value.. 5187.49 BogoMIPS (lpj=2593748)
[    0.002005] pid_max: default: 32768 minimum: 301
[    0.002393] ACPI: Core revision 20140724
[    0.003739] ACPI: All ACPI Tables successfully acquired
[    0.004441] Security Framework initialized
[    0.004809] SELinux:  Initializing.
[    0.005011] SELinux:  Starting in permissive mode
[    0.005045] Dentry cache hash table entries: 65536 (order: 7, 524288 bytes)
[    0.005748] Inode-cache hash table entries: 32768 (order: 6, 262144 bytes)
[    0.006135] Mount-cache hash table entries: 1024 (order: 1, 8192 bytes)
[    0.006549] Mountpoint-cache hash table entries: 1024 (order: 1, 8192 bytes)
[    0.007238] Initializing cgroup subsys freezer
[    0.008079] mce: CPU supports 10 MCE banks
[    0.008491] Last level iTLB entries: 4KB 0, 2MB 0, 4MB 0
               Last level dTLB entries: 4KB 0, 2MB 0, 4MB 0, 1GB 0
[    0.026277] Freeing SMP alternatives memory: 28K (ffffffff81fee000 - ffffffff81ff5000)
[    0.030669] ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1
[    0.031002] smpboot: CPU0: Intel Westmere E56xx/L56xx/X56xx (Nehalem-C) (fam: 06, model: 2c, stepping: 01)
[    0.032000] Performance Events: unsupported p6 CPU model 44 no PMU driver, software events only.
[    0.032522] x86: Booted up 1 node, 1 CPUs
[    0.033004] smpboot: Total of 1 processors activated (5187.49 BogoMIPS)
[    0.034267] devtmpfs: initialized
[    0.035367] RTC time: 22:47:53, date: 11/24/14
[    0.036009] NET: Registered protocol family 16
[    0.036494] cpuidle: using governor ladder
[    0.036809] cpuidle: using governor menu
[    0.037085] ACPI: bus type PCI registered
[    0.037498] PCI: Using configuration type 1 for base access
[    0.040447] kworker/u2:0 (15) used greatest stack depth: 14664 bytes left
[    0.041100] ACPI: Added _OSI(Module Device)
[    0.041421] ACPI: Added _OSI(Processor Device)
[    0.041738] ACPI: Added _OSI(3.0 _SCP Extensions)
[    0.042004] ACPI: Added _OSI(Processor Aggregator Device)
[    0.043755] ACPI: Interpreter enabled
[    0.044028] ACPI Exception: AE_NOT_FOUND, While evaluating Sleep State [\_S1_] (20140724/hwxface-580)
[    0.045429] ACPI Exception: AE_NOT_FOUND, While evaluating Sleep State [\_S2_] (20140724/hwxface-580)
[    0.046332] ACPI: (supports S0 S3 S4 S5)
[    0.047005] ACPI: Using IOAPIC for interrupt routing
[    0.047459] PCI: Ignoring host bridge windows from ACPI; if necessary, use "pci=use_crs" and report a bug
[    0.051042] ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-ff])
[    0.051560] acpi PNP0A03:00: _OSC: OS supports [ASPM ClockPM Segments MSI]
[    0.052048] acpi PNP0A03:00: _OSC failed (AE_NOT_FOUND); disabling ASPM
[    0.052558] acpi PNP0A03:00: host bridge window [io  0x0000-0x0cf7] (ignored)
[    0.052560] acpi PNP0A03:00: host bridge window [io  0x0d00-0xffff] (ignored)
[    0.052561] acpi PNP0A03:00: host bridge window [mem 0x000a0000-0x000bffff] (ignored)
[    0.052563] acpi PNP0A03:00: host bridge window [mem 0xe0000000-0xfebfffff] (ignored)
[    0.052565] PCI: root bus 00: using default resources
[    0.052567] acpi PNP0A03:00: fail to add MMCONFIG information, can't access extended PCI configuration space under this bridge.
[    0.053067] PCI host bridge to bus 0000:00
[    0.054046] pci_bus 0000:00: root bus resource [bus 00-ff]
[    0.054567] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
[    0.055045] pci_bus 0000:00: root bus resource [mem 0x00000000-0xffffffffff]
[    0.055696] pci 0000:00:00.0: [8086:1237] type 00 class 0x060000
[    0.056486] pci 0000:00:01.0: [8086:7000] type 00 class 0x060100
[    0.057055] pci 0000:00:01.1: [8086:7010] type 00 class 0x010180
[    0.058611] pci 0000:00:01.1: reg 0x20: [io  0xc0a0-0xc0af]
[    0.059317] pci 0000:00:01.1: legacy IDE quirk: reg 0x10: [io  0x01f0-0x01f7]
[    0.059691] pci 0000:00:01.1: legacy IDE quirk: reg 0x14: [io  0x03f6]
[    0.060005] pci 0000:00:01.1: legacy IDE quirk: reg 0x18: [io  0x0170-0x0177]
[    0.060435] pci 0000:00:01.1: legacy IDE quirk: reg 0x1c: [io  0x0376]
[    0.061090] pci 0000:00:01.2: [8086:7020] type 00 class 0x0c0300
[    0.062641] pci 0000:00:01.2: reg 0x20: [io  0xc040-0xc05f]
[    0.063530] pci 0000:00:01.3: [8086:7113] type 00 class 0x068000
[    0.063845] pci 0000:00:01.3: quirk: [io  0xb000-0xb03f] claimed by PIIX4 ACPI
[    0.064015] pci 0000:00:01.3: quirk: [io  0xb100-0xb10f] claimed by PIIX4 SMB
[    0.065052] pci 0000:00:02.0: [1013:00b8] type 00 class 0x030000
[    0.068043] pci 0000:00:02.0: reg 0x10: [mem 0xfc000000-0xfdffffff pref]
[    0.070033] pci 0000:00:02.0: reg 0x14: [mem 0xfebf0000-0xfebf0fff]
[    0.080036] pci 0000:00:02.0: reg 0x30: [mem 0xfebd0000-0xfebdffff pref]
[    0.080592] pci 0000:00:03.0: [1af4:1000] type 00 class 0x020000
[    0.081298] pci 0000:00:03.0: reg 0x10: [io  0xc060-0xc07f]
[    0.081854] pci 0000:00:03.0: reg 0x14: [mem 0xfebf1000-0xfebf1fff]
[    0.084556] pci 0000:00:03.0: reg 0x30: [mem 0xfebe0000-0xfebeffff pref]
[    0.085345] kworker/u2:0 (34) used greatest stack depth: 13912 bytes left
[    0.085919] pci 0000:00:04.0: [1af4:1001] type 00 class 0x010000
[    0.087004] pci 0000:00:04.0: reg 0x10: [io  0xc000-0xc03f]
[    0.088362] pci 0000:00:04.0: reg 0x14: [mem 0xfebf2000-0xfebf2fff]
[    0.093240] pci 0000:00:05.0: [1af4:1002] type 00 class 0x00ff00
[    0.093581] pci 0000:00:05.0: reg 0x10: [io  0xc080-0xc09f]
[    0.096168] ACPI: PCI Interrupt Link [LNKA] (IRQs 5 *10 11)
[    0.096890] ACPI: PCI Interrupt Link [LNKB] (IRQs 5 *10 11)
[    0.097607] ACPI: PCI Interrupt Link [LNKC] (IRQs 5 10 *11)
[    0.098849] ACPI: PCI Interrupt Link [LNKD] (IRQs 5 10 *11)
[    0.099661] ACPI: PCI Interrupt Link [LNKS] (IRQs 9) *0, disabled.
[    0.100837] ACPI: Enabled 16 GPEs in block 00 to 0F
[    0.101233] vgaarb: setting as boot device: PCI:0000:00:02.0
[    0.101579] vgaarb: device added: PCI:0000:00:02.0,decodes=io+mem,owns=io+mem,locks=none
[    0.102008] vgaarb: loaded
[    0.102294] vgaarb: bridge control possible 0000:00:02.0
[    0.103222] SCSI subsystem initialized
[    0.103651] libata version 3.00 loaded.
[    0.103733] ACPI: bus type USB registered
[    0.104067] usbcore: registered new interface driver usbfs
[    0.104428] usbcore: registered new interface driver hub
[    0.105034] usbcore: registered new device driver usb
[    0.105422] pps_core: LinuxPPS API ver. 1 registered
[    0.105750] pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti <[email protected]>
[    0.106031] PTP clock support registered
[    0.106501] Advanced Linux Sound Architecture Driver Initialized.
[    0.107019] PCI: Using ACPI for IRQ routing
[    0.107355] PCI: pci_cache_line_size set to 64 bytes
[    0.107494] e820: reserve RAM buffer [mem 0x0009dc00-0x0009ffff]
[    0.107498] e820: reserve RAM buffer [mem 0x1fffe000-0x1fffffff]
[    0.108102] NetLabel: Initializing
[    0.108410] NetLabel:  domain hash size = 128
[    0.108723] NetLabel:  protocols = UNLABELED CIPSOv4
[    0.109042] NetLabel:  unlabeled traffic allowed by default
[    0.109564] HPET: 3 timers in total, 0 timers will be used for per-cpu timer
[    0.110074] cfg80211: Calling CRDA to update world regulatory domain
[    0.110453] hpet0: at MMIO 0xfed00000, IRQs 2, 8, 0
[    0.111006] hpet0: 3 comparators, 64-bit 100.000000 MHz counter
[    0.117055] Switched to clocksource kvm-clock
[    0.125189] pnp: PnP ACPI init
[    0.125621] pnp 00:00: Plug and Play ACPI device, IDs PNP0b00 (active)
[    0.125682] pnp 00:01: Plug and Play ACPI device, IDs PNP0303 (active)
[    0.125735] pnp 00:02: Plug and Play ACPI device, IDs PNP0f13 (active)
[    0.125769] pnp 00:03: [dma 2]
[    0.125788] pnp 00:03: Plug and Play ACPI device, IDs PNP0700 (active)
[    0.125910] pnp 00:04: Plug and Play ACPI device, IDs PNP0501 (active)
[    0.126146] pnp: PnP ACPI: found 5 devices
[    0.131442] pci_bus 0000:00: resource 4 [io  0x0000-0xffff]
[    0.131445] pci_bus 0000:00: resource 5 [mem 0x00000000-0xffffffffff]
[    0.131483] NET: Registered protocol family 2
[    0.132100] TCP established hash table entries: 4096 (order: 3, 32768 bytes)
[    0.132521] TCP bind hash table entries: 4096 (order: 4, 65536 bytes)
[    0.132938] TCP: Hash tables configured (established 4096 bind 4096)
[    0.133368] TCP: reno registered
[    0.133720] UDP hash table entries: 256 (order: 1, 8192 bytes)
[    0.134082] UDP-Lite hash table entries: 256 (order: 1, 8192 bytes)
[    0.134549] NET: Registered protocol family 1
[    0.135049] RPC: Registered named UNIX socket transport module.
[    0.135403] RPC: Registered udp transport module.
[    0.135741] RPC: Registered tcp transport module.
[    0.136078] RPC: Registered tcp NFSv4.1 backchannel transport module.
[    0.136467] pci 0000:00:00.0: Limiting direct PCI/PCI transfers
[    0.136873] pci 0000:00:01.0: PIIX3: Enabling Passive Release
[    0.137255] pci 0000:00:01.0: Activating ISA DMA hang workarounds
[    0.137921] ACPI: PCI Interrupt Link [LNKD] enabled at IRQ 11
[    0.138494] pci 0000:00:02.0: Video device with shadowed ROM
[    0.138526] PCI: CLS 0 bytes, default 64
[    0.138767] microcode: CPU0 sig=0x206c1, pf=0x1, revision=0x1
[    0.139253] microcode: Microcode Update Driver: v2.00 <[email protected]>, Peter Oruba
[    0.139953] Scanning for low memory corruption every 60 seconds
[    0.140584] futex hash table entries: 256 (order: 2, 16384 bytes)
[    0.140974] audit: initializing netlink subsys (disabled)
[    0.141341] audit: type=2000 audit(1416869274.989:1): initialized
[    0.141994] HugeTLB registered 2 MB page size, pre-allocated 0 pages
[    0.144191] VFS: Disk quotas dquot_6.5.2
[    0.144556] Dquot-cache hash table entries: 512 (order 0, 4096 bytes)
[    0.145385] NFS: Registering the id_resolver key type
[    0.145735] Key type id_resolver registered
[    0.146065] Key type id_legacy registered
[    0.146434] 9p: Installing v9fs 9p2000 file system support
[    0.146815] msgmni has been set to 845
[    0.147387] SELinux:  Registering netfilter hooks
[    0.148606] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 251)
[    0.149235] io scheduler noop registered
[    0.149547] io scheduler deadline registered
[    0.149914] io scheduler cfq registered (default)
[    0.150339] pci_hotplug: PCI Hot Plug PCI Core version: 0.5
[    0.150732] Serial: 8250/16550 driver, 4 ports, IRQ sharing enabled
[    0.172665] 00:04: ttyS0 at I/O 0x3f8 (irq = 4, base_baud = 115200) is a 16550A
[    0.174119] Non-volatile memory driver v1.3
[    0.174447] Linux agpgart interface v0.103
[    0.174952] input: Power Button as /devices/LNXSYSTM:00/LNXPWRBN:00/input/input0
[    0.175554] ACPI: Power Button [PWRF]
[    0.176437] ACPI: PCI Interrupt Link [LNKC] enabled at IRQ 10
[    0.178508] ACPI: PCI Interrupt Link [LNKA] enabled at IRQ 10
[    0.178967] [drm] Initialized drm 1.1.0 20060810
[    0.179974] loop: module loaded
[    0.180479] virtio-pci 0000:00:04.0: irq 24 for MSI/MSI-X
[    0.180496] virtio-pci 0000:00:04.0: irq 25 for MSI/MSI-X
[    0.318792]  vda: vda1 vda2
[    0.320428] ata_piix 0000:00:01.1: version 2.13
[    0.320912] scsi host0: ata_piix
[    0.321367] scsi host1: ata_piix
[    0.321710] ata1: PATA max MWDMA2 cmd 0x1f0 ctl 0x3f6 bmdma 0xc0a0 irq 14
[    0.322092] ata2: PATA max MWDMA2 cmd 0x170 ctl 0x376 bmdma 0xc0a8 irq 15
[    0.322749] virtio-pci 0000:00:03.0: irq 26 for MSI/MSI-X
[    0.322766] virtio-pci 0000:00:03.0: irq 27 for MSI/MSI-X
[    0.322781] virtio-pci 0000:00:03.0: irq 28 for MSI/MSI-X
[    0.441106] e100: Intel(R) PRO/100 Network Driver, 3.5.24-k2-NAPI
[    0.441475] e100: Copyright(c) 1999-2006 Intel Corporation
[    0.441841] e1000: Intel(R) PRO/1000 Network Driver - version 7.3.21-k8-NAPI
[    0.442226] e1000: Copyright (c) 1999-2006 Intel Corporation.
[    0.442639] sky2: driver version 1.30
[    0.443190] ehci_hcd: USB 2.0 'Enhanced' Host Controller (EHCI) Driver
[    0.443558] ehci-pci: EHCI PCI platform driver
[    0.443920] ohci_hcd: USB 1.1 'Open' Host Controller (OHCI) Driver
[    0.444311] ohci-pci: OHCI PCI platform driver
[    0.444649] uhci_hcd: USB Universal Host Controller Interface driver
[    0.445262] uhci_hcd 0000:00:01.2: UHCI Host Controller
[    0.445657] uhci_hcd 0000:00:01.2: new USB bus registered, assigned bus number 1
[    0.446419] uhci_hcd 0000:00:01.2: irq 11, io base 0x0000c040
[    0.446851] usb usb1: New USB device found, idVendor=1d6b, idProduct=0001
[    0.447247] usb usb1: New USB device strings: Mfr=3, Product=2, SerialNumber=1
[    0.447792] usb usb1: Product: UHCI Host Controller
[    0.448127] usb usb1: Manufacturer: Linux 3.17.3-gentoo uhci_hcd
[    0.448476] usb usb1: SerialNumber: 0000:00:01.2
[    0.448926] hub 1-0:1.0: USB hub found
[    0.449263] hub 1-0:1.0: 2 ports detected
[    0.449689] usbcore: registered new interface driver usblp
[    0.450122] usbcore: registered new interface driver usb-storage
[    0.450516] i8042: PNP: PS/2 Controller [PNP0303:KBD,PNP0f13:MOU] at 0x60,0x64 irq 1,12
[    0.451720] serio: i8042 KBD port at 0x60,0x64 irq 1
[    0.452070] serio: i8042 AUX port at 0x60,0x64 irq 12
[    0.452483] mousedev: PS/2 mouse device common for all mice
[    0.453170] input: AT Translated Set 2 keyboard as /devices/platform/i8042/serio0/input/input1
[    0.455416] rtc_cmos 00:00: RTC can wake from S4
[    0.455993] rtc_cmos 00:00: rtc core: registered rtc_cmos as rtc0
[    0.456467] rtc_cmos 00:00: alarms up to one day, 114 bytes nvram, hpet irqs
[    0.456955] device-mapper: ioctl: 4.27.0-ioctl (2013-10-30) initialised: [email protected]
[    0.457632] hidraw: raw HID events driver (C) Jiri Kosina
[    0.458241] usbcore: registered new interface driver usbhid
[    0.458590] usbhid: USB HID core driver
[    0.459182] Netfilter messages via NETLINK v0.30.
[    0.459528] nf_conntrack version 0.5.0 (3380 buckets, 13520 max)
[    0.459984] ctnetlink v0.93: registering with nfnetlink.
[    0.460442] ip_tables: (C) 2000-2006 Netfilter Core Team
[    0.460807] TCP: cubic registered
[    0.461176] Initializing XFRM netlink socket
[    0.461608] NET: Registered protocol family 10
[    0.462268] ip6_tables: (C) 2000-2006 Netfilter Core Team
[    0.462660] sit: IPv6 over IPv4 tunneling driver
[    0.463146] NET: Registered protocol family 17
[    0.463533] 9pnet: Installing 9P2000 support
[    0.463904] Key type dns_resolver registered
[    0.464485] registered taskstats version 1
[    0.465119]   Magic number: 6:406:806
[    0.465470] console [netcon0] enabled
[    0.465777] netconsole: network logging started
[    0.466175] PM: Hibernation image not present or could not be loaded.
[    0.466178] ALSA device list:
[    0.466469]   No soundcards found.
[    0.511359] ata2.01: NODEV after polling detection
[    0.511632] ata2.00: ATAPI: QEMU DVD-ROM, 1.1.2, max UDMA/100
[    0.512499] ata2.00: configured for MWDMA2
[    0.513328] scsi 1:0:0:0: CD-ROM            QEMU     QEMU DVD-ROM     1.1. PQ: 0 ANSI: 5
[    0.524546] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray
[    0.524926] cdrom: Uniform CD-ROM driver Revision: 3.20
[    0.525456] sr 1:0:0:0: Attached scsi CD-ROM sr0
[    0.525739] sr 1:0:0:0: Attached scsi generic sg0 type 5
[    0.751163] usb 1-1: new full-speed USB device number 2 using uhci_hcd
[    1.009894] usb 1-1: New USB device found, idVendor=0627, idProduct=0001
[    1.010342] usb 1-1: New USB device strings: Mfr=1, Product=3, SerialNumber=5
[    1.010855] usb 1-1: Product: QEMU USB Tablet
[    1.011200] usb 1-1: Manufacturer: QEMU 1.1.2
[    1.011517] usb 1-1: SerialNumber: 42
[    1.028916] input: QEMU 1.1.2 QEMU USB Tablet as /devices/pci0000:00/0000:00:01.2/usb1/1-1/1-1:1.0/0003:0627:0001.0001/input/input4
[    1.030642] hid-generic 0003:0627:0001.0001: input,hidraw0: USB HID v0.01 Pointer [QEMU 1.1.2 QEMU USB Tablet] on usb-0000:00:01.2-1/input0
[    1.055863] input: ImExPS/2 Generic Explorer Mouse as /devices/platform/i8042/serio1/input/input3
[    1.056697] md: Skipping autodetection of RAID arrays. (raid=autodetect will force)
[    1.140105] tsc: Refined TSC clocksource calibration: 2593.617 MHz
[    1.432843] kjournald starting.  Commit interval 5 seconds
[    1.433255] EXT3-fs (vda1): mounted filesystem with ordered data mode
[    1.433638] VFS: Mounted root (ext3 filesystem) readonly on device 253:1.
[    2.042735] devtmpfs: mounted
[    2.044053] Freeing unused kernel memory: 1108K (ffffffff81ed9000 - ffffffff81fee000)
[    2.044626] Write protecting the kernel read-only data: 14336k
[    2.050512] Freeing unused kernel memory: 1964K (ffff880001815000 - ffff880001a00000)
[    2.054142] Freeing unused kernel memory: 1412K (ffff880001c9f000 - ffff880001e00000)
[   13.657617] init-early.sh (884) used greatest stack depth: 11992 bytes left
[   40.359139] systemd-udevd[1057]: starting version 216
[   42.700710] random: systemd-udevd urandom read with 39 bits of entropy available
[   42.773688] udevadm (1065) used greatest stack depth: 11768 bytes left
[   51.386937] EXT3-fs (vda1): using internal journal
[   51.730108] Adding 1571836k swap on /dev/vda2.  Priority:-1 extents:1 across:1571836k 
[   88.213351] device eth0 entered promiscuous mode
[  100.706265] random: nonblocking pool is initialized
[  252.755504] tun: Universal TUN/TAP device driver, 1.6
[  252.755508] tun: (C) 1999-2004 Max Krasnyansky <[email protected]>
[  254.300593] batman_adv: B.A.T.M.A.N. advanced 2014.4.0 (compatibility version 15) loaded
[  254.545508] batman_adv: bat0: Adding interface: fastd0
[  254.545516] batman_adv: bat0: The MTU of interface fastd0 is too small (1426) to handle the transport of batman-adv packets. Packets going over this interface will be fragmented on layer2 which could impact the performance. Setting the MTU to 1560 would solve the problem.
[  254.545521] batman_adv: bat0: Interface activated: fastd0
[  254.546593] batman_adv: bat0: orig_interval: Changing from: 1000 to: 5000
[  254.548281] batman_adv: bat0: bridge_loop_avoidance: Changing from: disabled to: enabled
[  254.548999] batman_adv: bat0: Changing gw mode from: off to: client
[  311.447606] ipip: IPv4 over IPv4 tunneling driver
[  672.444097] batman_adv: bat0: Changing gw mode from: client to: server
[  672.444119] batman_adv: bat0: Changing gateway bandwidth from: '10.0/2.0 MBit' to: '100.0/100.0 MBit'
[  698.298490] nf_conntrack: automatic helper assignment is deprecated and it will be removed soon. Use the iptables CT target to attach helpers instead.
[ 1184.606170] mkmap.sh (6307) used greatest stack depth: 11736 bytes left
[ 2395.269797] skbuff: skb_over_panic: text:ffffffffa001be32 len:1445 put:1380 head:ffff880017738c00 data:ffff880017738c62 tail:0x607 end:0x2c0 dev:fastd0
[ 2395.270151] ------------[ cut here ]------------
[ 2395.270257] kernel BUG at net/core/skbuff.c:100!
[ 2395.270352] invalid opcode: 0000 [#1] SMP 
[ 2395.270453] Modules linked in: xt_nat iptable_nat nf_nat_ipv4 nf_nat ipip batman_adv libcrc32c tun crc32c_intel
[ 2395.270738] CPU: 0 PID: 2036 Comm: fastd Not tainted 3.17.3-gentoo #1
[ 2395.270861] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2007
[ 2395.270987] task: ffff880019cef290 ti: ffff88001768c000 task.ti: ffff88001768c000
[ 2395.271111] RIP: 0010:[<ffffffff81806a38>]  [<ffffffff81806a38>] skb_panic+0x5e/0x60
[ 2395.271111] RSP: 0018:ffff88001fc03cf8  EFLAGS: 00010296
[ 2395.271111] RAX: 000000000000008b RBX: ffff88001a2bbd20 RCX: 0000000000000092
[ 2395.271111] RDX: 00000000000000c2 RSI: 0000000000000246 RDI: 0000000000000246
[ 2395.271111] RBP: ffff88001fc03d18 R08: 0000000000000000 R09: 0000000000000000
[ 2395.271111] R10: ffffffff81a6b240 R11: 0000000000000000 R12: 0000000000000564
[ 2395.271111] R13: ffff88001fc03da0 R14: ffff88000f877000 R15: ffff880002c56862
[ 2395.271111] FS:  00007f0c8256c700(0000) GS:ffff88001fc00000(0000) knlGS:0000000000000000
[ 2395.271111] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 2395.271111] CR2: 00007fff6da33ed0 CR3: 000000001766c000 CR4: 00000000000006f0
[ 2395.271111] Stack:
[ 2395.271111]  ffff880017738c62 0000000000000607 00000000000002c0 ffff880017638000
[ 2395.271111]  ffff88001fc03d28 ffffffff8162c921 ffff88001fc03d88 ffffffffa001be32
[ 2395.271111]  ffff88001a3a89c8 ffff880002c5684e ffff88001fc03d68 ffff88001a2bbd20
[ 2395.271111] Call Trace:
[ 2395.271111]  <IRQ> 
[ 2395.271111] 
[ 2395.271111]  [<ffffffff8162c921>] skb_put+0x41/0x50
[ 2395.271111]  [<ffffffffa001be32>] batadv_frag_skb_buffer+0x272/0x470 [batman_adv]
[ 2395.271111]  [<ffffffffa0026283>] batadv_recv_frag_packet+0x183/0x200 [batman_adv]
[ 2395.271111]  [<ffffffffa001ff15>] batadv_batman_skb_recv+0xd5/0x110 [batman_adv]
[ 2395.271111]  [<ffffffff812b0f76>] ? __blk_mq_complete_request+0x86/0x120
[ 2395.271111]  [<ffffffff8163c752>] __netif_receive_skb_core+0x222/0x740
[ 2395.271111]  [<ffffffff8146b13a>] ? virtblk_done+0xca/0xe0
[ 2395.271111]  [<ffffffff8163cc91>] __netif_receive_skb+0x21/0x70
[ 2395.271111]  [<ffffffff8163cd7e>] process_backlog+0x9e/0x170
[ 2395.271111]  [<ffffffff8163d531>] net_rx_action+0x141/0x240
[ 2395.271111]  [<ffffffff81052e28>] __do_softirq+0xe8/0x280
[ 2395.271111]  [<ffffffff8180e99c>] do_softirq_own_stack+0x1c/0x30
[ 2395.271111]  <EOI> 
[ 2395.271111] 
[ 2395.271111]  [<ffffffff81053065>] do_softirq+0x55/0x60
[ 2395.271111]  [<ffffffff8163bca4>] netif_rx_ni+0x34/0x70
[ 2395.271111]  [<ffffffffa00098e3>] tun_get_user+0x413/0x840 [tun]
[ 2395.271111]  [<ffffffffa0009e0b>] tun_chr_aio_write+0x7b/0xa0 [tun]
[ 2395.271111]  [<ffffffff8115fdb5>] do_sync_write+0x55/0x90
[ 2395.271111]  [<ffffffff81160ada>] vfs_write+0xba/0x1f0
[ 2395.271111]  [<ffffffff81160fea>] SyS_write+0x4a/0xa0
[ 2395.271111]  [<ffffffff8180ce69>] system_call_fastpath+0x16/0x1b
[ 2395.271111] Code: 00 00 48 89 44 24 10 8b 87 c0 00 00 00 48 89 44 24 08 48 8b 87 d0 00 00 00 48 c7 c7 c0 cc c3 81 48 89 04 24 31 c0 e8 ca 97 ff ff <0f> 0b 55 48 89 f8 48 8b 57 30 48 89 e5 48 8b 0f 5d 80 e5 80 48 
[ 2395.271111] RIP  [<ffffffff81806a38>] skb_panic+0x5e/0x60
[ 2395.271111]  RSP <ffff88001fc03cf8>

Hi Martin, hi all.

Should I keep on reporting? Meanwhile I'm running out of space for the vmcore dumps. Is it me, is it the kernel or is it Batman-adv? What should I do? Should I bother the kernel hackers?

This bug happened 44 minutes after changing gw client to gw server.

Best regards

Philipp

________________________
Freifunk Rheinland e. V.
– Funkzelle Wuppertal –

SYSTEM MAP: /boot/System.map
DEBUG KERNEL: /usr/src/linux-3.17.3-gentoo/vmlinux (3.17.3-gentoo)

    DUMPFILE: vmcore_20141125105708
        CPUS: 1
        DATE: Thu Jan  1 01:00:00 1970
      UPTIME: 10:23:00
LOAD AVERAGE: 0.25, 0.19, 0.35
       TASKS: 149
    NODENAME: wolke
     RELEASE: 3.17.3-gentoo
     VERSION: #1 SMP Wed Nov 19 15:42:49 CET 2014
     MACHINE: x86_64  (2593 Mhz)
      MEMORY: 511.6 MB
       PANIC: "kernel BUG at net/core/skbuff.c:100!"
         PID: 2058
     COMMAND: "fastd"
        TASK: ffff88001a3b91a0  [THREAD_INFO: ffff880006c04000]
         CPU: 0
       STATE: TASK_RUNNING (PANIC)

crash> bt
PID: 2058 TASK: ffff88001a3b91a0 CPU: 0 COMMAND: "fastd"
#0 [ffff88001fc03980] machine_kexec at ffffffff8103aace #1 [ffff88001fc039e0] crash_kexec at ffffffff810bfa43 #2 [ffff88001fc03ab0] oops_end at ffffffff810060f8 #3 [ffff88001fc03ae0] die at ffffffff81006593 #4 [ffff88001fc03b10] do_trap at ffffffff81002ef2 #5 [ffff88001fc03b70] do_error_trap at ffffffff8100305d #6 [ffff88001fc03c30] do_invalid_op at ffffffff81003a7b #7 [ffff88001fc03c40] invalid_op at ffffffff8180e79e

    [exception RIP: skb_panic+94]
    RIP: ffffffff81806a38  RSP: ffff88001fc03cf8  RFLAGS: 00010296
    RAX: 000000000000008b  RBX: ffff88001a3a7100  RCX: 0000000000000092
    RDX: 0000000000000033  RSI: 0000000000000246  RDI: 0000000000000246
    RBP: ffff88001fc03d18   R8: 0000000000000000   R9: 0000000000000000
    R10: ffffffff81a6b240  R11: 0000000000000000  R12: 0000000000000564
    R13: ffff88001fc03da0  R14: ffff880019e92b00  R15: ffff88000d8a7862
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018

#8 [ffff88001fc03d20] skb_put at ffffffff8162c921
#9 [ffff88001fc03d30] batadv_frag_skb_buffer at ffffffffa001be32 [batman_adv]
#10 [ffff88001fc03d90] batadv_recv_frag_packet at ffffffffa0026283 [batman_adv] #11 [ffff88001fc03dd0] batadv_batman_skb_recv at ffffffffa001ff15 [batman_adv] #12 [ffff88001fc03e10] __netif_receive_skb_core at ffffffff8163c752 #13 [ffff88001fc03e80] __netif_receive_skb at ffffffff8163cc91
#14 [ffff88001fc03ea0] process_backlog at ffffffff8163cd7e
#15 [ffff88001fc03ef0] net_rx_action at ffffffff8163d531
#16 [ffff88001fc03f50] __do_softirq at ffffffff81052e28
#17 [ffff88001fc03fb0] do_softirq_own_stack at ffffffff8180e99c --- <IRQ stack> ---
#18 [ffff880006c07d10] do_softirq_own_stack at ffffffff8180e99c

    [exception RIP: tun_get_user+1043]
    RIP: ffffffffa00098e3  RSP: 0000000000000001  RFLAGS: 7fff00000586
    RAX: ffffffff8163bca4  RBX: ffff880006c07d58  RCX: ffff880006cd4780
    RDX: 0000000000000000  RSI: ffff880006cd4780  RDI: 0000000000000586
    RBP: ffffffff8163b9d4   R8: ffff880006c07d88   R9: ffff880006cd4780
    R10: ffff880006cd4780  R11: ffffffff81053065  R12: ffff880006c07d58
    R13: 0000000000000586  R14: ffff880019e92300  R15: 0000000000000000
    ORIG_RAX: ffff880006c07e38  CS: 7fff4304a270  SS: 0000

bt: WARNING: possibly bogus exception frame
#19 [ffff880006c07e40] tun_chr_aio_write at ffffffffa0009e0b [tun] #20 [ffff880006c07e70] do_sync_write at ffffffff8115fdb5 #21 [ffff880006c07f00] vfs_write at ffffffff81160ada #22 [ffff880006c07f40] sys_write at ffffffff81160fea #23 [ffff880006c07f80] system_call_fastpath at ffffffff8180ce69

    RIP: 00007f8cb870b37d  RSP: 00007fff4304a3b8  RFLAGS: 00000202
    RAX: 0000000000000001  RBX: ffffffff8180ce69  RCX: 00000000000005b0
    RDX: 0000000000000586  RSI: 000000000127ff70  RDI: 0000000000000009
    RBP: 0000000000000586   R8: 00007f8cb86f4400   R9: 00007fff430499b8
    R10: 00007fff4304a0ff  R11: 0000000000000293  R12: 0000000001274fa8
    R13: 0000000000000001  R14: 000000000127ff60  R15: 00000000012636e0
    ORIG_RAX: 0000000000000001  CS: 0033  SS: 002b
crash> log
[    0.000000] Initializing cgroup subsys cpuset
[    0.000000] Initializing cgroup subsys cpu
[    0.000000] Initializing cgroup subsys cpuacct
[    0.000000] Linux version 3.17.3-gentoo (root@wolke) (gcc version 4.7.3 (Gentoo 4.7.3-r1 p1.4, pie-0.5.5) ) #1 SMP Wed Nov 19 15:42:49 CET 2014
[    0.000000] Command line: root=/dev/vda1 raid=noautodetect crashkernel=64M
[    0.000000] e820: BIOS-provided physical RAM map:
[    0.000000] BIOS-e820: [mem 0x0000000000000000-0x000000000009dbff] usable
[    0.000000] BIOS-e820: [mem 0x000000000009dc00-0x000000000009ffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000000f0000-0x00000000000fffff] reserved
[    0.000000] BIOS-e820: [mem 0x0000000000100000-0x000000001fffdfff] usable
[    0.000000] BIOS-e820: [mem 0x000000001fffe000-0x000000001fffffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000feffc000-0x00000000feffffff] reserved
[    0.000000] BIOS-e820: [mem 0x00000000fffc0000-0x00000000ffffffff] reserved
[    0.000000] NX (Execute Disable) protection: active
[    0.000000] SMBIOS 2.4 present.
[    0.000000] DMI: Bochs Bochs, BIOS Bochs 01/01/2007
[    0.000000] Hypervisor detected: KVM
[    0.000000] e820: update [mem 0x00000000-0x00000fff] usable ==> reserved
[    0.000000] e820: remove [mem 0x000a0000-0x000fffff] usable
[    0.000000] e820: last_pfn = 0x1fffe max_arch_pfn = 0x400000000
[    0.000000] MTRR default type: write-back
[    0.000000] MTRR fixed ranges enabled:
[    0.000000]   00000-9FFFF write-back
[    0.000000]   A0000-BFFFF uncachable
[    0.000000]   C0000-FFFFF write-protect
[    0.000000] MTRR variable ranges enabled:
[    0.000000]   0 base 00E0000000 mask FFE0000000 uncachable
[    0.000000]   1 disabled
[    0.000000]   2 disabled
[    0.000000]   3 disabled
[    0.000000]   4 disabled
[    0.000000]   5 disabled
[    0.000000]   6 disabled
[    0.000000]   7 disabled
[    0.000000] x86 PAT enabled: cpu 0, old 0x70406, new 0x7010600070106
[    0.000000] found SMP MP-table at [mem 0x000fdaf0-0x000fdaff] mapped at [ffff8800000fdaf0]
[    0.000000] Scanning 1 areas for low memory corruption
[    0.000000] Base memory trampoline at [ffff880000097000] 97000 size 24576
[    0.000000] init_memory_mapping: [mem 0x00000000-0x000fffff]
[    0.000000]  [mem 0x00000000-0x000fffff] page 4k
[    0.000000] BRK [0x020d7000, 0x020d7fff] PGTABLE
[    0.000000] BRK [0x020d8000, 0x020d8fff] PGTABLE
[    0.000000] BRK [0x020d9000, 0x020d9fff] PGTABLE
[    0.000000] init_memory_mapping: [mem 0x1fc00000-0x1fdfffff]
[    0.000000]  [mem 0x1fc00000-0x1fdfffff] page 2M
[    0.000000] init_memory_mapping: [mem 0x1c000000-0x1fbfffff]
[    0.000000]  [mem 0x1c000000-0x1fbfffff] page 2M
[    0.000000] init_memory_mapping: [mem 0x00100000-0x1bffffff]
[    0.000000]  [mem 0x00100000-0x001fffff] page 4k
[    0.000000]  [mem 0x00200000-0x1bffffff] page 2M
[    0.000000] init_memory_mapping: [mem 0x1fe00000-0x1fffdfff]
[    0.000000]  [mem 0x1fe00000-0x1fffdfff] page 4k
[    0.000000] BRK [0x020da000, 0x020dafff] PGTABLE
[    0.000000] ACPI: Early table checksum verification disabled
[    0.000000] ACPI: RSDP 0x00000000000FD990 000014 (v00 BOCHS )
[    0.000000] ACPI: RSDT 0x000000001FFFE5B0 000038 (v01 BOCHS  BXPCRSDT 00000001 BXPC 00000001)
[    0.000000] ACPI: FACP 0x000000001FFFFF80 000074 (v01 BOCHS  BXPCFACP 00000001 BXPC 00000001)
[    0.000000] ACPI: DSDT 0x000000001FFFE5F0 001121 (v01 BXPC   BXDSDT   00000001 INTL 20100528)
[    0.000000] ACPI: FACS 0x000000001FFFFF40 000040
[    0.000000] ACPI: SSDT 0x000000001FFFFEA0 00009E (v01 BOCHS  BXPCSSDT 00000001 BXPC 00000001)
[    0.000000] ACPI: APIC 0x000000001FFFFDB0 000078 (v01 BOCHS  BXPCAPIC 00000001 BXPC 00000001)
[    0.000000] ACPI: HPET 0x000000001FFFFD70 000038 (v01 BOCHS  BXPCHPET 00000001 BXPC 00000001)
[    0.000000] ACPI: SSDT 0x000000001FFFF720 000644 (v01 BXPC   BXSSDTPC 00000001 INTL 20100528)
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] No NUMA configuration found
[    0.000000] Faking a node at [mem 0x0000000000000000-0x000000001fffdfff]
[    0.000000] Initmem setup node 0 [mem 0x00000000-0x1fffdfff]
[    0.000000]   NODE_DATA [mem 0x1fffa000-0x1fffdfff]
[    0.000000] Reserving 64MB of memory at 432MB for crashkernel (System RAM: 511MB)
[    0.000000] kvm-clock: Using msrs 4b564d01 and 4b564d00
[    0.000000] kvm-clock: cpu 0, msr 0:1fff9001, primary cpu clock
[    0.000000]  [ffffea0000000000-ffffea00007fffff] PMD -> [ffff88001a800000-ffff88001affffff] on node 0
[    0.000000] Zone ranges:
[    0.000000]   DMA      [mem 0x00001000-0x00ffffff]
[    0.000000]   DMA32    [mem 0x01000000-0xffffffff]
[    0.000000]   Normal   empty
[    0.000000] Movable zone start for each node
[    0.000000] Early memory node ranges
[    0.000000]   node   0: [mem 0x00001000-0x0009cfff]
[    0.000000]   node   0: [mem 0x00100000-0x1fffdfff]
[    0.000000] On node 0 totalpages: 130970
[    0.000000]   DMA zone: 64 pages used for memmap
[    0.000000]   DMA zone: 21 pages reserved
[    0.000000]   DMA zone: 3996 pages, LIFO batch:0
[    0.000000]   DMA32 zone: 1984 pages used for memmap
[    0.000000]   DMA32 zone: 126974 pages, LIFO batch:31
[    0.000000] ACPI: PM-Timer IO Port: 0xb008
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] ACPI: LAPIC (acpi_id[0x00] lapic_id[0x00] enabled)
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xff] dfl dfl lint[0x1])
[    0.000000] ACPI: IOAPIC (id[0x01] address[0xfec00000] gsi_base[0])
[    0.000000] IOAPIC[0]: apic_id 1, version 17, address 0xfec00000, GSI 0-23
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 5 global_irq 5 high level)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 10 global_irq 10 high level)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 11 global_irq 11 high level)
[    0.000000] ACPI: IRQ0 used by override.
[    0.000000] ACPI: IRQ5 used by override.
[    0.000000] ACPI: IRQ9 used by override.
[    0.000000] ACPI: IRQ10 used by override.
[    0.000000] ACPI: IRQ11 used by override.
[    0.000000] Using ACPI (MADT) for SMP configuration information
[    0.000000] ACPI: HPET id: 0x8086a201 base: 0xfed00000
[    0.000000] smpboot: Allowing 1 CPUs, 0 hotplug CPUs
[    0.000000] PM: Registered nosave memory: [mem 0x0009d000-0x0009dfff]
[    0.000000] PM: Registered nosave memory: [mem 0x0009e000-0x0009ffff]
[    0.000000] PM: Registered nosave memory: [mem 0x000a0000-0x000effff]
[    0.000000] PM: Registered nosave memory: [mem 0x000f0000-0x000fffff]
[    0.000000] e820: [mem 0x20000000-0xfeffbfff] available for PCI devices
[    0.000000] Booting paravirtualized kernel on KVM
[    0.000000] setup_percpu: NR_CPUS:64 nr_cpumask_bits:64 nr_cpu_ids:1 nr_node_ids:1
[    0.000000] PERCPU: Embedded 27 pages/cpu @ffff88001fc00000 s79744 r8192 d22656 u2097152
[    0.000000] pcpu-alloc: s79744 r8192 d22656 u2097152 alloc=1*2097152
[    0.000000] pcpu-alloc: [0] 0 
[    0.000000] KVM setup async PF for cpu 0
[    0.000000] kvm-stealtime: cpu 0, msr 1fc0cf80
[    0.000000] Built 1 zonelists in Node order, mobility grouping on.  Total pages: 128901
[    0.000000] Policy zone: DMA32
[    0.000000] Kernel command line: root=/dev/vda1 raid=noautodetect crashkernel=64M
[    0.000000] PID hash table entries: 2048 (order: 2, 16384 bytes)
[    0.000000] Calgary: detecting Calgary via BIOS EBDA area
[    0.000000] Calgary: Unable to locate Rio Grande table in EBDA - bailing!
[    0.000000] Memory: 432620K/523880K available (8263K kernel code, 863K rwdata, 2684K rodata, 1108K init, 896K bss, 91260K reserved)
[    0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=1, Nodes=1
[    0.000000] Hierarchical RCU implementation.
[    0.000000]  RCU restricting CPUs from NR_CPUS=64 to nr_cpu_ids=1.
[    0.000000] RCU: Adjusting geometry for rcu_fanout_leaf=16, nr_cpu_ids=1
[    0.000000] NR_IRQS:4352 nr_irqs:256 0
[    0.000000] Console: colour VGA+ 80x25
[    0.000000] console [tty0] enabled
[    0.000000] hpet clockevent registered
[    0.000000] tsc: Detected 2593.748 MHz processor
[    0.002000] Calibrating delay loop (skipped) preset value.. 5187.49 BogoMIPS (lpj=2593748)
[    0.002005] pid_max: default: 32768 minimum: 301
[    0.002398] ACPI: Core revision 20140724
[    0.003751] ACPI: All ACPI Tables successfully acquired
[    0.004321] Security Framework initialized
[    0.004680] SELinux:  Initializing.
[    0.005011] SELinux:  Starting in permissive mode
[    0.005045] Dentry cache hash table entries: 65536 (order: 7, 524288 bytes)
[    0.005728] Inode-cache hash table entries: 32768 (order: 6, 262144 bytes)
[    0.006136] Mount-cache hash table entries: 1024 (order: 1, 8192 bytes)
[    0.006555] Mountpoint-cache hash table entries: 1024 (order: 1, 8192 bytes)
[    0.007242] Initializing cgroup subsys freezer
[    0.008004] mce: CPU supports 10 MCE banks
[    0.008406] Last level iTLB entries: 4KB 0, 2MB 0, 4MB 0
               Last level dTLB entries: 4KB 0, 2MB 0, 4MB 0, 1GB 0
[    0.027152] Freeing SMP alternatives memory: 28K (ffffffff81fee000 - ffffffff81ff5000)
[    0.031911] ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1
[    0.032002] smpboot: CPU0: Intel Westmere E56xx/L56xx/X56xx (Nehalem-C) (fam: 06, model: 2c, stepping: 01)
[    0.034000] APIC calibration not consistent with PM-Timer: 101ms instead of 100ms
[    0.034000] APIC delta adjusted to PM-Timer: 6250163 (6375026)
[    0.034000] Performance Events: unsupported p6 CPU model 44 no PMU driver, software events only.
[    0.034320] x86: Booted up 1 node, 1 CPUs
[    0.034635] smpboot: Total of 1 processors activated (5187.49 BogoMIPS)
[    0.035384] devtmpfs: initialized
[    0.036195] RTC time: 23:31:12, date: 11/24/14
[    0.036665] NET: Registered protocol family 16
[    0.037225] cpuidle: using governor ladder
[    0.037563] cpuidle: using governor menu
[    0.038040] ACPI: bus type PCI registered
[    0.038496] PCI: Using configuration type 1 for base access
[    0.041486] kworker/u2:0 (15) used greatest stack depth: 14664 bytes left
[    0.042191] kworker/u2:0 (20) used greatest stack depth: 13912 bytes left
[    0.042610] ACPI: Added _OSI(Module Device)
[    0.042928] ACPI: Added _OSI(Processor Device)
[    0.043005] ACPI: Added _OSI(3.0 _SCP Extensions)
[    0.043329] ACPI: Added _OSI(Processor Aggregator Device)
[    0.045318] ACPI: Interpreter enabled
[    0.045643] ACPI Exception: AE_NOT_FOUND, While evaluating Sleep State [\_S1_] (20140724/hwxface-580)
[    0.046284] ACPI Exception: AE_NOT_FOUND, While evaluating Sleep State [\_S2_] (20140724/hwxface-580)
[    0.047067] ACPI: (supports S0 S3 S4 S5)
[    0.047431] ACPI: Using IOAPIC for interrupt routing
[    0.047781] PCI: Ignoring host bridge windows from ACPI; if necessary, use "pci=use_crs" and report a bug
[    0.050765] ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-ff])
[    0.051016] acpi PNP0A03:00: _OSC: OS supports [ASPM ClockPM Segments MSI]
[    0.051389] acpi PNP0A03:00: _OSC failed (AE_NOT_FOUND); disabling ASPM
[    0.051762] acpi PNP0A03:00: host bridge window [io  0x0000-0x0cf7] (ignored)
[    0.051763] acpi PNP0A03:00: host bridge window [io  0x0d00-0xffff] (ignored)
[    0.051765] acpi PNP0A03:00: host bridge window [mem 0x000a0000-0x000bffff] (ignored)
[    0.051766] acpi PNP0A03:00: host bridge window [mem 0xe0000000-0xfebfffff] (ignored)
[    0.051768] PCI: root bus 00: using default resources
[    0.051770] acpi PNP0A03:00: fail to add MMCONFIG information, can't access extended PCI configuration space under this bridge.
[    0.052052] PCI host bridge to bus 0000:00
[    0.052371] pci_bus 0000:00: root bus resource [bus 00-ff]
[    0.053010] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
[    0.053369] pci_bus 0000:00: root bus resource [mem 0x00000000-0xffffffffff]
[    0.054069] pci 0000:00:00.0: [8086:1237] type 00 class 0x060000
[    0.054390] pci 0000:00:01.0: [8086:7000] type 00 class 0x060100
[    0.054861] pci 0000:00:01.1: [8086:7010] type 00 class 0x010180
[    0.056465] pci 0000:00:01.1: reg 0x20: [io  0xc0a0-0xc0af]
[    0.057026] pci 0000:00:01.1: legacy IDE quirk: reg 0x10: [io  0x01f0-0x01f7]
[    0.057442] pci 0000:00:01.1: legacy IDE quirk: reg 0x14: [io  0x03f6]
[    0.057799] pci 0000:00:01.1: legacy IDE quirk: reg 0x18: [io  0x0170-0x0177]
[    0.058003] pci 0000:00:01.1: legacy IDE quirk: reg 0x1c: [io  0x0376]
[    0.058491] pci 0000:00:01.2: [8086:7020] type 00 class 0x0c0300
[    0.060298] pci 0000:00:01.2: reg 0x20: [io  0xc040-0xc05f]
[    0.061094] pci 0000:00:01.3: [8086:7113] type 00 class 0x068000
[    0.061410] pci 0000:00:01.3: quirk: [io  0xb000-0xb03f] claimed by PIIX4 ACPI
[    0.061983] pci 0000:00:01.3: quirk: [io  0xb100-0xb10f] claimed by PIIX4 SMB
[    0.062241] pci 0000:00:02.0: [1013:00b8] type 00 class 0x030000
[    0.064036] pci 0000:00:02.0: reg 0x10: [mem 0xfc000000-0xfdffffff pref]
[    0.066031] pci 0000:00:02.0: reg 0x14: [mem 0xfebf0000-0xfebf0fff]
[    0.076064] pci 0000:00:02.0: reg 0x30: [mem 0xfebd0000-0xfebdffff pref]
[    0.077425] pci 0000:00:03.0: [1af4:1000] type 00 class 0x020000
[    0.078007] pci 0000:00:03.0: reg 0x10: [io  0xc060-0xc07f]
[    0.078583] pci 0000:00:03.0: reg 0x14: [mem 0xfebf1000-0xfebf1fff]
[    0.081331] pci 0000:00:03.0: reg 0x30: [mem 0xfebe0000-0xfebeffff pref]
[    0.082213] pci 0000:00:04.0: [1af4:1001] type 00 class 0x010000
[    0.084006] pci 0000:00:04.0: reg 0x10: [io  0xc000-0xc03f]
[    0.086004] pci 0000:00:04.0: reg 0x14: [mem 0xfebf2000-0xfebf2fff]
[    0.090583] pci 0000:00:05.0: [1af4:1002] type 00 class 0x00ff00
[    0.091029] pci 0000:00:05.0: reg 0x10: [io  0xc080-0xc09f]
[    0.093694] ACPI: PCI Interrupt Link [LNKA] (IRQs 5 *10 11)
[    0.094487] ACPI: PCI Interrupt Link [LNKB] (IRQs 5 *10 11)
[    0.095171] ACPI: PCI Interrupt Link [LNKC] (IRQs 5 10 *11)
[    0.096438] ACPI: PCI Interrupt Link [LNKD] (IRQs 5 10 *11)
[    0.097378] ACPI: PCI Interrupt Link [LNKS] (IRQs 9) *0, disabled.
[    0.099279] ACPI: Enabled 16 GPEs in block 00 to 0F
[    0.100066] vgaarb: setting as boot device: PCI:0000:00:02.0
[    0.100409] vgaarb: device added: PCI:0000:00:02.0,decodes=io+mem,owns=io+mem,locks=none
[    0.101006] vgaarb: loaded
[    0.101289] vgaarb: bridge control possible 0000:00:02.0
[    0.102258] SCSI subsystem initialized
[    0.102784] libata version 3.00 loaded.
[    0.102881] ACPI: bus type USB registered
[    0.103089] usbcore: registered new interface driver usbfs
[    0.103447] usbcore: registered new interface driver hub
[    0.104055] usbcore: registered new device driver usb
[    0.104442] pps_core: LinuxPPS API ver. 1 registered
[    0.104778] pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti <[email protected]>
[    0.105029] PTP clock support registered
[    0.105492] Advanced Linux Sound Architecture Driver Initialized.
[    0.106018] PCI: Using ACPI for IRQ routing
[    0.106335] PCI: pci_cache_line_size set to 64 bytes
[    0.106477] e820: reserve RAM buffer [mem 0x0009dc00-0x0009ffff]
[    0.106481] e820: reserve RAM buffer [mem 0x1fffe000-0x1fffffff]
[    0.107116] NetLabel: Initializing
[    0.107421] NetLabel:  domain hash size = 128
[    0.107734] NetLabel:  protocols = UNLABELED CIPSOv4
[    0.108034] NetLabel:  unlabeled traffic allowed by default
[    0.108530] cfg80211: Calling CRDA to update world regulatory domain
[    0.110200] HPET: 3 timers in total, 0 timers will be used for per-cpu timer
[    0.110608] hpet0: at MMIO 0xfed00000, IRQs 2, 8, 0
[    0.111303] hpet0: 3 comparators, 64-bit 100.000000 MHz counter
[    0.116055] Switched to clocksource kvm-clock
[    0.124288] pnp: PnP ACPI init
[    0.124734] pnp 00:00: Plug and Play ACPI device, IDs PNP0b00 (active)
[    0.124796] pnp 00:01: Plug and Play ACPI device, IDs PNP0303 (active)
[    0.124848] pnp 00:02: Plug and Play ACPI device, IDs PNP0f13 (active)
[    0.124882] pnp 00:03: [dma 2]
[    0.124901] pnp 00:03: Plug and Play ACPI device, IDs PNP0700 (active)
[    0.124998] pnp 00:04: Plug and Play ACPI device, IDs PNP0501 (active)
[    0.125211] pnp: PnP ACPI: found 5 devices
[    0.133974] pci_bus 0000:00: resource 4 [io  0x0000-0xffff]
[    0.133977] pci_bus 0000:00: resource 5 [mem 0x00000000-0xffffffffff]
[    0.134029] NET: Registered protocol family 2
[    0.134796] TCP established hash table entries: 4096 (order: 3, 32768 bytes)
[    0.135269] TCP bind hash table entries: 4096 (order: 4, 65536 bytes)
[    0.135800] TCP: Hash tables configured (established 4096 bind 4096)
[    0.136301] TCP: reno registered
[    0.136637] UDP hash table entries: 256 (order: 1, 8192 bytes)
[    0.137044] UDP-Lite hash table entries: 256 (order: 1, 8192 bytes)
[    0.137659] NET: Registered protocol family 1
[    0.138209] RPC: Registered named UNIX socket transport module.
[    0.138622] RPC: Registered udp transport module.
[    0.138977] RPC: Registered tcp transport module.
[    0.139372] RPC: Registered tcp NFSv4.1 backchannel transport module.
[    0.139844] pci 0000:00:00.0: Limiting direct PCI/PCI transfers
[    0.140230] pci 0000:00:01.0: PIIX3: Enabling Passive Release
[    0.140595] pci 0000:00:01.0: Activating ISA DMA hang workarounds
[    0.141416] ACPI: PCI Interrupt Link [LNKD] enabled at IRQ 11
[    0.142292] pci 0000:00:02.0: Video device with shadowed ROM
[    0.142342] PCI: CLS 0 bytes, default 64
[    0.142619] microcode: CPU0 sig=0x206c1, pf=0x1, revision=0x1
[    0.143237] microcode: Microcode Update Driver: v2.00 <[email protected]>, Peter Oruba
[    0.144440] Scanning for low memory corruption every 60 seconds
[    0.145176] futex hash table entries: 256 (order: 2, 16384 bytes)
[    0.145593] audit: initializing netlink subsys (disabled)
[    0.146031] audit: type=2000 audit(1416871875.448:1): initialized
[    0.146850] HugeTLB registered 2 MB page size, pre-allocated 0 pages
[    0.149375] VFS: Disk quotas dquot_6.5.2
[    0.149771] Dquot-cache hash table entries: 512 (order 0, 4096 bytes)
[    0.150727] NFS: Registering the id_resolver key type
[    0.151096] Key type id_resolver registered
[    0.151409] Key type id_legacy registered
[    0.151814] 9p: Installing v9fs 9p2000 file system support
[    0.152265] msgmni has been set to 845
[    0.152748] SELinux:  Registering netfilter hooks
[    0.153579] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 251)
[    0.154190] io scheduler noop registered
[    0.154538] io scheduler deadline registered
[    0.154918] io scheduler cfq registered (default)
[    0.155364] pci_hotplug: PCI Hot Plug PCI Core version: 0.5
[    0.155785] Serial: 8250/16550 driver, 4 ports, IRQ sharing enabled
[    0.177702] 00:04: ttyS0 at I/O 0x3f8 (irq = 4, base_baud = 115200) is a 16550A
[    0.179250] Non-volatile memory driver v1.3
[    0.179571] Linux agpgart interface v0.103
[    0.180115] input: Power Button as /devices/LNXSYSTM:00/LNXPWRBN:00/input/input0
[    0.180669] ACPI: Power Button [PWRF]
[    0.181527] ACPI: PCI Interrupt Link [LNKC] enabled at IRQ 10
[    0.183449] ACPI: PCI Interrupt Link [LNKA] enabled at IRQ 10
[    0.184006] [drm] Initialized drm 1.1.0 20060810
[    0.185229] loop: module loaded
[    0.185708] virtio-pci 0000:00:04.0: irq 24 for MSI/MSI-X
[    0.185726] virtio-pci 0000:00:04.0: irq 25 for MSI/MSI-X
[    0.328773]  vda: vda1 vda2
[    0.330879] ata_piix 0000:00:01.1: version 2.13
[    0.331497] scsi host0: ata_piix
[    0.331996] scsi host1: ata_piix
[    0.332344] ata1: PATA max MWDMA2 cmd 0x1f0 ctl 0x3f6 bmdma 0xc0a0 irq 14
[    0.332770] ata2: PATA max MWDMA2 cmd 0x170 ctl 0x376 bmdma 0xc0a8 irq 15
[    0.333446] virtio-pci 0000:00:03.0: irq 26 for MSI/MSI-X
[    0.333463] virtio-pci 0000:00:03.0: irq 27 for MSI/MSI-X
[    0.333478] virtio-pci 0000:00:03.0: irq 28 for MSI/MSI-X
[    0.406083] e100: Intel(R) PRO/100 Network Driver, 3.5.24-k2-NAPI
[    0.406452] e100: Copyright(c) 1999-2006 Intel Corporation
[    0.406812] e1000: Intel(R) PRO/1000 Network Driver - version 7.3.21-k8-NAPI
[    0.407201] e1000: Copyright (c) 1999-2006 Intel Corporation.
[    0.407643] sky2: driver version 1.30
[    0.408139] ehci_hcd: USB 2.0 'Enhanced' Host Controller (EHCI) Driver
[    0.408501] ehci-pci: EHCI PCI platform driver
[    0.408878] ohci_hcd: USB 1.1 'Open' Host Controller (OHCI) Driver
[    0.409261] ohci-pci: OHCI PCI platform driver
[    0.409598] uhci_hcd: USB Universal Host Controller Interface driver
[    0.410282] uhci_hcd 0000:00:01.2: UHCI Host Controller
[    0.410694] uhci_hcd 0000:00:01.2: new USB bus registered, assigned bus number 1
[    0.411474] uhci_hcd 0000:00:01.2: irq 11, io base 0x0000c040
[    0.412025] usb usb1: New USB device found, idVendor=1d6b, idProduct=0001
[    0.412552] usb usb1: New USB device strings: Mfr=3, Product=2, SerialNumber=1
[    0.413114] usb usb1: Product: UHCI Host Controller
[    0.413450] usb usb1: Manufacturer: Linux 3.17.3-gentoo uhci_hcd
[    0.413797] usb usb1: SerialNumber: 0000:00:01.2
[    0.414279] hub 1-0:1.0: USB hub found
[    0.419053] hub 1-0:1.0: 2 ports detected
[    0.419865] usbcore: registered new interface driver usblp
[    0.420254] usbcore: registered new interface driver usb-storage
[    0.420670] i8042: PNP: PS/2 Controller [PNP0303:KBD,PNP0f13:MOU] at 0x60,0x64 irq 1,12
[    0.421891] serio: i8042 KBD port at 0x60,0x64 irq 1
[    0.422244] serio: i8042 AUX port at 0x60,0x64 irq 12
[    0.422663] mousedev: PS/2 mouse device common for all mice
[    0.423329] input: AT Translated Set 2 keyboard as /devices/platform/i8042/serio0/input/input1
[    0.425456] rtc_cmos 00:00: RTC can wake from S4
[    0.426047] rtc_cmos 00:00: rtc core: registered rtc_cmos as rtc0
[    0.426499] rtc_cmos 00:00: alarms up to one day, 114 bytes nvram, hpet irqs
[    0.426994] device-mapper: ioctl: 4.27.0-ioctl (2013-10-30) initialised: [email protected]
[    0.427656] hidraw: raw HID events driver (C) Jiri Kosina
[    0.428222] usbcore: registered new interface driver usbhid
[    0.428564] usbhid: USB HID core driver
[    0.429172] Netfilter messages via NETLINK v0.30.
[    0.429517] nf_conntrack version 0.5.0 (3380 buckets, 13520 max)
[    0.430045] ctnetlink v0.93: registering with nfnetlink.
[    0.430485] ip_tables: (C) 2000-2006 Netfilter Core Team
[    0.430846] TCP: cubic registered
[    0.431278] Initializing XFRM netlink socket
[    0.431726] NET: Registered protocol family 10
[    0.432393] ip6_tables: (C) 2000-2006 Netfilter Core Team
[    0.432783] sit: IPv6 over IPv4 tunneling driver
[    0.433277] NET: Registered protocol family 17
[    0.433632] 9pnet: Installing 9P2000 support
[    0.433974] Key type dns_resolver registered
[    0.434570] registered taskstats version 1
[    0.435247]   Magic number: 6:753:555
[    0.435607] console [netcon0] enabled
[    0.435921] netconsole: network logging started
[    0.436343] PM: Hibernation image not present or could not be loaded.
[    0.436346] ALSA device list:
[    0.436658]   No soundcards found.
[    0.526525] ata2.01: NODEV after polling detection
[    0.526799] ata2.00: ATAPI: QEMU DVD-ROM, 1.1.2, max UDMA/100
[    0.527680] ata2.00: configured for MWDMA2
[    0.528512] scsi 1:0:0:0: CD-ROM            QEMU     QEMU DVD-ROM     1.1. PQ: 0 ANSI: 5
[    0.539497] sr 1:0:0:0: [sr0] scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray
[    0.540080] cdrom: Uniform CD-ROM driver Revision: 3.20
[    0.540584] sr 1:0:0:0: Attached scsi CD-ROM sr0
[    0.540869] sr 1:0:0:0: Attached scsi generic sg0 type 5
[    0.721091] usb 1-1: new full-speed USB device number 2 using uhci_hcd
[    1.010027] usb 1-1: New USB device found, idVendor=0627, idProduct=0001
[    1.010488] usb 1-1: New USB device strings: Mfr=1, Product=3, SerialNumber=5
[    1.010922] usb 1-1: Product: QEMU USB Tablet
[    1.011335] usb 1-1: Manufacturer: QEMU 1.1.2
[    1.011703] usb 1-1: SerialNumber: 42
[    1.025854] input: ImExPS/2 Generic Explorer Mouse as /devices/platform/i8042/serio1/input/input3
[    1.030611] input: QEMU 1.1.2 QEMU USB Tablet as /devices/pci0000:00/0000:00:01.2/usb1/1-1/1-1:1.0/0003:0627:0001.0001/input/input4
[    1.031462] hid-generic 0003:0627:0001.0001: input,hidraw0: USB HID v0.01 Pointer [QEMU 1.1.2 QEMU USB Tablet] on usb-0000:00:01.2-1/input0
[    1.032473] md: Skipping autodetection of RAID arrays. (raid=autodetect will force)
[    1.144085] tsc: Refined TSC clocksource calibration: 2593.620 MHz
[    1.233818] kjournald starting.  Commit interval 5 seconds
[    1.234229] EXT3-fs (vda1): mounted filesystem with ordered data mode
[    1.234611] VFS: Mounted root (ext3 filesystem) readonly on device 253:1.
[    1.247665] devtmpfs: mounted
[    1.248951] Freeing unused kernel memory: 1108K (ffffffff81ed9000 - ffffffff81fee000)
[    1.249698] Write protecting the kernel read-only data: 14336k
[    1.255459] Freeing unused kernel memory: 1964K (ffff880001815000 - ffff880001a00000)
[    1.259167] Freeing unused kernel memory: 1412K (ffff880001c9f000 - ffff880001e00000)
[    8.571405] init-early.sh (884) used greatest stack depth: 11992 bytes left
[   36.577174] systemd-udevd[1057]: starting version 216
[   38.041743] random: systemd-udevd urandom read with 39 bits of entropy available
[   47.096362] EXT3-fs (vda1): using internal journal
[   47.640896] Adding 1571836k swap on /dev/vda2.  Priority:-1 extents:1 across:1571836k 
[   78.490697] device eth0 entered promiscuous mode
[   83.505336] vnstatd (1925) used greatest stack depth: 11960 bytes left
[  104.815490] random: nonblocking pool is initialized
[  413.636050] tun: Universal TUN/TAP device driver, 1.6
[  413.636053] tun: (C) 1999-2004 Max Krasnyansky <[email protected]>
[  416.404415] batman_adv: B.A.T.M.A.N. advanced 2014.4.0 (compatibility version 15) loaded
[  416.883606] batman_adv: bat0: Adding interface: fastd0
[  416.883628] batman_adv: bat0: The MTU of interface fastd0 is too small (1426) to handle the transport of batman-adv packets. Packets going over this interface will be fragmented on layer2 which could impact the performance. Setting the MTU to 1560 would solve the problem.
[  416.883632] batman_adv: bat0: Interface activated: fastd0
[  416.884747] batman_adv: bat0: orig_interval: Changing from: 1000 to: 5000
[  416.886336] batman_adv: bat0: bridge_loop_avoidance: Changing from: disabled to: enabled
[  416.887079] batman_adv: bat0: Changing gw mode from: off to: client
[  457.100304] ipip: IPv4 over IPv4 tunneling driver
[ 2165.176351] ldconfig (20423) used greatest stack depth: 11784 bytes left
[ 2546.656064] kworker/dying (114) used greatest stack depth: 11752 bytes left
[ 2847.713111] kworker/dying (16437) used greatest stack depth: 11704 bytes left
[ 4562.065973] nf_conntrack: automatic helper assignment is deprecated and it will be removed soon. Use the iptables CT target to attach helpers instead.
[ 6440.928104] kworker/dying (9625) used greatest stack depth: 11688 bytes left
[34735.321142] batman_adv: bat0: Changing gw mode from: client to: server
[34735.321164] batman_adv: bat0: Changing gateway bandwidth from: '10.0/2.0 MBit' to: '100.0/100.0 MBit'
[37380.212097] skbuff: skb_over_panic: text:ffffffffa001be32 len:1439 put:1380 head:ffff88000d2d9c00 data:ffff88000d2d9c62 tail:0x601 end:0x2c0 dev:fastd0
[37380.212566] ------------[ cut here ]------------
[37380.212713] kernel BUG at net/core/skbuff.c:100!
[37380.212870] invalid opcode: 0000 [#1] SMP 
[37380.213017] Modules linked in: xt_nat iptable_nat nf_nat_ipv4 nf_nat ipip batman_adv libcrc32c tun crc32c_intel
[37380.213017] CPU: 0 PID: 2058 Comm: fastd Not tainted 3.17.3-gentoo #1
[37380.213017] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2007
[37380.213017] task: ffff88001a3b91a0 ti: ffff880006c04000 task.ti: ffff880006c04000
[37380.213017] RIP: 0010:[<ffffffff81806a38>]  [<ffffffff81806a38>] skb_panic+0x5e/0x60
[37380.213017] RSP: 0018:ffff88001fc03cf8  EFLAGS: 00010296
[37380.213017] RAX: 000000000000008b RBX: ffff88001a3a7100 RCX: 0000000000000092
[37380.213017] RDX: 0000000000000033 RSI: 0000000000000246 RDI: 0000000000000246
[37380.213017] RBP: ffff88001fc03d18 R08: 0000000000000000 R09: 0000000000000000
[37380.213017] R10: ffffffff81a6b240 R11: 0000000000000000 R12: 0000000000000564
[37380.213017] R13: ffff88001fc03da0 R14: ffff880019e92b00 R15: ffff88000d8a7862
[37380.213017] FS:  00007f8cb93b6700(0000) GS:ffff88001fc00000(0000) knlGS:0000000000000000
[37380.213017] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[37380.213017] CR2: 00000000015fe000 CR3: 00000000176cf000 CR4: 00000000000006f0
[37380.213017] Stack:
[37380.213017]  ffff88000d2d9c62 0000000000000601 00000000000002c0 ffff880006cd4000
[37380.213017]  ffff88001fc03d28 ffffffff8162c921 ffff88001fc03d88 ffffffffa001be32
[37380.213017]  ffff88000377e508 ffff88000d8a784e ffff88001fc12a80 ffff88001a3a7100
[37380.213017] Call Trace:
[37380.213017]  <IRQ> 
[37380.213017] 
[37380.213017]  [<ffffffff8162c921>] skb_put+0x41/0x50
[37380.213017]  [<ffffffffa001be32>] batadv_frag_skb_buffer+0x272/0x470 [batman_adv]
[37380.213017]  [<ffffffffa0026283>] batadv_recv_frag_packet+0x183/0x200 [batman_adv]
[37380.213017]  [<ffffffffa001ff15>] batadv_batman_skb_recv+0xd5/0x110 [batman_adv]
[37380.213017]  [<ffffffff8107579d>] ? default_wake_function+0xd/0x10
[37380.213017]  [<ffffffff8163c752>] __netif_receive_skb_core+0x222/0x740
[37380.213017]  [<ffffffff81072695>] ? check_preempt_curr+0x85/0xa0
[37380.213017]  [<ffffffff8163cc91>] __netif_receive_skb+0x21/0x70
[37380.213017]  [<ffffffff8163cd7e>] process_backlog+0x9e/0x170
[37380.213017]  [<ffffffff8163d531>] net_rx_action+0x141/0x240
[37380.213017]  [<ffffffff81052e28>] __do_softirq+0xe8/0x280
[37380.213017]  [<ffffffff8180e99c>] do_softirq_own_stack+0x1c/0x30
[37380.213017]  <EOI> 
[37380.213017] 
[37380.213017]  [<ffffffff81053065>] do_softirq+0x55/0x60
[37380.213017]  [<ffffffff8163bca4>] netif_rx_ni+0x34/0x70
[37380.213017]  [<ffffffffa00098e3>] tun_get_user+0x413/0x840 [tun]
[37380.213017]  [<ffffffffa0009e0b>] tun_chr_aio_write+0x7b/0xa0 [tun]
[37380.213017]  [<ffffffff8115fdb5>] do_sync_write+0x55/0x90
[37380.213017]  [<ffffffff81160ada>] vfs_write+0xba/0x1f0
[37380.213017]  [<ffffffff81160fea>] SyS_write+0x4a/0xa0
[37380.213017]  [<ffffffff8180ce69>] system_call_fastpath+0x16/0x1b
[37380.213017] Code: 00 00 48 89 44 24 10 8b 87 c0 00 00 00 48 89 44 24 08 48 8b 87 d0 00 00 00 48 c7 c7 c0 cc c3 81 48 89 04 24 31 c0 e8 ca 97 ff ff <0f> 0b 55 48 89 f8 48 8b 57 30 48 89 e5 48 8b 0f 5d 80 e5 80 48 
[37380.213017] RIP  [<ffffffff81806a38>] skb_panic+0x5e/0x60
[37380.213017]  RSP <ffff88001fc03cf8>