Script started on Tue Jan 17 16:52:00 2017
Command: kgdb kernel.debug /var/crash/vmcore.8
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for details.
This GDB was configured as "i386-marcel-freebsd"...

Unread portion of the kernel message buffer:
panic: rm_rlock: recursed on non-recursive rmlock IPFW static rules @ /usr/local/src/sys/netpfil/ipfw/ip_fw2.c:1260

cpuid = 0
KDB: stack backtrace:
db_trace_self_wrapper(c0a7b2d2,0,e0705233,705258,1,...) at 0xc05065ca = db_trace_self_wrapper+0x2a/frame 0xe0705218
kdb_backtrace(c0acc711,0,c0a75559,e07052e4,0,...) at 0xc07710ad = kdb_backtrace+0x2d/frame 0xe0705280
vpanic(c0a75559,e07052e4,c0a75559,e07052e4,e07052e4,...) at 0xc072ee15 = vpanic+0x115/frame 0xe07052b4
kassert_panic(c0a75559,c0aa15f7,c0aa12bb,4ec,c0b6a81c,...) at 0xc072ecf9 = kassert_panic+0xd9/frame 0xe07052d8
_rm_rlock_debug(c0c2737c,e07053f8,0,c0aa12bb,4ec,...) at 0xc0729187 = _rm_rlock_debug+0x197/frame 0xe0705308
ipfw_chk(e0705490,4b3cc420,0,0,e07054b4,...) at 0xc08c7c88 = ipfw_chk+0xc88/frame 0xe0705448
ipfw_check_packet(0,e070557c,c5ed6800,2,0,...) at 0xc08ce768 = ipfw_check_packet+0xd8/frame 0xe0705558
pfil_run_hooks(c0c25f78,e070564c,c5ed6800,2,0,...) at 0xc0835628 = pfil_run_hooks+0x98/frame 0xe07055b0
ip_output(c6058d00,0,0,0,0,...) at 0xc086ff04 = ip_output+0xd34/frame 0xe0705698
icmp_reflect(c6053968,c6058dc8,14,c6058de4,c0ba9ab8,...) at 0xc086a86a = icmp_reflect+0x63a/frame 0xe0705740
icmp_error(c6053900,3,3,0,0,...) at 0xc086a0f6 = icmp_error+0x6c6/frame 0xe0705798
ipfw_chk(e0705920,4b3cc420,0,0,e0705910,...) at 0xc08ca66f = ipfw_chk+0x366f/frame 0xe07058d8
ipfw_check_packet(0,e0705a0c,c5ed6800,1,0,...) at 0xc08ce768 = ipfw_check_packet+0xd8/frame 0xe07059e8
pfil_run_hooks(c0c25f78,e0705a8c,c5ed6800,1,0,...) at 0xc0835628 = pfil_run_hooks+0x98/frame 0xe0705a40
ip_input(c6053900,e0705ae8,c0714c4b,c5d77300,8,...) at 0xc086c270 = ip_input+0x720/frame 0xe0705ab0
netisr_dispatch_src(1,0,c6053900) at 0xc08343c3 = netisr_dispatch_src+0xa3/frame 0xe0705ae8
netisr_dispatch(1,c6053900,e0705b30,96,0,...) at 0xc0834790 = netisr_dispatch+0x20/frame 0xe0705afc
ether_demux(c5ed6800,c6053900,6,0,80d0c,...) at 0xc081f286 = ether_demux+0x146/frame 0xe0705b30
ether_nh_input(c6053900,9a,96,c6053900,e0705bd4,...) at 0xc081fffe = ether_nh_input+0x36e/frame 0xe0705b60
netisr_dispatch_src(5,0,c6053900) at 0xc08343c3 = netisr_dispatch_src+0xa3/frame 0xe0705b98
netisr_dispatch(5,c6053900,4c7,c075b987,c0b6a81c,...) at 0xc0834790 = netisr_dispatch+0x20/frame 0xe0705bac
ether_input(c5ed6800,c6053900,c0a51fad,4c7,0,...) at 0xc081f5d1 = ether_input+0x61/frame 0xe0705bd4
rl_rxeof(c5ea8140,0,c0a51fad,5d5,c5ea8140,...) at 0xc05eace6 = rl_rxeof+0x126/frame 0xe0705c18
rl_intr(c5ea6000,0,0,55bb858c,c5de3948,...) at 0xc05e9c4c = rl_intr+0xec/frame 0xe0705c48
intr_event_execute_handlers(c0b6a790,c5de3900,c0a6f5a9,560,c0a6f303,...) at 0xc06f9f4e = intr_event_execute_handlers+0x8e/frame 0xe0705c70
ithread_loop(c5ecff50,e0705ce8,c0a6f303,408,0,...) at 0xc06fa540 = ithread_loop+0x90/frame 0xe0705cac
fork_exit(c06fa4b0,c5ecff50,e0705ce8) at 0xc06f756e = fork_exit+0x7e/frame 0xe0705cd4
fork_trampoline() at 0xc09ce130 = fork_trampoline+0x8/frame 0xe0705cd4
--- trap 0, eip = 0, esp = 0xe0705d20, ebp = 0 ---
Uptime: 8h8m11s
Physical memory: 1522 MB
Dumping 153 MB: 138 122 106 90 74 58 42 26 10

Reading symbols from /boot/modules/fdescfs.ko...done.
Loaded symbols for /boot/modules/fdescfs.ko
#0  doadump (textdump=1) at pcpu.h:206
206		__asm("movl %%fs:%1,%0" : "=r" (td)
(kgdb) bt
#0  doadump (textdump=1) at pcpu.h:206
#1  0xc072e913 in kern_reboot (howto=<value optimized out>) at /usr/local/src/sys/kern/kern_shutdown.c:366
#2  0xc072ee4d in vpanic (fmt=<value optimized out>, ap=<value optimized out>) at /usr/local/src/sys/kern/kern_shutdown.c:759
#3  0xc072ecf9 in kassert_panic (fmt=<value optimized out>) at /usr/local/src/sys/kern/kern_shutdown.c:649
#4  0xc0729187 in _rm_rlock_debug (rm=<value optimized out>, tracker=<value optimized out>, trylock=<value optimized out>)
    at /usr/local/src/sys/kern/kern_rmlock.c:644
#5  0xc08c7c88 in ipfw_chk (args=<value optimized out>) at /usr/local/src/sys/netpfil/ipfw/ip_fw2.c:1260
#6  0xc08ce768 in ipfw_check_packet (arg=<value optimized out>, m0=<value optimized out>, ifp=<value optimized out>)
    at /usr/local/src/sys/netpfil/ipfw/ip_fw_pfil.c:151
#7  0xc0835628 in pfil_run_hooks (ph=<value optimized out>, mp=<value optimized out>, dir=<value optimized out>)
    at /usr/local/src/sys/net/pfil.c:83
#8  0xc086ff04 in ip_output (m=0x0, opt=<value optimized out>, ro=<value optimized out>, imo=<value optimized out>, 
    inp=<value optimized out>) at /usr/local/src/sys/netinet/ip_output.c:121
#9  0xc086a86a in icmp_reflect (m=<value optimized out>) at /usr/local/src/sys/netinet/ip_icmp.c:908
#10 0xc086a0f6 in icmp_error (n=<value optimized out>, type=<value optimized out>, code=<value optimized out>, 
    dest=<value optimized out>, mtu=<value optimized out>) at /usr/local/src/sys/netinet/ip_icmp.c:352
#11 0xc08ca66f in ipfw_chk (args=<value optimized out>) at /usr/local/src/sys/netpfil/ipfw/ip_fw2.c:624
#12 0xc08ce768 in ipfw_check_packet (arg=<value optimized out>, m0=<value optimized out>, ifp=<value optimized out>)
    at /usr/local/src/sys/netpfil/ipfw/ip_fw_pfil.c:151
#13 0xc0835628 in pfil_run_hooks (ph=<value optimized out>, mp=<value optimized out>, dir=<value optimized out>)
    at /usr/local/src/sys/net/pfil.c:83
#14 0xc086c270 in ip_input (m=<value optimized out>) at /usr/local/src/sys/netinet/ip_input.c:584
#15 0xc08343c3 in netisr_dispatch_src (proto=<value optimized out>, source=<value optimized out>, m=0xc6053900)
    at /usr/local/src/sys/net/netisr.c:1120
#16 0xc0834790 in netisr_dispatch (proto=1, m=0xc6053900) at /usr/local/src/sys/net/netisr.c:1211
#17 0xc081f286 in ether_demux (ifp=<value optimized out>, m=0x0) at /usr/local/src/sys/net/if_ethersubr.c:850
#18 0xc081fffe in ether_nh_input (m=<value optimized out>) at /usr/local/src/sys/net/if_ethersubr.c:639
#19 0xc08343c3 in netisr_dispatch_src (proto=<value optimized out>, source=<value optimized out>, m=0xc6053900)
    at /usr/local/src/sys/net/netisr.c:1120
#20 0xc0834790 in netisr_dispatch (proto=5, m=0xc6053900) at /usr/local/src/sys/net/netisr.c:1211
#21 0xc081f5d1 in ether_input (ifp=<value optimized out>, m=0x0) at /usr/local/src/sys/net/if_ethersubr.c:759
#22 0xc05eace6 in rl_rxeof () at /usr/local/src/sys/dev/rl/if_rl.c:1224
#23 0xc05e9c4c in rl_intr (arg=<value optimized out>) at /usr/local/src/sys/dev/rl/if_rl.c:1516
#24 0xc06f9f4e in intr_event_execute_handlers (p=0xc0b6a790, ie=<value optimized out>)
    at /usr/local/src/sys/kern/kern_intr.c:1262
#25 0xc06fa540 in ithread_loop (arg=<value optimized out>) at /usr/local/src/sys/kern/kern_intr.c:1275
#26 0xc06f756e in fork_exit (callout=0xc06fa4b0 <ithread_loop>) at /usr/local/src/sys/kern/kern_fork.c:1040
#27 0xc09ce130 in fork_trampoline () at /usr/local/src/sys/i386/i386/exception.s:287
Current language:  auto; currently minimal
(kgdb) frame 5
#5  0xc08c7c88 in ipfw_chk (args=<value optimized out>) at /usr/local/src/sys/netpfil/ipfw/ip_fw2.c:1260
1260		IPFW_PF_RLOCK(chain);
(kgdb) l
1255			args->f_id.proto = proto;
1256			args->f_id.src_port = src_port = ntohs(src_port);
1257			args->f_id.dst_port = dst_port = ntohs(dst_port);
1258		}
1259	
1260		IPFW_PF_RLOCK(chain);
1261		if (! V_ipfw_vnet_ready) { /* shutting down, leave NOW. */
1262			IPFW_PF_RUNLOCK(chain);
1263			return (IP_FW_PASS);	/* accept */
1264		}
(kgdb) p args
$1 = <value optimized out>
(kgdb) p chain
No symbol "chain" in current context.
(kgdb) frame 4
#4  0xc0729187 in _rm_rlock_debug (rm=<value optimized out>, tracker=<value optimized out>, trylock=<value optimized out>)
    at /usr/local/src/sys/kern/kern_rmlock.c:644
644			KASSERT(rm_trackers_present(pcpu_find(curcpu), rm,
(kgdb) l
639			return (1);
640	
641	#ifdef INVARIANTS
642		if (!(rm->lock_object.lo_flags & LO_RECURSABLE) && !trylock) {
643			critical_enter();
644			KASSERT(rm_trackers_present(pcpu_find(curcpu), rm,
645			    curthread) == 0,
646			    ("rm_rlock: recursed on non-recursive rmlock %s @ %s:%d\n",
647			    rm->lock_object.lo_name, file, line));
648			critical_exit();
(kgdb) p rm
$2 = <value optimized out>
(kgdb) 
Command exit status: 0
Script done on Tue Jan 17 16:52:51 2017