The panic message itself a problem in the pf_find_state_all_exists() function FreeBSD-14.0-CURRENT-amd64-20230727-474708c334a7-264358 b> bt Tracing pid 19986 tid 104653 td 0xfffffe008b9c4000 kdb_enter() at kdb_enter+0x32/frame 0xfffffe008c47e220 vpanic() at vpanic+0x17a/frame 0xfffffe008c47e270 panic() at panic+0x43/frame 0xfffffe008c47e2d0 pf_find_state_all_exists() at pf_find_state_all_exists+0xb1/frame 0xfffffe008c47e2f0 pf_get_sport() at pf_get_sport+0x24b/frame 0xfffffe008c47e390 pf_get_translation() at pf_get_translation+0x410/frame 0xfffffe008c47e420 pf_test_rule() at pf_test_rule+0x2f4/frame 0xfffffe008c47e880 pf_test() at pf_test+0x81d/frame 0xfffffe008c47ea30 pf_check_out() at pf_check_out+0x22/frame 0xfffffe008c47ea50 pfil_mbuf_out() at pfil_mbuf_out+0x58/frame 0xfffffe008c47ea80 ip_output() at ip_output+0xce0/frame 0xfffffe008c47eb80 udp_send() at udp_send+0xa0b/frame 0xfffffe008c47ec80 sosend_dgram() at sosend_dgram+0x321/frame 0xfffffe008c47ece0 sousrsend() at sousrsend+0x79/frame 0xfffffe008c47ed40 dofilewrite() at dofilewrite+0x82/frame 0xfffffe008c47ed90 sys_write() at sys_write+0xc2/frame 0xfffffe008c47ee00 amd64_syscall() at amd64_syscall+0x138/frame 0xfffffe008c47ef30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe008c47ef30 --- syscall (4, FreeBSD ELF64, write), rip = 0x49b0bb, rsp = 0x84d255a40, rbp = 0x84d255a90 --- db> reboot Uptime: 55m50s
What's the configuration here? (pf rules, network layout, ...) Can this be reproduced? Consistently or intermittently? Follow the instructions on https://docs.freebsd.org/en/books/developers-handbook/kerneldebug/ and extract a kgdb backtrace as well as all local variables in the pf_find_state_all_exists() and pf_get_sport() frames (`frame XX`, `info locals`).
(In reply to Kristof Provost from comment #1) Hi, "pf rules" set optimization aggressive set timeout { adaptive.start 0, adaptive.end 0 } set limit states 200000 set limit src-nodes 20000 set timeout interval 10 set timeout frag 30 #Interface defines loopback = "lo0" igc1_if_gateway = "10.10.10.1" igc2_if_gateway = "192.168.111.1" igc0_if = "igc0" pppoe_igc1_if = "pppoe_igc1" igc2_if = "igc2" igc3_if = "igc3" igc4_if = "igc4" igc5_if = "igc5" igc3_20_if = "igc3.20" igc4_4093_if = "igc4.4093" igc3_2_if = "igc3.2" # Private networks, we are going to block incoming traffic from them priv_nets = "{ 127.0.0.0/8, 192.168.0.0/16, 172.16.0.0/12, 10.0.0.0/8 }" table <country_xyz> persist file "/etc/pf/country_xyz.txt" table <quarantine_ip_list> { } table <block_ip_list> persist file "/tmp/block_ip_list.txt" table <botnet_ip_list> persist file "/tmp/botnet_ip_list.txt" table <interface_ip_addresses> { 192.168.111.100,192.168.10.1,10.10.20.1,192.168.169.1,172.16.0.0/24 } ### options set block-policy drop set loginterface igc1 #Global Mac Adress ether anchor "global_mac" #Captive Portal Rules ether pass on { igc3.20 } tag "captiveportal_rdr_igc3.20" ether anchor "captiveportal_allowed_mac_igc3.20" on { igc3.20 } ether anchor "captiveportal_allowed_ip_igc3.20" on { igc3.20 } ether anchor "captiveportal_auth_igc3.20" on { igc3.20 } ### Scrub scrub from any to <vpn_networks> fragment no reassemble scrub from <vpn_networks> to any fragment no reassemble scrub in log nat-anchor "ftp-proxy/*" rdr-anchor "ftp-proxy/*" #### LOOPBACK NAT no nat proto carp all nat on igc0 from { 10.10.10.1/24 } to { 10.10.10.1/24 } -> 10.10.10.1 nat on igc3 from { 192.168.10.1/24 } to { 192.168.10.1/24 } -> 192.168.10.1 nat on igc3.20 from { 10.10.20.1/24 } to { 10.10.20.1/24 } -> 10.10.20.1 nat on wg0 from any to { 192.168.237.1/24 } -> 192.168.237.1/32 source-hash #### NAT RULES nat on igc2 inet from { any } to { a.b.c.d/32 } -> igc2:0 source-hash nat on enc0 inet from { 192.168.10.0/24 } to { 192.168.1.0/24 } -> x.y.z.t/32 source-hash nat on igc2 inet from { any } to { a.b.c.d/32 } -> igc2:0 source-hash #nat_id=17 nat on enc0 inet from { 192.168.10.0/24 } to { 192.168.1.0/24 } -> x.y.z.t/32 source-hash #DEFAULT NAT RULES nat on pppoe_igc1 inet from { !pppoe_igc1 } to any port 500 -> (pppoe_igc1:0) static-port #static-port -> for_ipsec nat on pppoe_igc1 inet from { !pppoe_igc1 } to any -> (pppoe_igc1:0) port 1024:65535 nat on igc2 inet from { !igc2 } to any port 500 -> (igc2:0) static-port #static-port -> for_ipsec nat on igc2 inet from { !igc2 } to any -> (igc2:0) port 1024:65535 nat on igc4 inet from { !igc4 } to any port 500 -> (igc4:0) static-port #static-port -> for_ipsec nat on igc4 inet from { !igc4 } to any -> (igc4:0) port 1024:65535 #captive_portal_redirect rules rdr pass on igc3.20 proto { tcp } from any to !10.10.20.1 port { 80 } tagged captiveportal_rdr_igc3.20 -> 10.10.20.1 port 80 rdr pass on igc3.20 proto { tcp } from any to !10.10.20.1 port { 443 } tagged captiveportal_rdr_igc3.20 -> 10.10.20.1 port 443 #Port Forwarding Rules rdr log on igc3 proto { tcp udp } from any to any port { 53 } -> 192.168.10.1 port 53 #LAN_interface_redirect_rule_130 #WAN_interface_redirect_rule_111 rdr log on { pppoe_igc1 } proto { tcp } from { any } to any port { 21 } -> 172.16.0.2 port 21 #forwarding_111 ### CARP Firewall Failover pass in quick on igc0 proto carp from any to any keep state pass in quick on igc2 proto carp from any to any keep state pass in quick on igc3 proto carp from any to any keep state pass in quick on igc4 proto carp from any to any keep state pass in quick on igc5 proto carp from any to any keep state pass in quick on lo0 inet from 127.0.0.1 to 127.0.0.1 keep state pass out quick on lo0 inet from 127.0.0.1 to 127.0.0.1 keep state #### DROP Mysql/Postgresql Ports #### pass in quick inet proto { tcp udp } from 127.0.0.1 to 127.0.0.1 block drop in log quick inet proto { tcp, udp } from any to (self) port 9300 block drop in log quick inet proto { tcp, udp } from any to (self) port 9200 block drop in log quick inet proto { tcp, udp } from any to (self) port 6379 ## Quarantine IP List block in log quick inet from <quarantine_ip_list> to any block in log quick inet from any to <quarantine_ip_list> block out log quick inet from <quarantine_ip_list> to any block out log quick inet from any to <quarantine_ip_list> ## BLOCK botnet AND blocked ip list block in log quick inet from <block_ip_list> to any block in log quick inet from any to <block_ip_list> block out log quick inet from <block_ip_list> to any block out log quick inet from any to <block_ip_list> block in log quick inet from <botnet_ip_list> to any block in log quick inet from any to <botnet_ip_list> block out log quick inet from <botnet_ip_list> to any block out log quick inet from any to <botnet_ip_list> ## block ipv6 block in quick inet6 all block out quick inet6 all load anchor "anomaly" from "/etc/anomaly.txt" #anomaly anchor "anomaly" ## Port Forwarding Allow Rules # Antispoof WAN ports antispoof log for pppoe_igc1 antispoof log for igc2 antispoof log for igc4 # IPSec Allow Rule pass out quick route-to ( igc2 192.168.111.1 ) proto udp from (self) to a.b.c.d port = 500 keep state label "ipsec_vpn_1_500" pass in quick on igc2 reply-to ( igc2 192.168.111.1 ) proto udp from a.b.c.d to (self) port = 500 keep state label "ipsec_vpn_1_500" pass out quick route-to ( igc2 192.168.111.1 ) proto udp from (self) to a.b.c.d port = 4500 keep state label "ipsec_vpn_1_4500" pass in quick on igc2 reply-to ( igc2 192.168.111.1 ) proto udp from a.b.c.d to (self) port = 4500 keep state label "ipsec_vpn_1_4500" pass out quick route-to ( igc2 192.168.111.1 ) proto esp from (self) to a.b.c.d keep state label "ipsec_vpn_1_esp" pass in quick on igc2 reply-to ( igc2 192.168.111.1 ) proto udp from a.b.c.d to (self) keep state label "ipsec_vpn_1_esp" # SSLVPN Allow Rule pass out log quick route-to ( pppoe_igc1 10.10.10.1 ) proto { udp } from (self) port 1194 to any keep state label "ssl_vpn_1" pass in log quick on pppoe_igc1 reply-to ( pppoe_igc1 10.10.10.1 ) proto udp from any to (self) port = 1194 keep state label "ssl_vpn_1" #### Lan access rule pass in quick on igc0 from any to (igc0) keep state label "anti_lock_rule" pass in quick on igc3 from any to (igc3) keep state label "anti_lock_rule" pass in quick on igc5 from any to (igc5) keep state label "anti_lock_rule" pass in quick on igc3.20 from any to (igc3.20) keep state label "anti_lock_rule" pass in quick on igc4.4093 from any to (igc4.4093) keep state label "anti_lock_rule" pass in quick on igc3.2 from any to (igc3.2) keep state label "anti_lock_rule" #Syslog send pass out quick proto { udp } from any to 192.168.10.244 port 514 pass out quick proto { udp } from any to 192.168.10.233 port 514 #Captive Portal Allow rules #captive_portal_pass_rules pass in quick on igc3.20 proto { tcp } from any to 10.10.20.1 port { 80 } keep state(sloppy) pass out quick on igc3.20 proto { tcp } from 10.10.20.1 port { 80 } to any flags any keep state(sloppy) block in quick on igc3.20 from any to !10.10.20.1 tagged captiveportal_rdr_igc3.20 # Custom Rules pass in log quick on { LAN_ZONE } inet from { 192.168.10.233/32 } to { any } flags S/SA keep state label "custom_rule_57" tag "ltag_57" block drop in log quick on { igc0 } inet from { any } to { 192.168.10.0/24 } label "custom_rule_71" tag "ltag_71" block drop in log quick on { igc3 } proto { tcp udp } from { any } to { any } port { 25 } label "custom_rule_40" tag "ltag_40" pass in log quick on { igc3 } inet from { 192.168.10.1/32 } to { 192.168.10.244/32 } flags S/SA keep state label "custom_rule_59" tag "ltag_59" block drop in log quick on { igc3 } inet from { 192.168.10.244/32 } to { any } label "custom_rule_56" tag "ltag_56" pass in log quick on { igc3.20 } inet from { any } to { 192.168.10.183/32 } flags S/SA keep state label "custom_rule_61" tag "ltag_61" block drop in log quick on { igc3.20 } inet from { any } to { 192.168.10.0/24 172.16.0.0/24 } label "custom_rule_50" tag "ltag_50" pass in log quick on { igc3 igc0 } inet from { any } to { any } flags S/SA keep state label "custom_rule_41" tag "ltag_41" pass out log quick reply-to { ( pppoe_igc1 10.10.10.1 ) } proto { tcp } from { any } to { any } port { 3535 } flags S/SA keep state label "custom_rule_73" tag "ltag_73" # WAN to LAN deny all block drop in log quick on pppoe_igc1 from any to any label "custom_rule_-1" tag "ltag_-1" block drop in log quick on igc2 from any to any label "custom_rule_-1" tag "ltag_-1" block drop in log quick on igc4 from any to any label "custom_rule_-1" tag "ltag_-1" pass in log label "custom_rule_-2" tag "ltag_-2" pass out log label "custom_rule_-3" tag "ltag_-3" ########################################################### # ether rules # pfctl -se ether anchor "global_mac" l3 all ether pass on igc3.20 l3 all tag captiveportal_rdr_igc3.20 ether anchor "captiveportal_allowed_mac_igc3.20" on igc3.20 l3 all ether anchor "captiveportal_allowed_ip_igc3.20" on igc3.20 l3 all ether anchor "captiveportal_auth_igc3.20" on igc3.20 l3 all # pfctl -a captiveportal_auth_igc3.20 -se ether pass in quick from 7c:6a:ab:7d:da:a6 l3 all tag captiveportal_auth_igc3.20 dnpipe 1006 ether pass out quick to 7c:6a:ab:7d:da:a6 l3 all tag captiveportal_auth_igc3.20 dnpipe 6 # pipes # dnctl pipe 1006 show you have mail 01006: 2.000 Mbit/s 0 ms burst 0 q132078 50 sl. 0 flows (1 buckets) sched 66542 weight 0 lmax 0 pri 0 droptail sched 66542 type FIFO flags 0x1 64 buckets 0 active mask: 0x00 0xffffffff/0x0000 -> 0x00000000/0x0000 # dnctl pipe 6 show 00006: 10.000 Mbit/s 0 ms burst 0 q131078 50 sl. 0 flows (1 buckets) sched 65542 weight 0 lmax 0 pri 0 droptail sched 65542 type FIFO flags 0x1 64 buckets 0 active mask: 0x00 0x00000000/0x0000 -> 0xffffffff/0x0000 # kldstat you have mail Id Refs Address Size Name 1 74 0xffffffff80200000 1d4f6a0 kernel 2 1 0xffffffff81f50000 108e8 carp.ko 3 2 0xffffffff81f62000 946c8 pf.ko 4 2 0xffffffff81ff7000 4c108 ipfw.ko 5 1 0xffffffff82600000 462be0 zfs.ko 6 1 0xffffffff82520000 4240 ichsmb.ko 7 1 0xffffffff82525000 2178 smbus.ko 8 1 0xffffffff82528000 12808 dummynet.ko 9 1 0xffffffff8253b000 42a0 ipfw_nat.ko 10 1 0xffffffff82540000 d932 libalias.ko 11 1 0xffffffff8254e000 2e560 if_wg.ko 12 1 0xffffffff8257d000 2240 pflog.ko 13 1 0xffffffff82580000 2224 speaker.ko 14 1 0xffffffff82583000 72f8 if_vxlan.ko 15 1 0xffffffff8258b000 25b8 if_enc.ko 16 1 0xffffffff8258e000 76b0 if_ovpn.ko 17 1 0xffffffff82596000 12848 ipsec.ko 18 1 0xffffffff825a9000 52e0 ng_pppoe.ko 19 8 0xffffffff825af000 bb28 netgraph.ko 20 1 0xffffffff825bb000 38b8 ng_socket.ko 21 1 0xffffffff825bf000 4404 ng_mppc.ko 22 1 0xffffffff825c4000 20b0 rc4.ko 23 1 0xffffffff825c7000 23b8 ng_iface.ko 24 1 0xffffffff825ca000 61e8 ng_ppp.ko 25 1 0xffffffff825d1000 2138 ng_tee.ko 26 1 0xffffffff825d4000 31c8 ng_ether.ko 27 1 0xffffffff825d8000 2138 ng_tcpmss.ko 28 1 0xffffffff825db000 2538 ipdivert.ko igc0@pci0:1:0:0: class=0x020000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x125c subvendor=0x8086 subdevice=0x0000 vendor = 'Intel Corporation' device = 'Ethernet Controller I226-V' class = network subclass = ethernet "Can this be reproduced? Consistently or intermittently?" It occurs intermittently. When this panic happens, four days later the same panic occurred again
We're going to need to look at a core dump. It's a very odd place for a panic, in the sense that nothing that's changed in the past several years should affect that, and it's also a relatively straightforward call. It really shouldn't be panicking there. You're using a couple of new-ish features (dummynet, ethernet rules, openvpn DCO), but they don't appear to be involved here, and you're also using them the way pfsense does and I've not seen any similar reports there. So, when this recurs please do the kgdb dance. We'll want the local variables in pf_find_state_all_exists() and pf_find_state_all() for starters. Do keep that core file around, because we're almost certainly going to want to poke it more. Also gather the full panic output, not just the backtrace, because it's not quite clear to me if we're running into an assertion failure or a segmentation fault or something else.
(In reply to Kristof Provost from comment #3) "It's a very odd place for a panic, in the sense that nothing that's changed in the past several years should affect that, and it's also a relatively straightforward call. It really shouldn't be panicking there." Hi, Yes you are right. I have recognized that one of my services get trouble and it tried to create 10 thousands redis connection to my redis server. After corrected this issue and reloaded my FreeBSD Captive Portal gw with nearly 100 clients there is no problem like panic. If any pf related panic occurs i wlil inform you Thanks.
(In reply to Kristof Provost from comment #3) Hi, i have encountered the same panic, you can find pf_find_state_all_exists frame bt and info locals at #12 from debug logs shown below Not: i have had solved redis problem which is i mentioned #Comment4 and there are the same panic with ALPHA2 image too. root@test:~/crash1 # kgdb /boot/kernel/kernel vmcore.0 ... Reading symbols from /boot/kernel/kernel... Reading symbols from /usr/lib/debug//boot/kernel/kernel.debug... Unread portion of the kernel message buffer: panic: Assertion _s->lock == &_ih->lock failed at /usr/src/sys/netpfil/pf/pf.c:1653 cpuid = 2 time = 1692945003 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe000785f200 vpanic() at vpanic+0x149/frame 0xfffffe000785f250 panic() at panic+0x43/frame 0xfffffe000785f2b0 pf_find_state_all_exists() at pf_find_state_all_exists+0xb1/frame 0xfffffe000785f2d0 pf_get_sport() at pf_get_sport+0x2bb/frame 0xfffffe000785f370 pf_get_translation() at pf_get_translation+0x424/frame 0xfffffe000785f400 pf_test_rule() at pf_test_rule+0x304/frame 0xfffffe000785f870 pf_test() at pf_test+0x139b/frame 0xfffffe000785fa30 pf_check_out() at pf_check_out+0x22/frame 0xfffffe000785fa50 pfil_mbuf_out() at pfil_mbuf_out+0x58/frame 0xfffffe000785fa80 ip_output() at ip_output+0xce7/frame 0xfffffe000785fb80 udp_send() at udp_send+0xa0b/frame 0xfffffe000785fc80 sosend_dgram() at sosend_dgram+0x321/frame 0xfffffe000785fce0 sousrsend() at sousrsend+0x79/frame 0xfffffe000785fd40 dofilewrite() at dofilewrite+0x82/frame 0xfffffe000785fd90 sys_write() at sys_write+0xc2/frame 0xfffffe000785fe00 amd64_syscall() at amd64_syscall+0x138/frame 0xfffffe000785ff30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe000785ff30 --- syscall (4, FreeBSD ELF64, write), rip = 0x49b17b, rsp = 0x851127a40, rbp = 0x851127a90 --- KDB: enter: panic __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:59 59 __asm("movq %%gs:%P1,%0" : "=r" (td) : "n" (offsetof(struct pcpu, ============================================================================= root@test:~/crash1 # cat info.last Dump header from device: /dev/ada0s1b Architecture: amd64 Architecture Version: 2 Dump Length: 672002048 Blocksize: 512 Compression: none Dumptime: 2023-08-25 09:30:03 +0300 Hostname: test Magic: FreeBSD Kernel Dump Version String: FreeBSD 14.0-CURRENT amd64 1400093 #0 main-n264491-8a5c836b51ce: Thu Aug 3 08:15:15 UTC 2023 root@releng1.nyi.freebsd.org:/usr/obj/usr/src/amd64.amd64/sys/GENERIC Panic String: Assertion _s->lock == &_ih->lock failed at /usr/src/sys/netpfil/pf/pf.c:1653 Dump Parity: 1534047024 Bounds: 0 Dump Status: good ============================================================================ ### frame bt and info locals ### (kgdb) frame 0 #0 __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:59 59 __asm("movq %%gs:%P1,%0" : "=r" (td) : "n" (offsetof(struct pcpu, (kgdb) info locals td = <optimized out> (kgdb) up #1 doadump (textdump=textdump@entry=0) at /usr/src/sys/kern/kern_shutdown.c:407 407 dump_savectx(); (kgdb) info locals error = 0 coredump = <optimized out> (kgdb) up #2 0xffffffff804a2f1a in db_dump (dummy=<optimized out>, dummy2=<optimized out>, dummy3=<optimized out>, dummy4=<optimized out>) at /usr/src/sys/ddb/db_command.c:593 593 error = doadump(false); (kgdb) info locals error = <optimized out> (kgdb) up #3 0xffffffff804a2d1d in db_command (last_cmdp=<optimized out>, cmd_table=<optimized out>, dopager=true) at /usr/src/sys/ddb/db_command.c:506 506 (*cmd->fcn)(addr, have_addr, count, modif); (kgdb) info locals modif = "\000\003\000\000\000\000\000\000$\320y\201\377\377\377\377\020\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\b\000\000\0000\000\000\000\260\356\205\a\000\376\377\377P\356\205\a\000\376\377\377$\320y\201\377\377\377\377\360\356\205\a\000\376\377\377\030QJ\200\377\377\377\377@\361\205\a\000\376\377\377\n\000\000\000\000\000\000\000 \320y\201\377\377\377\377\000\000\000\000\000\000\000\000@\361\205\a\000\376\377\377" cmd = 0xffffffff8161b4c8 <db_cmds+616> have_addr = <optimized out> t = <optimized out> result = <optimized out> addr = <unavailable> count = <unavailable> (kgdb) up #4 0xffffffff804a29dd in db_command_loop () at /usr/src/sys/ddb/db_command.c:553 553 db_command(&db_last_command, &db_cmd_table, /* dopager */ true); (kgdb) info locals No locals. (kgdb) up #5 0xffffffff804a60b6 in db_trap (type=<optimized out>, code=<optimized out>) at /usr/src/sys/ddb/db_main.c:270 270 db_command_loop(); (kgdb) info locals jb = {{_jb = {0, -2198897037544, -2198897037408, -2198897036992, 0, -2198897037536, 3, -2142609340, -2198897037440, -2137449745, -2127020736, -2120941952}}} bkpt = <optimized out> watchpt = false prev_jb = 0x0 why = <optimized out> (kgdb) up #6 0xffffffff80b99d53 in kdb_trap (type=type@entry=3, code=code@entry=0, tf=tf@entry=0xfffffe000785f140) at /usr/src/sys/kern/subr_kdb.c:792 792 handled = be->dbbe_trap(type, code); (kgdb) info locals __pc = <optimized out> __pc = <optimized out> other_cpus = {__bits = {-2199023255552, 72057594037927968, -2198897036848, -2128592352}} be = 0xffffffff8161c020 <ddb_dbbe> intr = 70 did_stop_cpus = <optimized out> handled = <optimized out> (kgdb) up #7 0xffffffff81045db9 in trap (frame=0xfffffe000785f140) at /usr/src/sys/amd64/amd64/trap.c:610 610 if (kdb_trap(type, dr6, frame)) (kgdb) info locals __pc = <optimized out> __pc = <optimized out> __pc = <optimized out> ksi = {ksi_link = {tqe_next = 0xffffffff804a624c <db_printf+92>, tqe_prev = 0x319097d937330e12}, ksi_info = { si_signo = 4, si_errno = 0, si_code = 4829563, si_pid = 0, si_uid = 1360165440, si_status = 8, si_addr = 0x319097d937330e12, si_value = {sival_int = 1007, sival_ptr = 0x3ef, sigval_int = 1007, sigval_ptr = 0x3ef}, _reason = {_fault = {_trapno = 4829563}, _timer = {_timerid = 4829563, _overrun = 0}, _mesgq = {_mqd = 4829563}, _poll = {_band = 4829563}, _capsicum = {_syscall = 4829563}, __spare__ = { __spare1__ = 4829563, __spare2__ = {1360165440, 8, 926092818, 831559641, 256, 0, -1}}}}, ksi_flags = 65062, ksi_sigq = 0xffffffff81ff79f1} td = 0xfffffe008d272ac0 p = <optimized out> dr6 = 0 type = 3 addr = <optimized out> signo = <optimized out> ucode = <optimized out> pf = <optimized out> i = <optimized out> (kgdb) up #8 <signal handler called> (kgdb) (kgdb) info locals No locals. (kgdb) up #9 kdb_enter (why=<optimized out>, msg=<optimized out>) at /usr/src/sys/kern/subr_kdb.c:558 558 kdb_why = KDB_WHY_UNSET; (kgdb) info locals No locals. (kgdb) up #10 0xffffffff80b4b86a in vpanic (fmt=0xffffffff81ff79f1 "Assertion %s failed at %s:%d", ap=ap@entry=0xfffffe000785f290) at /usr/src/sys/kern/kern_shutdown.c:960 960 else if (!newpanic && debugger_on_recursive_panic) (kgdb) info locals buf = "Assertion _s->lock == &_ih->lock failed at /usr/src/sys/netpfil/pf/pf.c:1653", '\000' <repeats 179 times> __pc = <optimized out> __pc = <optimized out> __pc = <optimized out> other_cpus = {__bits = {11, 0, 0, 0}} td = 0xfffffe008d272ac0 bootopt = <optimized out> newpanic = <optimized out> (kgdb) up #11 0xffffffff80b4b633 in panic (fmt=0xffffffff8194fec0 <cnputs_mtx> "\257\346\023\201\377\377\377\377") at /usr/src/sys/kern/kern_shutdown.c:896 896 vpanic(fmt, ap); (kgdb) info locals ap = {{gp_offset = 32, fp_offset = 48, overflow_arg_area = 0xfffffe000785f2c0, reg_save_area = 0xfffffe000785f260}} (kgdb) frame 12 #12 0xffffffff81fb0641 in pf_find_state_all_exists (key=<optimized out>, dir=<optimized out>) at /usr/src/sys/netpfil/pf/pf.c:1653 1653 PF_STATE_UNLOCK(s); (kgdb) info locals _s = 0xfffff8010fff4c60 _ih = <optimized out> s = 0xfffff8010fff4c60 (kgdb) up #13 0xffffffff81fcae5b in pf_get_sport (af=2 '\002', proto=17 '\021', r=0xfffff80042c13800, saddr=0xfffff800b77e583c, sport=23689, daddr=<optimized out>, dport=514, naddr=0xfffff80036adacc8, nport=0xfffff80036adacda, low=50001, high=65535, sn=0xfffffe000785f750) at /usr/src/sys/netpfil/pf/pf_lb.c:282 282 if (!pf_find_state_all_exists(&key, PF_IN)) { (kgdb) info locals cut = 59865 tmp = 65062 key = {addr = {{{v4 = {s_addr = 4094994624}, v6 = {__u6_addr = {__u6_addr8 = "\300\250\024\364", '\000' <repeats 11 times>, __u6_addr16 = {43200, 62484, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {4094994624, 0, 0, 0}}}, addr8 = "\300\250\024\364", '\000' <repeats 11 times>, addr16 = {43200, 62484, 0, 0, 0, 0, 0, 0}, addr32 = {4094994624, 0, 0, 0}}}, {{v4 = {s_addr = 18131136}, v6 = {__u6_addr = { __u6_addr8 = "\300\250\024\001", '\000' <repeats 11 times>, __u6_addr16 = {43200, 276, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {18131136, 0, 0, 0}}}, addr8 = "\300\250\024\001", '\000' <repeats 11 times>, addr16 = {43200, 276, 0, 0, 0, 0, 0, 0}, addr32 = {18131136, 0, 0, 0}}}}, port = {514, 9982}, af = 2 '\002', proto = 17 '\021', pad = "\000"} init_addr = {{v4 = {s_addr = 0}, v6 = {__u6_addr = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, addr8 = '\000' <repeats 15 times>, addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, addr32 = {0, 0, 0, 0}}} (kgdb) up #14 0xffffffff81fca044 in pf_get_translation (pd=pd@entry=0xfffffe000785f8f8, m=m@entry=0xfffff8005db0ad00, off=off@entry=20, kif=kif@entry=0xfffff80004a82d00, sn=sn@entry=0xfffffe000785f750, skp=skp@entry=0xfffffe000785f780, nkp=0xfffffe000785f818, saddr=0xfffff800b77e583c, daddr=0xfffff800b77e5840, sport=23689, dport=514, anchor_stack=0xfffffe000785f440) at /usr/src/sys/netpfil/pf/pf_lb.c:641 641 } else if (pf_get_sport(pd->af, pd->proto, r, saddr, sport, (kgdb) info locals r = 0xfffff80042c13800 naddr = <optimized out> nport = 0x12 low = <optimized out> high = <optimized out> (kgdb) up #15 0xffffffff81fba584 in pf_test_rule (rm=rm@entry=0xfffffe000785f9d0, sm=sm@entry=0xfffffe000785f9e0, kif=kif@entry=0xfffff80004a82d00, m=0xfffff8005db0ad00, off=off@entry=20, pd=pd@entry=0xfffffe000785f8f8, am=0xfffffe000785f9c8, rsm=0xfffffe000785f9b0, inp=0xfffff80058c69380) at /usr/src/sys/netpfil/pf/pf.c:4430 4430 if ((nr = pf_get_translation(pd, m, off, kif, &nsn, &sk, (kgdb) info locals anchor_stack = {{rs = 0x0, r = 0xfffff8007649f000, child = 0x0}, {rs = 0xfffffe0008cb8b38, r = 0x1081, child = 0xffffffff81ff69e0}, {rs = 0xfffffe000785f4b0, r = 0xffffffff80b446c4 <_rm_runlock_debug+100>, child = 0xfffff80003531040}, {rs = 0x0, r = 0x0, child = 0xfffff8004e151c02}, {rs = 0xfffffe008d272ac0, r = 0xfffff80004a82d00, child = 0xfffffe000785f970}, {rs = 0xffffffff81fb5359 <pf_test_eth+4233>, r = 0xfffffe0008cbb9c8, child = 0xfffff8005dc68800}, {rs = 0x0, r = 0xfffffe008b94fe40, child = 0x0}, { rs = 0xfffff80068b9cb18, r = 0xfffffe000785f6b0, child = 0xffffffff80bbd057 <witness_checkorder+919>}, {rs = 0x11c, r = 0xfffff80068b9cb60, child = 0x8}, {rs = 0xfffff80068b9cb60, r = 0x0, child = 0xfffffe0008cb8530}, {rs = 0xfffffe000785f6f0, r = 0xffffffff80bbd057 <witness_checkorder+919>, child = 0xfffffe000785f600}, {rs = 0xffffffff80c12f8c <getblkx+1484>, r = 0x40, child = 0x46}, { rs = 0xfffffe000785f5a0, r = 0xffffffff8117132c, child = 0x0}, {rs = 0x0, r = 0xfffffffffffffff4, child = 0xfffff80068b9cb60}, {rs = 0x0, r = 0xfffff80068b9cb60, child = 0x0}, {rs = 0x11c, r = 0x0, child = 0xfffffe006a3c4478}, {rs = 0xfffffe000785f780, r = 0xffffffff80bbd057 <witness_checkorder+919>, child = 0x8bd48b34a}, {rs = 0x11c, r = 0x8000, child = 0xfffff80068b9ca80}, {rs = 0x8e01cd, r = 0x8, child = 0xffffffff819d7f20 <w_locklistdata+248912>}, {rs = 0xffffffff80c27518 <cluster_read+232>, r = 0x1af200000008, child = 0xfffffe000785f6c0}, {rs = 0x0, r = 0xfffff80068b9cb60, child = 0xfffffe000785f7f0}, {rs = 0xffffffff80bbd057 <witness_checkorder+919>, r = 0xffffffff819d7f08 <w_locklistdata+248888>, child = 0xffffffff80b1a1fe <__lockmgr_args+2270>}, {rs = 0x7238d272ac0, r = 0xfffffe000897b400, child = 0xfffff80093aaa2e0}, {rs = 0x10000, r = 0xfffffe000785f6c0, child = 0xffffffff81f68822 <hash_packet+322>}, {rs = 0x0, r = 0xfffffe0008cb8b38, child = 0xfffffe000785f850}, {rs = 0xffffffff80bbd057 <witness_checkorder+919>, r = 0xfffffe000785f7d8, child = 0xfffffe008d272ac0}, {rs = 0xfffffe000785f7d8, r = 0x246, child = 0xfffffe008d272ac0}, { rs = 0xfffffe008d272ac0, r = 0xffffffff819d7f20 <w_locklistdata+248912>, child = 0xfffffe0008cb8530}, {rs = 0x60900000d34, r = 0xffffffff81f847b2, child = 0xfffffe000785f730}, { rs = 0xffffffff80b446c4 <_rm_runlock_debug+100>, r = 0xfffff800caf7c280, child = 0xfffff800caf7c180}, {rs = 0xfffffe000785f990, r = 0xfffffe0008cb8500, child = 0x0}, {rs = 0x0, r = 0xfffffe000785f960, child = 0xffffffff81f6518f <ipfw_chk+15967>}} nr = 0x0 saddr = 0xfffff800b77e583c daddr = 0xfffff800b77e5840 a = 0x0 ruleset = 0x0 nsn = 0x0 th = <optimized out> sk = 0xfffff80059271dc0 nk = 0xfffff80036adacb8 rewrite = 0 hdrlen = 8 tag = -1 asd = 0 match = 0 state_icmp = <optimized out> sport = 23689 dport = 514 bproto_sum = 0 bip_sum = 0 icmptype = <optimized out> icmpcode = 0 '\000' r = 0xfffff8001522c800 match_rules = <optimized out> ri = <optimized out> reason = <optimized out> af = <optimized out> (kgdb) up #16 0xffffffff81fb6d9b in pf_test (dir=dir@entry=2, pflags=<optimized out>, ifp=<optimized out>, m0=m0@entry=0xfffffe000785fb18, inp=0xfffff80058c69380, default_actions=default_actions@entry=0x0) at /usr/src/sys/netpfil/pf/pf.c:7680 7680 } else if (s == NULL) (kgdb) info locals __pc = <optimized out> pd = {lookup = {done = 1, uid = 0, gid = 0}, tot_len = 416, hdr = {tcp = {th_sport = 23689, th_dport = 514, th_seq = 3819670529, th_ack = 0, th_x2 = 0 '\000', th_off = 0 '\000', th_flags = 0 '\000', th_win = 0, th_sum = 0, th_urp = 0}, udp = {uh_sport = 23689, uh_dport = 514, uh_ulen = 35841, uh_sum = 58283}, sctp = {src_port = 23689, dest_port = 514, v_tag = 3819670529, checksum = 0}, icmp = { icmp_type = 137 '\211', icmp_code = 92 '\\', icmp_cksum = 514, icmp_hun = {ih_pptr = 1 '\001', ih_gwaddr = {s_addr = 3819670529}, ih_idseq = {icd_id = 35841, icd_seq = 58283}, ih_void = -475296767, ih_pmtu = {ipm_void = 35841, ipm_nextmtu = 58283}, ih_rtradv = {irt_num_addrs = 1 '\001', irt_wpa = 140 '\214', irt_lifetime = 58283}}, icmp_dun = {id_ts = {its_otime = 0, its_rtime = 0, its_ttime = 0}, id_ip = {idi_ip = {ip_hl = 0 '\000', ip_v = 0 '\000', ip_tos = 0 '\000', ip_len = 0, ip_id = 0, ip_off = 0, ip_ttl = 0 '\000', ip_p = 0 '\000', ip_sum = 0, ip_src = {s_addr = 0}, ip_dst = {s_addr = 0}}}, id_radv = {ira_addr = 0, ira_preference = 0}, id_mask = 0, id_data = ""}}, icmp6 = {icmp6_type = 137 '\211', icmp6_code = 92 '\\', icmp6_cksum = 514, icmp6_dataun = { icmp6_un_data32 = {3819670529}, icmp6_un_data16 = {35841, 58283}, icmp6_un_data8 = "\001\214\253", <incomplete sequence \343>}}, any = 0xfffffe000785f910 "\211\\\002\002\001\214\253", <incomplete sequence \343>}, nat_rule = 0x0, src = 0xfffff800b77e583c, dst = 0xfffff800b77e5840, sport = 0xfffffe000785f910, dport = 0xfffffe000785f912, pf_mtag = 0x0, act = {rtableid = -1, qid = 0, pqid = 0, max_mss = 0, log = 0 '\000', set_tos = 0 '\000', min_ttl = 0 '\000', dnpipe = 0, dnrpipe = 0, flags = 0, set_prio = "\000"}, p_len = 0, ip_sum = 0xfffff800b77e583a, proto_sum = 0x0, flags = 0, af = 2 '\002', proto = 17 '\021', tos = 0 '\000', dir = 2 '\002', sidx = 1 '\001', didx = 0 '\000', sctp_flags = 0, sctp_initiate_tag = 0} _pf_rules_tracker = {rmp_cpuQueue = {rmq_next = 0xffffffff82412090, rmq_prev = 0xffffffff82412090}, rmp_rmlock = 0xfffffe0008cb8b38, rmp_thread = 0xfffffe008d272ac0, rmp_flags = 0, rmp_qentry = { le_next = 0x1a0, le_prev = 0x0}} reason = <optimized out> m = 0xffffffff8120422b h = 0xfffff800b77e5830 a = 0x0 r = 0xfffffe0008cba420 s = 0x0 ruleset = 0x0 use_2nd_queue = 0 kif = 0xfffff80004a82d00 action = <optimized out> ipfwtag = <optimized out> off = 20 tag = <optimized out> rt = <optimized out> dirndx = <optimized out> nr = <optimized out> tr = <optimized out> (kgdb) up #17 0xffffffff81fdcb52 in pf_check_out (m=0xfffffe000785fb18, ifp=0x12, flags=128, ruleset=<optimized out>, inp=0x91170e1f0470) at /usr/src/sys/netpfil/pf/pf_ioctl.c:6560 6560 chk = pf_test(PF_OUT, flags, ifp, m, inp, NULL); (kgdb) info locals chk = <optimized out> (kgdb) up #18 0xffffffff80ca4928 in pfil_mbuf_common (pch=<optimized out>, m=0xfffffe000785fb18, m@entry=0xfffffe000785fa18, ifp=0xfffff80003931800, flags=131072, inp=0xfffff80058c69380, inp@entry=0x0) at /usr/src/sys/net/pfil.c:213 213 rv = link->link_mbuf_chk(m, ifp, flags, link->link_ruleset, (kgdb) info locals rv = <optimized out> link = 0xfffff800036e4bc0 (kgdb) up #19 pfil_mbuf_out (head=<optimized out>, m=m@entry=0xfffffe000785fb18, ifp=0xfffff80003931800, inp=inp@entry=0xfffff80058c69380) at /usr/src/sys/net/pfil.c:234 234 return (pfil_mbuf_common(&head->head_out, m, ifp, PFIL_OUT, inp)); (kgdb) info locals No locals. (kgdb) up #20 0xffffffff80d286e7 in ip_output_pfil (mp=0xfffffe000785fb18, ifp=0xfffff80003931800, flags=96, inp=0xfffff80058c69380, dst=<optimized out>, fibnum=<optimized out>, error=<optimized out>) at /usr/src/sys/netinet/ip_output.c:125 125 switch (pfil_mbuf_out(V_inet_pfil_head, mp, ifp, inp)) { (kgdb) info locals fwd_tag = 0x0 m = 0xfffff8005db0ad00 ip = 0xfffff800b77e5830 odst = {s_addr = 4094994624} (kgdb) up #21 ip_output (m=m@entry=0xfffff8005db0ad00, opt=opt@entry=0x0, ro=<optimized out>, flags=flags@entry=96, imo=0x0, inp=inp@entry=0xfffff80058c69380) at /usr/src/sys/netinet/ip_output.c:701 701 switch (ip_output_pfil(&m, ifp, flags, inp, dst, &fibnum, (kgdb) info locals iproute = {ro_nh = 0xfffff80058c693b8, ro_lle = 0xfffffe008d272ac0, ro_prepend = 0xffffffff811f8e7a "/usr/src/sys/netinet/udp_usrreq.c", ro_plen = 37792, ro_flags = 22726, ro_mtu = 63488, spare = 65535, ro_dst = {sa_len = 128 '\200', sa_family = 251 '\373', sa_data = "\205\a\000\376\377\377C\363\273\200\377\377\377\377"}} ifp = 0xfffff80003931800 hlen = 20 mtu = <optimized out> error = <optimized out> vlan_pcp = -1 ia = 0xfffff800036e7a80 no_route_but_check_spd = <optimized out> ip = 0xfffff800b77e5830 ip_len = 416 fibnum = <optimized out> gw = 0xfffff80058c694d8 dst = 0xfffff80058c694d8 isbroadcast = <optimized out> src = <optimized out> m0 = <optimized out> ip_off = <optimized out>