Sometimes (1 crash per 2 weeks or even more) machine panics with: Fatal trap 12: page fault while in kernel mode cpuid = 3; apic id = 03 fault virtual address = 0x4 fault code = supervisor read data, page not present instruction pointer = 0x8:0xffffffffb2f3a316 stack pointer = 0x10:0xffffffffb0a28220 frame pointer = 0x10:0xffffffffb0a28270 code segment = base rx0, limit 0xfffff, type 0x1b = DPL 0, pres 1, long 1, def32 0, gran 1 processor eflags = interrupt enabled, resume, IOPL = 0 current process = 37 (mskc1 taskq) trap number = 12 panic: page fault cpuid = 3 Uptime: 20d13h12m43s Physical memory: 4084 MB Dumping 777 MB: 762 746 730 714 698 682 666 650 634 618 602 586 570 554 538 522 506 490 474 458 442 426 410 394 378 362 346 330 314 298 282 266 250 234 218 202 186 170 154 138 122 106 90 74 58 42 26 1 0 backtrace: #0 doadump () at ../../../kern/kern_shutdown.c:244 244 dumptid = curthread->td_tid; (kgdb) bt #0 doadump () at ../../../kern/kern_shutdown.c:244 #1 0xffffffff803908be in boot (howto=260) at ../../../kern/kern_shutdown.c:418 #2 0xffffffff80390e0d in panic (fmt=Could not find the frame base for "panic". ) at ../../../kern/kern_shutdown.c:574 #3 0xffffffff806d8892 in trap_fatal (frame=0xffffffffb0a28170, eva=4) at ../../../amd64/amd64/trap.c:764 #4 0xffffffff806d8342 in trap_pfault (frame=0xffffffffb0a28170, usermode=0) at ../../../amd64/amd64/trap.c:680 #5 0xffffffff806d7d20 in trap (frame=0xffffffffb0a28170) at ../../../amd64/amd64/trap.c:449 #6 0xffffffff806b73ee in calltrap () at ../../../amd64/amd64/exception.S:209 #7 0xffffffffb2f3a316 in nat_finalise (fin=0xffffffffb0a28440, nat=0xffffff002502da00, ni=0xffffffffb0a282b0, tcp=0x0, natsave=0x0, direction=0) at /usr/src/sys/modules/ipfilter/../../contrib/ipfilter/netinet/ip_nat.c:2577 #8 0xffffffffb2f3a11d in nat_new () from /boot/kernel/ipl.ko #9 0xffffffffb2f3d53a in fr_checknatin (fin=0xffffffffb0a28440, passp=0xffffffffb0a2843c) at /usr/src/sys/modules/ipfilter/../../contrib/ipfilter/netinet/ip_nat.c:4122 #10 0xffffffffb2f5c822 in fr_check (ip=0xffffff004f583810, hlen=20, ifp=0xffffff0003370800, out=0, mp=0xffffffffb0a285c8) at /usr/src/sys/modules/ipfilter/../../contrib/ipfilter/netinet/fil.c:2572 #11 0xffffffffb2f56ec8 in fr_check_wrapper (arg=0x0, mp=0xffffffffb0a285c8, ifp=0xffffff0003370800, dir=1) at /usr/src/sys/modules/ipfilter/../../contrib/ipfilter/netinet/ip_fil_freebsd.c:178 #12 0xffffffff8047ae88 in pfil_run_hooks (ph=0xffffffff80928320, mp=0xffffffffb0a28608, ifp=0xffffff0003370800, dir=1, inp=0x0) at ../../../net/pfil.c:78 #13 0xffffffff804b0cae in ip_input (m=0xffffff00105be300) at ../../../netinet/ip_input.c:417 #14 0xffffffff8047891c in netisr_dispatch (num=2, m=0xffffff00105be300) at ../../../net/netisr.c:185 #15 0xffffffff8046d0b7 in ether_demux (ifp=0xffffff0003370800, m=0xffffff00105be300) at ../../../net/if_ethersubr.c:834 #16 0xffffffffb30f50a6 in ng_ether_rcv_upper (node=0xffffff0009f8b100, m=0xffffff00105be300) at /usr/src/sys/modules/netgraph/ether/../../../netgraph/ng_ether.c:664 #17 0xffffffffb30f4e02 in ng_ether_rcvdata (hook=0xffffff00097f3e00, item=0xffffff008569a690) at /usr/src/sys/modules/netgraph/ether/../../../netgraph/ng_ether.c:586 #18 0xffffffffb30ea8be in ng_apply_item (node=0xffffff0009f8b100, item=0xffffff008569a690, rw=0) at /usr/src/sys/modules/netgraph/netgraph/../../../netgraph/ng_base.c:2331 #19 0xffffffffb30ea446 in ng_snd_item (item=0xffffff008569a690, flags=0) at /usr/src/sys/modules/netgraph/netgraph/../../../netgraph/ng_base.c:2249 #20 0xffffffffb30f86ef in ng_tee_rcvdata (hook=0xffffff00097f4080, item=0xffffff008569a690) at /usr/src/sys/modules/netgraph/tee/../../../netgraph/ng_tee.c:326 #21 0xffffffffb30ea8be in ng_apply_item (node=0xffffff003b38f000, item=0xffffff008569a690, rw=0) at /usr/src/sys/modules/netgraph/netgraph/../../../netgraph/ng_base.c:2331 #22 0xffffffffb30ea446 in ng_snd_item (item=0xffffff008569a690, flags=0) at /usr/src/sys/modules/netgraph/netgraph/../../../netgraph/ng_base.c:2249 #23 0xffffffffb30f4087 in ng_ether_input () from /boot/kernel/ng_ether.ko #24 0xffffffff8046cd7a in ether_input (ifp=0xffffff0003370800, m=0xffffff00105be300) at ../../../net/if_ethersubr.c:643 #25 0xffffffff802849af in msk_rxeof (sc_if=0xffffffff80c67000, status=3932416, len=60) at ../../../dev/msk/if_msk.c:2966 #26 0xffffffff80285934 in msk_handle_events (sc=0xffffff0003348600) at ../../../dev/msk/if_msk.c:3341 #27 0xffffffff802862e5 in msk_int_task (arg=0xffffff0003348600, pending=1) at ../../../dev/msk/if_msk.c:3523 #28 0xffffffff803daa33 in taskqueue_run (queue=0xffffff0005c09e00) at ../../../kern/subr_taskqueue.c:282 #29 0xffffffff803db0e1 in taskqueue_thread_loop (arg=0xffffff00033486d8) at ../../../kern/subr_taskqueue.c:401 #30 0xffffffff80360f72 in fork_exit (callout=0xffffffff803db0b0 <taskqueue_thread_loop>, arg=0xffffff00033486d8, frame=0xffffffffb0a28c80) at ../../../kern/kern_fork.c:804 #31 0xffffffff806b77be in fork_trampoline () at ../../../amd64/amd64/exception.S:455 #32 0x0000000000000000 in ?? () #33 0x0000000000000000 in ?? () #34 0x0000000000000001 in ?? () #35 0x0000000000000000 in ?? () #36 0x0000000000000000 in ?? () #37 0x0000000000000000 in ?? () #38 0x0000000000000000 in ?? () #39 0x0000000000000000 in ?? () #40 0x0000000000000000 in ?? () #41 0x0000000000000000 in ?? () #42 0x0000000000000000 in ?? () #43 0x0000000000000000 in ?? () #44 0x0000000000000000 in ?? () #45 0x0000000000000000 in ?? () #46 0x0000000000000000 in ?? () #47 0x0000000000000000 in ?? () #48 0x0000000000000000 in ?? () #49 0x0000000000000000 in ?? () #50 0x0000000000000000 in ?? () #51 0x0000000000000000 in ?? () #52 0x0000000000000000 in ?? () #53 0x0000000000000000 in ?? () #54 0x0000000000000000 in ?? () #55 0x0000000000000000 in ?? () #56 0x0000000000bcf000 in ?? () #57 0x0000000000000000 in ?? () #58 0x0000000000000000 in ?? () #59 0x0000000000000000 in ?? () #60 0xffffffff803db0b0 in taskqueue_start_threads () at ../../../kern/subr_taskqueue.c:390 (kgdb) list *0xffffffffb2f3a316 0xffffffffb2f3a316 is in nat_finalise (/usr/src/sys/modules/ipfilter/../../contrib/ipfilter/netinet/ ip_nat.c:2577). 2572 nat->nat_ifps[1] = np->in_ifps[1]; 2573 nat->nat_ptr = np; 2574 nat->nat_p = fin->fin_p; 2575 nat->nat_mssclamp = np->in_mssclamp; 2576 if (nat->nat_p == IPPROTO_TCP) 2577 nat->nat_seqnext[0] = ntohl(tcp->th_seq); 2578 2579 if ((np->in_apr != NULL) && ((ni->nai_flags & NAT_SLAVE) == 0)) 2580 if (appr_new(fin, nat) == -1) 2581 return -1; Coredump is available by request Fix: Unknown How-To-Repeat: Floating bug, can't repeat
(kgdb) frame 7 #7 0xffffffffb2f3a316 in nat_finalise (fin=0xffffffffb0a28440, nat=0xffffff002502da00, ni=0xffffffffb0a282b0, tcp=0x0, natsave=0x0, direction=0) at /usr/src/sys/modules/ipfilter/../../contrib/ipfilter/netinet/ip_nat.c:2577 2577 nat->nat_seqnext[0] = ntohl(tcp->th_seq); (kgdb) p nat->nat_seqnext $1 = {0, 0} (kgdb) p tcp $2 = (tcphdr_t *) 0x0 (kgdb) p tcp->th_seq Cannot access memory at address 0x4 --- Best regards, Vladimir
Responsible Changed From-To: freebsd-amd64->freebsd-net Over to maintainer(s). PR has a full backtrace and submitter has a core file for further investigation.
Responsible Changed From-To: freebsd-net->freebsd-ipfw Over to maintainer(s).
Quick fix, tested, no panic. apply in /sys/contrib/ipfilter/netinet --- ip_nat.c.std 2007-10-31 12:00:38.000000000 +0700 +++ ip_nat.c 2009-02-19 10:20:05.000000000 +0700 @@ -2552,6 +2552,10 @@ { frentry_t *fr; ipnat_t *np; + + if (fin->fin_p == IPPROTO_TCP && tcp == NULL) { + return -1; + } np = ni->nai_np; --- Best regards, Vladimir
FreeBSD 7.2 - i386 # kgdb kernel.debug /var/crash/vmcore.0 GNU gdb 6.1.1 [FreeBSD] Copyright 2004 Free Software Foundation, Inc. GDB is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Type "show copying" to see the conditions. There is absolutely no warranty for GDB. Type "show warranty" for details. This GDB was configured as "i386-marcel-freebsd"... Unread portion of the kernel message buffer: Fatal trap 12: page fault while in kernel mode cpuid = 2; apic id = 06 fault virtual address = 0x4 fault code = supervisor read, page not present instruction pointer = 0x20:0xc5abe98b stack pointer = 0x28:0xe58109ac frame pointer = 0x28:0xe5810a28 code segment = base rx0, limit 0xfffff, type 0x1b = DPL 0, pres 1, def32 1, gran 1 processor eflags = interrupt enabled, resume, IOPL = 0 current process = 30 (em1 taskq) trap number = 12 panic: page fault cpuid = 2 Uptime: 1d3h25m11s Physical memory: 2035 MB Dumping 209 MB: 194 178 162 146 130 114 98 82 66 50 34 18 2 Reading symbols from /boot/kernel/acpi.ko...Reading symbols from /boot/kernel/acpi.ko.symbols...done. done. Loaded symbols for /boot/kernel/acpi.ko Reading symbols from /boot/kernel/ipl.ko...Reading symbols from /boot/kernel/ipl.ko.symbols...done. done. Loaded symbols for /boot/kernel/ipl.ko #0 doadump () at pcpu.h:196 196 __asm __volatile("movl %%fs:0,%0" : "=r" (td)); (kgdb) list *0xc5abe98b 0xc5abe98b is in nat_new (/usr/src/sys/modules/ipfilter/../../contrib/ipfilter/netinet/ip_nat.c:2577) . 2572 nat->nat_ifps[1] = np->in_ifps[1]; 2573 nat->nat_ptr = np; 2574 nat->nat_p = fin->fin_p; 2575 nat->nat_mssclamp = np->in_mssclamp; 2576 if (nat->nat_p == IPPROTO_TCP) 2577 nat->nat_seqnext[0] = ntohl(tcp->th_seq); 2578 2579 if ((np->in_apr != NULL) && ((ni->nai_flags & NAT_SLAVE) == 0)) 2580 if (appr_new(fin, nat) == -1) 2581 return -1; (kgdb) backtrace #0 doadump () at pcpu.h:196 #1 0xc07dfa4f in boot (howto=260) at ../../../kern/kern_shutdown.c:418 #2 0xc07dfd32 in panic (fmt=Variable "fmt" is not available. ) at ../../../kern/kern_shutdown.c:574 #3 0xc0ae8573 in trap_fatal (frame=0xe581096c, eva=4) at ../../../i386/i386/trap.c:939 #4 0xc0ae8763 in trap_pfault (frame=0xe581096c, usermode=0, eva=4) at ../../../i386/i386/trap.c:852 #5 0xc0ae90e8 in trap (frame=0xe581096c) at ../../../i386/i386/trap.c:530 #6 0xc0acd16b in calltrap () at ../../../i386/i386/exception.s:159 #7 0xc5abe98b in nat_new (fin=0xe5810a84, np=0xc5b13400, natsave=0x0, flags=Variable "flags" is not available. ) at /usr/src/sys/modules/ipfilter/../../contrib/ipfilter/netinet/ip_nat.c:2577 #8 0xc5ac2654 in fr_checknatin (fin=0xe5810a84, passp=0xe5810b30) at /usr/src/sys/modules/ipfilter/../../contrib/ipfilter/netinet/ip_nat.c:4122 #9 0xc5adb833 in fr_check (ip=0xc5ba5010, hlen=20, ifp=0xc567a800, out=0, mp=0xe5810b7c) at /usr/src/sys/modules/ipfilter/../../contrib/ipfilter/netinet/fil.c:2572 #10 0xc5ad37ee in fr_check_wrapper (arg=0x0, mp=0xe5810b7c, ifp=0xc567a800, dir=1) at /usr/src/sys/modules/ipfilter/../../contrib/ipfilter/netinet/ip_fil_freebsd. c:178 #11 0xc08894a8 in pfil_run_hooks (ph=0xc0ce5580, mp=0xe5810bcc, ifp=0xc567a800, dir=1, inp=0x0) at ../../../net/pfil.c:78 #12 0xc08cf801 in ip_input (m=0xc6c8de00) at ../../../netinet/ip_input.c:416 #13 0xc0887903 in netisr_dispatch (num=2, m=0xc6c8de00) at ../../../net/netisr.c:185 #14 0xc087b9c1 in ether_demux (ifp=0xc567a800, m=0xc6c8de00) at ../../../net/if_ethersubr.c:834 #15 0xc087be2f in ether_input (ifp=0xc567a800, m=0xc6c8de00) at ../../../net/if_ethersubr.c:692 #16 0xc05bf099 in em_rxeof (adapter=0xc567d000, count=99) at ../../../dev/e1000/if_em.c:4539 #17 0xc05bf21e in em_handle_rxtx (context=0xc567d000, pending=1) at ../../../dev/e1000/if_em.c:1702 ---Type <return> to continue, or q <return> to quit--- #18 0xc0815eab in taskqueue_run (queue=0xc566c480) at ../../../kern/subr_taskqueue.c:282 #19 0xc0816008 in taskqueue_thread_loop (arg=0xc568135c) at ../../../kern/subr_taskqueue.c:401 #20 0xc07bc298 in fork_exit (callout=0xc0815fa0 <taskqueue_thread_loop>, arg=0xc568135c, frame=0xe5810d38) at ../../../kern/kern_fork.c:810 #21 0xc0acd1e0 in fork_trampoline () at ../../../i386/i386/exception.s:264
use my patch, it works also, these who using ipnat ftp proxy, need to apply this patch. --- ip_ftp_pxy.c.orig 2007-06-04 10:54:35.000000000 +0800 +++ ip_ftp_pxy.c 2009-05-08 08:48:04.000000000 +0800 @@ -1010,6 +1010,8 @@ return 0; } else if (mlen < 0) { return 0; + } else if (nat->nat_aps == NULL) { + return 0; } aps = nat->nat_aps; --- Best regards, Vladimir
Responsible Changed From-To: freebsd-ipfw->freebsd-net Reassign to freebsd-net@.
Responsible Changed From-To: freebsd-net->cy Mine.
For bugs matching the following criteria: Status: In Progress Changed: (is less than) 2014-06-01 Reset to default assignee and clear in-progress tags. Mail being skipped
This no longer applies to FreeBSD 10+.