Kernel Panic and core dump. The box panics and reboots frequently
(perhaps 2 times daily). I found a bug report in GNATS that looks similar
to this panic (See GNATS 129316 or 127699 for example). All of these
faults seem to be similar and the bt for each looks VERY similar, though
the displayed "current process" varies from the other bug reports. This
box is running IPNAT, IPFW and several instances of OpenVPN, though the
problem appears to be related to NAT (though not specifically to IPNAT). 

KGDB BT Info:

Fatal trap 12: page fault while in kernel mode
cpuid = 0; apic id = 00
fault virtual address   = 0x4
fault code              = supervisor read, page not present
instruction pointer     = 0x20:0xc388994b
stack pointer           = 0x28:0xdceb59c8
frame pointer           = 0x28:0xdceb5a44
code segment            = base rx0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, def32 1, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 26 (irq23: vr0)
trap number             = 12
panic: page fault
cpuid = 0
Uptime: 5d7h9m18s
Physical memory: 742 MB
Dumping 119 MB: 104 88 72 56 40 24 8

#0  doadump () at pcpu.h:195
195     pcpu.h: No such file or directory.
        in pcpu.h
(kgdb) bt
#0  doadump () at pcpu.h:195
#1  0xc05ba657 in boot (howto=260) at ../../../kern/kern_shutdown.c:409
#2  0xc05ba919 in panic (fmt=Variable "fmt" is not available.
) at ../../../kern/kern_shutdown.c:563
#3  0xc080c54c in trap_fatal (frame=0xdceb5988, eva=4) at ../../../i386/i386/trap.c:899
#4  0xc080c7b0 in trap_pfault (frame=0xdceb5988, usermode=0, eva=4) at ../../../i386/i386/trap.c:812
#5  0xc080d132 in trap (frame=0xdceb5988) at ../../../i386/i386/trap.c:490
#6  0xc07f3abb in calltrap () at ../../../i386/i386/exception.s:139
#7  0xc388994b in ?? ()
Previous frame inner to this frame (corrupt stack?) 

Here's a pciconf -lv output:

hostb0@pci0:0:0:0:      class=0x060000 card=0x02041106 chip=0x02041106 rev=0x00 hdr=0x00
    vendor     = 'VIA Technologies Inc'
    device     = 'K8M400 CPU to PCI Bridge'
    class      = bridge
    subclass   = HOST-PCI
hostb1@pci0:0:0:1:      class=0x060000 card=0x00000000 chip=0x12041106 rev=0x00 hdr=0x00
    vendor     = 'VIA Technologies Inc'
    device     = 'K8M400 CPU to PCI Bridge'
    class      = bridge
    subclass   = HOST-PCI
hostb2@pci0:0:0:2:      class=0x060000 card=0x00000000 chip=0x22041106 rev=0x00 hdr=0x00
    vendor     = 'VIA Technologies Inc'
    device     = 'K8M400 CPU to PCI Bridge'
    class      = bridge
    subclass   = HOST-PCI
hostb3@pci0:0:0:3:      class=0x060000 card=0x00000000 chip=0x32041106 rev=0x00 hdr=0x00
    vendor     = 'VIA Technologies Inc'
    device     = '1394 i2c CPU to PCI Bridge'
    class      = bridge
    subclass   = HOST-PCI
hostb4@pci0:0:0:4:      class=0x060000 card=0x00000000 chip=0x42041106 rev=0x00 hdr=0x00
    vendor     = 'VIA Technologies Inc'
    device     = 'K8M400 CPU to PCI Bridge'
    class      = bridge
    subclass   = HOST-PCI
hostb5@pci0:0:0:7:      class=0x060000 card=0x00000000 chip=0x72041106 rev=0x00 hdr=0x00
    vendor     = 'VIA Technologies Inc'
    device     = 'K8M400 CPU to PCI Bridge'
    class      = bridge
    subclass   = HOST-PCI
pcib1@pci0:0:1:0:       class=0x060400 card=0x00000000 chip=0xb1881106 rev=0x00 hdr=0x01
    vendor     = 'VIA Technologies Inc'
    device     = 'VT8237 K8HTB CPU to AGP 2.0/3.0 Bridge'
    class      = bridge
    subclass   = PCI-PCI
pcib2@pci0:0:8:0:       class=0x060400 card=0x00000000 chip=0x00241011 rev=0x03 hdr=0x01
    vendor     = 'Digital Equipment Corporation'
    device     = '21151/2 PCI to PCI Bridge'
    class      = bridge
    subclass   = PCI-PCI
atapci0@pci0:0:15:0:    class=0x01018f card=0x32061565 chip=0x31491106 rev=0x80 hdr=0x00
    vendor     = 'VIA Technologies Inc'
    device     = 'VT8237  VT6410 SATA RAID Controller'
    class      = mass storage
    subclass   = ATA
atapci1@pci0:0:15:1:    class=0x01018a card=0x32061565 chip=0x05711106 rev=0x06 hdr=0x00
    vendor     = 'VIA Technologies Inc'
    device     = 'VT82C586A/B/VT82C686/A/B/VT823x/A/C Bus Master IDE Controller'
    class      = mass storage
    subclass   = ATA
isab0@pci0:0:17:0:      class=0x060100 card=0x32061565 chip=0x32271106 rev=0x00 hdr=0x00
    vendor     = 'VIA Technologies Inc'
    device     = 'VT8237 PCI-to-ISA Bridge'
    class      = bridge
    subclass   = PCI-ISA
vr0@pci0:0:18:0:        class=0x020000 card=0x01021106 chip=0x30651106 rev=0x78 hdr=0x00
    vendor     = 'VIA Technologies Inc'
    device     = 'VT6102 Rhine II PCI Fast Ethernet Controller||Used by GERICOM in laptop Webengine Advanced'
    class      = network
    subclass   = ethernet
hostb6@pci0:0:24:0:     class=0x060000 card=0x00000000 chip=0x11001022 rev=0x00 hdr=0x00
    vendor     = 'Advanced Micro Devices (AMD)'
    device     = '(K8) Athlon 64/Opteron HyperTransport Technology Configuration'
    class      = bridge
    subclass   = HOST-PCI
hostb7@pci0:0:24:1:     class=0x060000 card=0x00000000 chip=0x11011022 rev=0x00 hdr=0x00
    vendor     = 'Advanced Micro Devices (AMD)'
    device     = '(K8) Athlon 64/Opteron Address Map'
    class      = bridge
    subclass   = HOST-PCI
hostb8@pci0:0:24:2:     class=0x060000 card=0x00000000 chip=0x11021022 rev=0x00 hdr=0x00
    vendor     = 'Advanced Micro Devices (AMD)'
    device     = '(K8) Athlon 64/Opteron DRAM Controller'
    class      = bridge
    subclass   = HOST-PCI
hostb9@pci0:0:24:3:     class=0x060000 card=0x00000000 chip=0x11031022 rev=0x00 hdr=0x00
    vendor     = 'Advanced Micro Devices (AMD)'
    device     = '(K8) Athlon 64/Opteron Miscellaneous Control'
    class      = bridge
    subclass   = HOST-PCI
vgapci0@pci0:1:0:0:     class=0x030000 card=0x12031565 chip=0x31081106 rev=0x01 hdr=0x00
    vendor     = 'VIA Technologies Inc'
    device     = 'Unknown Via Unichrome S3 VGA - part of a VIA Northbridge'
    class      = display
    subclass   = VGA
de0@pci0:2:4:0: class=0x020000 card=0x24001109 chip=0x00091011 rev=0x22 hdr=0x00
    vendor     = 'Digital Equipment Corporation'
    device     = 'DecChip 21140 Fast Ethernet Adapter'
    class      = network
    subclass   = ethernet
de1@pci0:2:5:0: class=0x020000 card=0x00000000 chip=0x00091011 rev=0x22 hdr=0x00
    vendor     = 'Digital Equipment Corporation'
    device     = 'DecChip 21140 Fast Ethernet Adapter'
    class      = network
    subclass   = ethernet
de2@pci0:2:6:0: class=0x020000 card=0x00000000 chip=0x00091011 rev=0x22 hdr=0x00
    vendor     = 'Digital Equipment Corporation'
    device     = 'DecChip 21140 Fast Ethernet Adapter'
    class      = network
    subclass   = ethernet
de3@pci0:2:7:0: class=0x020000 card=0x00000000 chip=0x00091011 rev=0x22 hdr=0x00
    vendor     = 'Digital Equipment Corporation'
    device     = 'DecChip 21140 Fast Ethernet Adapter'
    class      = network
    subclass   = ethernet 

I've tried this configuration on Intel and AMD processors, and with
multiple makers of NIC's to eliminate the possibility of a hardware
problem. The system panics in the same manner on all hardware tested.

How-To-Repeat: Configure IPFW, IPNAT and OpenVPN (bridged not routed VPN) and two
subnets behind the box. In this case, the "current process" as reported
by BT is an IRQ related request to the vr0 integrated NIC, though this
is repeatable with all NIC's I've tested (DEC, Intel, RealTek, and Broadcom).