Bug 194225

On a server running 8.4-STABLE #0 r268802 i386 I got the following double fault and need help to debug this, because I like to know the reason (hardware or software ?). The server runs FreeBSD for many years without any problems:

Fatal double fault:
eip = 0xc0910b45
esp = 0xc75cbc30
ebp = 0xc75cbc30
cpuid = 1; apic id = 01
kernel trap 12 with interrupts disabled

Fatal trap 12: page fault while in kernel mode
cpuid = 2; apic id = 06
fault virtual address   = 0x0
fault code              = supervisor write, page not present
instruction pointer     = 0x20:0xc092fd4e
stack pointer           = 0x28:0xea85c7d8
frame pointer           = 0x28:0xea85c7e0
code segment            = base rx0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, def32 1, gran 1
processor eflags        = resume, IOPL = 0
current process         = 20528 (sh)
timeout stopping cpus
[thread pid 20528 tid 100522 ]
Stopped at      bcopy+0x1a:     repe movsl      (%esi),%es:(%edi)
db:0:kdb.enter.default> watchdog
No argument provided, disabling watchdog
db:0:kdb.enter.default>  run ddbinfo
db:1:ddbinfo> capture on
db:1:on>  run lockinfo
db:2:lockinfo> show lock Giant
 class: sleep mutex
 name: Giant
 flags: {DEF, RECURSE}
 state: {UNOWNED}
db:2:Giant>  show lockedvnods
Locked vnodes
db:2:lockedvnods>  show lockchain
thread 100522 (pid 20528, sh) running on CPU 2
db:2:lockchain>  show sleepchain
thread 100522 (pid 20528, sh) running on CPU 2
db:1:sleepchain>  show pcpu
cpuid        = 2
dynamic pcpu = 0x6b71200
curthread    = 0xcafb3b80: pid 20528 "sh"
curpcb       = 0xea85cd80
fpcurthread  = none
idlethread   = 0xc79355c0: tid 100004 "idle: cpu2"
APIC ID      = 6
currentldt   = 0x50
db:1:pcpu>  show allpcpu
Current CPU: 2

cpuid        = 0
dynamic pcpu = 0x1df200
curthread    = 0xcb825000: pid 20527 "tifftopnm"
curpcb       = 0xeac84d80
fpcurthread  = none
idlethread   = 0xc7935000: tid 100006 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x50

cpuid        = 1
dynamic pcpu = 0x6b6e200
curthread    = 0xc79352e0: pid 11 "idle: cpu1"
curpcb       = 0xc75cbd80
fpcurthread  = none
idlethread   = 0xc79352e0: tid 100005 "idle: cpu1"
APIC ID      = 1
currentldt   = 0x50

cpuid        = 2
dynamic pcpu = 0x6b71200
curthread    = 0xcafb3b80: pid 20528 "sh"
curpcb       = 0xea85cd80
fpcurthread  = none
idlethread   = 0xc79355c0: tid 100004 "idle: cpu2"
APIC ID      = 6
currentldt   = 0x50

cpuid        = 3
dynamic pcpu = 0x6b74200
curthread    = 0xc79358a0: pid 11 "idle: cpu3"
curpcb       = 0xc75c5d80
fpcurthread  = none
idlethread   = 0xc79358a0: tid 100003 "idle: cpu3"
APIC ID      = 7
currentldt   = 0x50
db:1:allpcpu>  bt
Tracing pid 20528 tid 100522 td 0xcafb3b80
bcopy(ea85cdc0,0,200) at bcopy+0x1a
savectx(4,ea85c8a8,c09328b6,cafb3b80,50,...) at savectx+0x63
ipi_nmi_handler(cafb3b80,50,33,0,cf52b000,...) at ipi_nmi_handler+0x2f
trap(ea85c8b4) at trap+0x36
calltrap() at calltrap+0x6
--- trap 0x13, eip = 0xc0927bb2, esp = 0xea85c8f4, ebp = 0xea85c91c ---
smp_tlb_shootdown(ea85c944,c09299bf,c5e6f000,c5e70000,0,...) at smp_tlb_shootdown+0xd2
smp_invlpg_range(c5e6f000,c5e70000,0,ea85c964,1,...) at smp_invlpg_range+0x1c
pmap_invalidate_range(c0adb8a0,c5e6f000,c5e70000) at pmap_invalidate_range+0x4f
pmap_qremove(c5e6f000,1,c06ed30a,c8261d9c,cafb3b80,...) at pmap_qremove+0x58
pmap_remove_pages(cce9b0b0,cf52b000,ea85cbb4,0,c0a1fbc0,...) at pmap_remove_pages+0x410
exec_new_vmspace(ea85cbb4,c0a31c20,8,c826bd48,80,...) at exec_new_vmspace+0x1b0
exec_elf32_imgact(ea85cbb4,ea85cbfc,c09b88e7,cafb3b80,50,...) at exec_elf32_imgact+0x48e
kern_execve(cafb3b80,ea85cc48,0,883024b4,8830250c,e4c17000,e4c17000,e4c170b3,e4c17264,e4c57400,3fd9c,8,e,0) at kern_execve+0x541
execve(cafb3b80,ea85ccec,c,c,c,...) at execve+0x4c
syscall(ea85cd28) at syscall+0x342
Xint0x80_syscall() at Xint0x80_syscall+0x21
--- syscall (59, FreeBSD ELF32, execve), eip = 0x88169c2b, esp = 0xbfbfe9bc, ebp = 0xbfbfe9d8 ---

db:1:bt>  ps
  pid  ppid  pgrp   uid   state   wmesg     wchan    cmd
20528 29408 62482   993  R       CPU 2               sh
20527 20526  3552   993  RL      CPU 0               tifftopnm
20526  3749  3552   993  S       wait     0xca80b560 initial thread
19983  3099    26     0  S       nanslp   0xc0a77c04 sleep
20578  2917  2917   125  S       kqread   0xca258180 initial thread
 3749  3552  3552   993  S       wait     0xcd17e560 sh
 3552  3550  3552   993  Ss      wait     0xc8607810 sh
.................

db:1:ps>  show thread
Thread 100522 at 0xcafb3b80:
 proc (pid 20528): 0xcf52b000
 name: sh
 stack: 0xea85b000-0xea85cfff
 flags: 0x4  pflags: 0
 state: RUNNING (CPU 2)
 priority: 180
 container lock: sched lock 2 (0xc0a7c900)
db:1:thread>  alltrace

Tracing command sh pid 20528 tid 100522 td 0xcafb3b80
bcopy(ea85cdc0,0,200) at bcopy+0x1a
savectx(4,ea85c8a8,c09328b6,cafb3b80,50,...) at savectx+0x63
ipi_nmi_handler(cafb3b80,50,33,0,cf52b000,...) at ipi_nmi_handler+0x2f
trap(ea85c8b4) at trap+0x36
calltrap() at calltrap+0x6
--- trap 0x13, eip = 0xc0927bb2, esp = 0xea85c8f4, ebp = 0xea85c91c ---
smp_tlb_shootdown(ea85c944,c09299bf,c5e6f000,c5e70000,0,...) at smp_tlb_shootdown+0xd2
smp_invlpg_range(c5e6f000,c5e70000,0,ea85c964,1,...) at smp_invlpg_range+0x1c
pmap_invalidate_range(c0adb8a0,c5e6f000,c5e70000) at pmap_invalidate_range+0x4f
pmap_qremove(c5e6f000,1,c06ed30a,c8261d9c,cafb3b80,...) at pmap_qremove+0x58
pmap_remove_pages(cce9b0b0,cf52b000,ea85cbb4,0,c0a1fbc0,...) at pmap_remove_pages+0x410
exec_new_vmspace(ea85cbb4,c0a31c20,8,c826bd48,80,...) at exec_new_vmspace+0x1b0
exec_elf32_imgact(ea85cbb4,ea85cbfc,c09b88e7,cafb3b80,50,...) at exec_elf32_imgact+0x48e
kern_execve(cafb3b80,ea85cc48,0,883024b4,8830250c,e4c17000,e4c17000,e4c170b3,e4c17264,e4c57400,3fd9c,8,e,0) at kern_execve+0x541
execve(cafb3b80,ea85ccec,c,c,c,...) at execve+0x4c
syscall(ea85cd28) at syscall+0x342
Xint0x80_syscall() at Xint0x80_syscall+0x21
--- syscall (59, FreeBSD ELF32, execve), eip = 0x88169c2b, esp = 0xbfbfe9bc, ebp = 0xbfbfe9d8 ---

Tracing command tifftopnm pid 20527 tid 100845 td 0xcb825000
cpustop_handler(1,eac849fc,c09328b6,1,eac849a8,...) at cpustop_handler+0x34
ipi_nmi_handler(1,eac849a8,c062a16b,c7bca000,cb1d6560,...) at ipi_nmi_handler+0x2f
trap(eac84a08) at trap+0x36
calltrap() at calltrap+0x6
--- trap 0x13, eip = 0xc06ecd99, esp = 0xeac84a48, ebp = 0xeac84a60 ---
_mtx_lock_sleep(c0a94ce4,cb825000,0,0,0,...) at _mtx_lock_sleep+0x79
pmap_enter(ca507198,88326000,2,c28a2120,3,...) at pmap_enter+0x66
vm_fault(ca5070e8,88326000,2,8,eac84c70,...) at vm_fault+0x1c14
trap_pfault(0,eac84cc8,c062a16b,c7bca000,cb1d6560,...) at trap_pfault+0x1ce
trap(eac84d28) at trap+0x263
calltrap() at calltrap+0x6
--- trap 0xc, eip = 0x880d5cdd, esp = 0xbfbfb640, ebp = 0xbfbfb698 ---

Tracing command perl5.14.2 pid 20526 tid 100278 td 0xcbc84b80
sched_switch(cbc84b80,0,104,3b38c51a,2123d7,...) at sched_switch+0x297
mi_switch(104,0,15c,ca80b560,ea3a5b70,...) at mi_switch+0x12f
sleepq_switch(cbc84b80,0,c09c15c1,1a3,cbc84b80,...) at sleepq_switch+0xcc
sleepq_catch_signals(15c,0,ea3a5bc4,c07073bc,ca80b560,...) at sleepq_catch_signals+0x52
sleepq_wait_sig(ca80b560,5c,c09c1fa4,100,0,...) at sleepq_wait_sig+0x18
_sleep(ca80b560,ca80b5e8,15c,c09c1fa4,0,...) at _sleep+0x2bc
kern_wait(cbc84b80,502f,ea3a5c64,0,0,...) at kern_wait+0xfa1
wait4(cbc84b80,ea3a5cec,c,c,c,...) at wait4+0x3b
syscall(ea3a5d28) at syscall+0x342
Xint0x80_syscall() at Xint0x80_syscall+0x21
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x882a1c6b, esp = 0xbfbfeb2c, ebp = 0xbfbfeb48 ---

I can give more information from ddb output and or the written kerneldump.