On a server running 8.4-STABLE #0 r268802 i386 I got the following double fault and need help to debug this, because I like to know the reason (hardware or software ?). The server runs FreeBSD for many years without any problems: Fatal double fault: eip = 0xc0910b45 esp = 0xc75cbc30 ebp = 0xc75cbc30 cpuid = 1; apic id = 01 kernel trap 12 with interrupts disabled Fatal trap 12: page fault while in kernel mode cpuid = 2; apic id = 06 fault virtual address = 0x0 fault code = supervisor write, page not present instruction pointer = 0x20:0xc092fd4e stack pointer = 0x28:0xea85c7d8 frame pointer = 0x28:0xea85c7e0 code segment = base rx0, limit 0xfffff, type 0x1b = DPL 0, pres 1, def32 1, gran 1 processor eflags = resume, IOPL = 0 current process = 20528 (sh) timeout stopping cpus [thread pid 20528 tid 100522 ] Stopped at bcopy+0x1a: repe movsl (%esi),%es:(%edi) db:0:kdb.enter.default> watchdog No argument provided, disabling watchdog db:0:kdb.enter.default> run ddbinfo db:1:ddbinfo> capture on db:1:on> run lockinfo db:2:lockinfo> show lock Giant class: sleep mutex name: Giant flags: {DEF, RECURSE} state: {UNOWNED} db:2:Giant> show lockedvnods Locked vnodes db:2:lockedvnods> show lockchain thread 100522 (pid 20528, sh) running on CPU 2 db:2:lockchain> show sleepchain thread 100522 (pid 20528, sh) running on CPU 2 db:1:sleepchain> show pcpu cpuid = 2 dynamic pcpu = 0x6b71200 curthread = 0xcafb3b80: pid 20528 "sh" curpcb = 0xea85cd80 fpcurthread = none idlethread = 0xc79355c0: tid 100004 "idle: cpu2" APIC ID = 6 currentldt = 0x50 db:1:pcpu> show allpcpu Current CPU: 2 cpuid = 0 dynamic pcpu = 0x1df200 curthread = 0xcb825000: pid 20527 "tifftopnm" curpcb = 0xeac84d80 fpcurthread = none idlethread = 0xc7935000: tid 100006 "idle: cpu0" APIC ID = 0 currentldt = 0x50 cpuid = 1 dynamic pcpu = 0x6b6e200 curthread = 0xc79352e0: pid 11 "idle: cpu1" curpcb = 0xc75cbd80 fpcurthread = none idlethread = 0xc79352e0: tid 100005 "idle: cpu1" APIC ID = 1 currentldt = 0x50 cpuid = 2 dynamic pcpu = 0x6b71200 curthread = 0xcafb3b80: pid 20528 "sh" curpcb = 0xea85cd80 fpcurthread = none idlethread = 0xc79355c0: tid 100004 "idle: cpu2" APIC ID = 6 currentldt = 0x50 cpuid = 3 dynamic pcpu = 0x6b74200 curthread = 0xc79358a0: pid 11 "idle: cpu3" curpcb = 0xc75c5d80 fpcurthread = none idlethread = 0xc79358a0: tid 100003 "idle: cpu3" APIC ID = 7 currentldt = 0x50 db:1:allpcpu> bt Tracing pid 20528 tid 100522 td 0xcafb3b80 bcopy(ea85cdc0,0,200) at bcopy+0x1a savectx(4,ea85c8a8,c09328b6,cafb3b80,50,...) at savectx+0x63 ipi_nmi_handler(cafb3b80,50,33,0,cf52b000,...) at ipi_nmi_handler+0x2f trap(ea85c8b4) at trap+0x36 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0927bb2, esp = 0xea85c8f4, ebp = 0xea85c91c --- smp_tlb_shootdown(ea85c944,c09299bf,c5e6f000,c5e70000,0,...) at smp_tlb_shootdown+0xd2 smp_invlpg_range(c5e6f000,c5e70000,0,ea85c964,1,...) at smp_invlpg_range+0x1c pmap_invalidate_range(c0adb8a0,c5e6f000,c5e70000) at pmap_invalidate_range+0x4f pmap_qremove(c5e6f000,1,c06ed30a,c8261d9c,cafb3b80,...) at pmap_qremove+0x58 pmap_remove_pages(cce9b0b0,cf52b000,ea85cbb4,0,c0a1fbc0,...) at pmap_remove_pages+0x410 exec_new_vmspace(ea85cbb4,c0a31c20,8,c826bd48,80,...) at exec_new_vmspace+0x1b0 exec_elf32_imgact(ea85cbb4,ea85cbfc,c09b88e7,cafb3b80,50,...) at exec_elf32_imgact+0x48e kern_execve(cafb3b80,ea85cc48,0,883024b4,8830250c,e4c17000,e4c17000,e4c170b3,e4c17264,e4c57400,3fd9c,8,e,0) at kern_execve+0x541 execve(cafb3b80,ea85ccec,c,c,c,...) at execve+0x4c syscall(ea85cd28) at syscall+0x342 Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (59, FreeBSD ELF32, execve), eip = 0x88169c2b, esp = 0xbfbfe9bc, ebp = 0xbfbfe9d8 --- db:1:bt> ps pid ppid pgrp uid state wmesg wchan cmd 20528 29408 62482 993 R CPU 2 sh 20527 20526 3552 993 RL CPU 0 tifftopnm 20526 3749 3552 993 S wait 0xca80b560 initial thread 19983 3099 26 0 S nanslp 0xc0a77c04 sleep 20578 2917 2917 125 S kqread 0xca258180 initial thread 3749 3552 3552 993 S wait 0xcd17e560 sh 3552 3550 3552 993 Ss wait 0xc8607810 sh ................. db:1:ps> show thread Thread 100522 at 0xcafb3b80: proc (pid 20528): 0xcf52b000 name: sh stack: 0xea85b000-0xea85cfff flags: 0x4 pflags: 0 state: RUNNING (CPU 2) priority: 180 container lock: sched lock 2 (0xc0a7c900) db:1:thread> alltrace Tracing command sh pid 20528 tid 100522 td 0xcafb3b80 bcopy(ea85cdc0,0,200) at bcopy+0x1a savectx(4,ea85c8a8,c09328b6,cafb3b80,50,...) at savectx+0x63 ipi_nmi_handler(cafb3b80,50,33,0,cf52b000,...) at ipi_nmi_handler+0x2f trap(ea85c8b4) at trap+0x36 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc0927bb2, esp = 0xea85c8f4, ebp = 0xea85c91c --- smp_tlb_shootdown(ea85c944,c09299bf,c5e6f000,c5e70000,0,...) at smp_tlb_shootdown+0xd2 smp_invlpg_range(c5e6f000,c5e70000,0,ea85c964,1,...) at smp_invlpg_range+0x1c pmap_invalidate_range(c0adb8a0,c5e6f000,c5e70000) at pmap_invalidate_range+0x4f pmap_qremove(c5e6f000,1,c06ed30a,c8261d9c,cafb3b80,...) at pmap_qremove+0x58 pmap_remove_pages(cce9b0b0,cf52b000,ea85cbb4,0,c0a1fbc0,...) at pmap_remove_pages+0x410 exec_new_vmspace(ea85cbb4,c0a31c20,8,c826bd48,80,...) at exec_new_vmspace+0x1b0 exec_elf32_imgact(ea85cbb4,ea85cbfc,c09b88e7,cafb3b80,50,...) at exec_elf32_imgact+0x48e kern_execve(cafb3b80,ea85cc48,0,883024b4,8830250c,e4c17000,e4c17000,e4c170b3,e4c17264,e4c57400,3fd9c,8,e,0) at kern_execve+0x541 execve(cafb3b80,ea85ccec,c,c,c,...) at execve+0x4c syscall(ea85cd28) at syscall+0x342 Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (59, FreeBSD ELF32, execve), eip = 0x88169c2b, esp = 0xbfbfe9bc, ebp = 0xbfbfe9d8 --- Tracing command tifftopnm pid 20527 tid 100845 td 0xcb825000 cpustop_handler(1,eac849fc,c09328b6,1,eac849a8,...) at cpustop_handler+0x34 ipi_nmi_handler(1,eac849a8,c062a16b,c7bca000,cb1d6560,...) at ipi_nmi_handler+0x2f trap(eac84a08) at trap+0x36 calltrap() at calltrap+0x6 --- trap 0x13, eip = 0xc06ecd99, esp = 0xeac84a48, ebp = 0xeac84a60 --- _mtx_lock_sleep(c0a94ce4,cb825000,0,0,0,...) at _mtx_lock_sleep+0x79 pmap_enter(ca507198,88326000,2,c28a2120,3,...) at pmap_enter+0x66 vm_fault(ca5070e8,88326000,2,8,eac84c70,...) at vm_fault+0x1c14 trap_pfault(0,eac84cc8,c062a16b,c7bca000,cb1d6560,...) at trap_pfault+0x1ce trap(eac84d28) at trap+0x263 calltrap() at calltrap+0x6 --- trap 0xc, eip = 0x880d5cdd, esp = 0xbfbfb640, ebp = 0xbfbfb698 --- Tracing command perl5.14.2 pid 20526 tid 100278 td 0xcbc84b80 sched_switch(cbc84b80,0,104,3b38c51a,2123d7,...) at sched_switch+0x297 mi_switch(104,0,15c,ca80b560,ea3a5b70,...) at mi_switch+0x12f sleepq_switch(cbc84b80,0,c09c15c1,1a3,cbc84b80,...) at sleepq_switch+0xcc sleepq_catch_signals(15c,0,ea3a5bc4,c07073bc,ca80b560,...) at sleepq_catch_signals+0x52 sleepq_wait_sig(ca80b560,5c,c09c1fa4,100,0,...) at sleepq_wait_sig+0x18 _sleep(ca80b560,ca80b5e8,15c,c09c1fa4,0,...) at _sleep+0x2bc kern_wait(cbc84b80,502f,ea3a5c64,0,0,...) at kern_wait+0xfa1 wait4(cbc84b80,ea3a5cec,c,c,c,...) at wait4+0x3b syscall(ea3a5d28) at syscall+0x342 Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x882a1c6b, esp = 0xbfbfeb2c, ebp = 0xbfbfeb48 --- I can give more information from ddb output and or the written kerneldump.