Bug 241417

Summary: panic: Fatal trap 1: privileged instruction fault while in kernel mode
Product: Base System Reporter: Nick Briggs <nicholas.h.briggs>
Component: kernAssignee: freebsd-bugs (Nobody) <bugs>
Status: Closed Unable to Reproduce    
Severity: Affects Only Me CC: chris, cy, markj
Priority: --- Keywords: panic
Version: CURRENT   
Hardware: i386   
OS: Any   
Attachments:
Description Flags
core.txt.0 from crash
none
Output from "/usr/local/bin/acpidump -o /tmp/acpidump.txt" none

Description Nick Briggs 2019-10-22 18:38:42 UTC
Created attachment 208510 [details]
core.txt.0 from crash

FreeBSD flap.gateway.sonic.net 12.0-RELEASE-p10 FreeBSD 12.0-RELEASE-p10 GENERIC  i386

+Fatal trap 1: privileged instruction fault while in kernel mode
+cpuid = 0; apic id = 00
+error code		= 0
+instruction pointer	= 0x20:0xffc033d9
+stack pointer	        = 0x28:0xfba0bc0
+frame pointer	        = 0x28:0xfba0bd0
+code segment		= base rx0, limit 0xfffff, type 0x1b
+			= DPL 0, pres 1, def32 1, gran 1
+processor eflags	= resume, IOPL = 0
+current process		= 11 (idle: cpu0)
+trap number		= 1
+panic: privileged instruction fault
+cpuid = 0
+time = 1571652112
+KDB: stack backtrace:
+#0 0x110848f at kdb_backtrace+0x4f
+#1 0x10bb467 at vpanic+0x147
+#2 0x10bb31b at panic+0x1b
+#3 0x16920b5 at trap_fatal+0x395
+#4 0x169144e at trap+0xde
+#5 0xffc0315d at PTDpde+0x4165
+#6 0xa0c89c at acpi_cpu_idle+0x30c
+#7 0x16aa93f at cpu_idle_acpi+0x3f


+#8 0x16aa9fb at cpu_idle+0xab
+#9 0x10ef456 at sched_idletd+0x376
+#10 0x107cad1 at fork_exit+0x71
+#11 0xffc033ba at PTDpde+0x43c2
+Uptime: 2d12h11m14s
+Physical memory: 931 MB
+Dumping 161 MB: 146 130 114 98 82 66 50 34 18 2
+Dump complete

I have attached core.txt.0, and have the vmcore.0 if it is of use.
Comment 1 Andriy Gapon freebsd_committer 2019-10-23 08:06:43 UTC
This looks like a memory corruption, maybe a hardware a problem.
Specifically I mean this address: 0xffc033ba.
Comment 2 Nick Briggs 2019-10-24 01:50:54 UTC
This is the code sequence it was executing:

   0x00a0c87c <+748>:	mov    %eax,%edx
   0x00a0c87e <+750>:	in     (%dx),%al
   0x00a0c87f <+751>:	mov    -0x14(%ebp),%ebx
   0x00a0c882 <+754>:	cmpb   $0x0,0x30(%edi,%ebx,4)
   0x00a0c887 <+759>:	je     0xa0c897 <acpi_cpu_idle+775>
   0x00a0c889 <+761>:	mov    0x34(%edi,%ebx,4),%eax
   0x00a0c88d <+765>:	mov    %eax,(%esp)
   0x00a0c890 <+768>:	call   0x16aa370 <acpi_cpu_idle_mwait>
   0x00a0c895 <+773>:	jmp    0xa0c89c <acpi_cpu_idle+780>
   0x00a0c897 <+775>:	call   0x16aa360 <acpi_cpu_c1>
   0x00a0c89c <+780>:	lea    0x30(%edi,%ebx,4),%eax
   0x00a0c8a0 <+784>:	mov    %eax,-0x10(%ebp)
   0x00a0c8a3 <+787>:	call   *0x1d2af74


Are you (Andriy) sure that you're interpreting the stack trace correctly?
I see exactly the same location in the stack trace in, for example, bug #227654, comment #4
Comment 3 Nick Briggs 2019-10-24 02:16:30 UTC
Created attachment 208547 [details]
Output from "/usr/local/bin/acpidump -o /tmp/acpidump.txt"
Comment 4 Nick Briggs 2019-10-24 02:18:33 UTC
Since this crash seems to be in an ACPI related area, I've attached the acpidump output.  If I use the system supplied /usr/sbin/acpidump -dt, it produces inconsistent results on each execution, so I've attached the output from the ports version.
Comment 5 Andriy Gapon freebsd_committer 2019-10-24 11:12:40 UTC
Maybe I am indeed incorrectly interpreting 0xffc0315d. Maybe it's a low level trap handler that calls trap().  But I see do not see anything near 0x00a0c89c in acpi_cpu_idle or in acpi_cpu_c1 that could cause that fault.
Were you, by chance, doing anything with DTrace when the crash happened?
Comment 6 Nick Briggs 2019-10-24 17:37:26 UTC
(In reply to Andriy Gapon from comment #5)
No DTrace.  The machine was sitting idle with nobody logged in when it failed.
The only trap that the LEA instruction is documented as able to cause is:
     #UD If source operand is not a memory location.
Comment 7 Mark Johnston freebsd_committer 2020-06-02 14:38:29 UTC
Have you seen any similar crashes since the initial one?
Comment 8 Nick Briggs 2020-06-05 18:23:01 UTC
Re: "Have you seen any similar crashes since the initial one?"

No, it has not happened again.

Since the original report I upgraded the system to FreeBSD 12.1 (as soon it was released) and am currently on 12.1-RELEASE-p5.
Comment 9 Mark Johnston freebsd_committer 2020-06-06 15:34:33 UTC
(In reply to Nick Briggs from comment #8)
Thanks for the reply.  Since you have upgraded, I will close the bug for now.  Please re-open the bug if the panic occurs again.
Comment 10 Cy Schubert freebsd_committer 2021-09-21 19:53:23 UTC
I'm seeing this while executing poudriere in i386 mode:

Fatal trap 1: privileged instruction fault while in kernel mode^M
cpuid = 3; apic id = 03^M
instruction pointer     = 0x20:0xffffffff80a5fef1^M
stack pointer           = 0x0:0xfffffe00e5237ad0^M
frame pointer           = 0x0:0xfffffe00e5237af0^M
code segment            = base rx0, limit 0xfffff, type 0x1b^M
                        = DPL 0, pres 1, long 1, def32 0, gran 1^M
processor eflags        = interrupt enabled, resume, IOPL = 0^M
current process         = 5200 (cc)^M
trap number             = 1^M
panic: privileged instruction fault^M
cpuid = 3^M
time = 1632253591^M
KDB: stack backtrace:^M
db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe00e52377d0^M
vpanic() at vpanic+0x187/frame 0xfffffe00e5237830^M
panic() at panic+0x43/frame 0xfffffe00e5237890^M
trap_fatal() at trap_fatal+0x387/frame 0xfffffe00e52378f0^M
trap() at trap+0x8b/frame 0xfffffe00e5237a00^M
calltrap() at calltrap+0x8/frame 0xfffffe00e5237a00^M
--- trap 0x1, rip = 0xffffffff80a5fef1, rsp = 0xfffffe00e5237ad0, rbp = 0xfffffe00e5237af0 ---^M
ia32_get_mcontext() at ia32_get_mcontext+0x181/frame 0xfffffe00e5237af0^M
freebsd32_getcontext() at freebsd32_getcontext+0x52/frame 0xfffffe00e5237df0^M
ia32_syscall() at ia32_syscall+0x126/frame 0xfffffe00e5237f30^M
int0x80_syscall_common() at int0x80_syscall_common+0x9c/frame 0xffffca38^M
Uptime: 8m48s^M
Dumping 526 out of 8161 MB: (CTRL-C to abort) ..4%..13%..22%..31%..43%..52%..61%..73%..83%..92%^M
Dump complete^M
acpi0: reset failed - timeout^M
Automatic reboot in 15 seconds - press a key on the console to abort^M
Rebooting...^M
cpu_reset: Restarting BSP^M
cpu_reset_proxy: Stopped CPU 3^M

14-CURRENT as of noon PDT today

commit bcdc599dc2a187052cb13e18f22d3f0c655f95e6 (freebsd/main, freebsd/HEAD, main)
Comment 11 Cy Schubert freebsd_committer 2021-09-21 20:05:44 UTC
This BTW was the last commit, not the cause.
Comment 12 Cy Schubert freebsd_committer 2021-09-21 20:21:34 UTC
Reading symbols from /boot/kernel/kernel...
Reading symbols from /usr/lib/debug//boot/kernel/kernel.debug...

Unread portion of the kernel message buffer:
interrupt enabled, resume, IOPL = 0
current process		= 5200 (cc)
trap number		= 1
panic: privileged instruction fault
cpuid = 3
time = 1632253591
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe00e52377d0
vpanic() at vpanic+0x187/frame 0xfffffe00e5237830
panic() at panic+0x43/frame 0xfffffe00e5237890
trap_fatal() at trap_fatal+0x387/frame 0xfffffe00e52378f0
trap() at trap+0x8b/frame 0xfffffe00e5237a00
calltrap() at calltrap+0x8/frame 0xfffffe00e5237a00
--- trap 0x1, rip = 0xffffffff80a5fef1, rsp = 0xfffffe00e5237ad0, rbp = 0xfffffe00e5237af0 ---
ia32_get_mcontext() at ia32_get_mcontext+0x181/frame 0xfffffe00e5237af0
freebsd32_getcontext() at freebsd32_getcontext+0x52/frame 0xfffffe00e5237df0
ia32_syscall() at ia32_syscall+0x126/frame 0xfffffe00e5237f30
int0x80_syscall_common() at int0x80_syscall_common+0x9c/frame 0xffffca38
Uptime: 8m48s
Dumping 526 out of 8161 MB: (CTRL-C to abort) ..4%..13%..22%..31%..43%..52%..61%..73%..83%..92%

__curthread () at /opt/src/git-src/sys/amd64/include/pcpu_aux.h:55
55		__asm("movq %%gs:%P1,%0" : "=r" (td) : "n" (offsetof(struct pcpu,
(kgdb) bt
#0  __curthread () at /opt/src/git-src/sys/amd64/include/pcpu_aux.h:55
#1  doadump (textdump=textdump@entry=1)
    at /opt/src/git-src/sys/kern/kern_shutdown.c:399
#2  0xffffffff806d417b in kern_reboot (howto=260)
    at /opt/src/git-src/sys/kern/kern_shutdown.c:486
#3  0xffffffff806d45f6 in vpanic (fmt=0xffffffff80a9b26c "%s", 
    ap=<optimized out>) at /opt/src/git-src/sys/kern/kern_shutdown.c:919
#4  0xffffffff806d43f3 in panic (fmt=<unavailable>)
    at /opt/src/git-src/sys/kern/kern_shutdown.c:843
#5  0xffffffff80a3a587 in trap_fatal (frame=0xfffffe00e5237a10, eva=0)
    at /opt/src/git-src/sys/amd64/amd64/trap.c:946
#6  0xffffffff80a39a7b in trap (frame=0xfffffe00e5237a10)
    at /opt/src/git-src/sys/amd64/amd64/trap.c:251
#7  <signal handler called>
#8  0xffffffff80a5fef1 in ia32_get_fpcontext (td=0xfffffe00e468f3a0, 
    mcp=0xfffffe00e5237b10, xfpusave=0x0, xfpusave_len=0x0)
    at /opt/src/git-src/sys/amd64/ia32/ia32_signal.c:107
#9  ia32_get_mcontext (td=td@entry=0xfffffe00e468f3a0, 
    mcp=mcp@entry=0xfffffe00e5237b10, flags=1)
    at /opt/src/git-src/sys/amd64/ia32/ia32_signal.c:177
#10 0xffffffff80a5fca2 in freebsd32_getcontext (td=0xfffffe00e468f3a0, 
    uap=0xfffffe00e468f790)
    at /opt/src/git-src/sys/amd64/ia32/ia32_signal.c:260
#11 0xffffffff80a61a16 in syscallenter (td=0xfffffe00e468f3a0)
    at /opt/src/git-src/sys/amd64/ia32/../../kern/subr_syscall.c:189
#12 ia32_syscall (frame=0xfffffe00e5237f40)
    at /opt/src/git-src/sys/amd64/ia32/ia32_syscall.c:218
#13 0xffffffff80a12bcf in int0x80_syscall_common ()
    at /opt/src/git-src/sys/amd64/ia32/ia32_exception.S:77
#14 0x00000000ffffc710 in ?? ()
#15 0x00000000ffffc9d0 in ?? ()
#16 0x000000002540b9ad in ?? ()
#17 0x000000002526104f in ?? ()
#18 0x0000000000000000 in ?? ()
(kgdb) frame 8
#8  0xffffffff80a5fef1 in ia32_get_fpcontext (td=0xfffffe00e468f3a0, 
    mcp=0xfffffe00e5237b10, xfpusave=0x0, xfpusave_len=0x0)
    at /opt/src/git-src/sys/amd64/ia32/ia32_signal.c:107
107			*xfpusave_len = mcp->mc_xfpustate_len =
(kgdb) l
102		if (!use_xsave || cpu_max_ext_state_size <= sizeof(struct savefpu)) {
103			*xfpusave_len = 0;
104			*xfpusave = NULL;
105		} else {
106			mcp->mc_flags |= _MC_IA32_HASFPXSTATE;
107			*xfpusave_len = mcp->mc_xfpustate_len =
108			    cpu_max_ext_state_size - sizeof(struct savefpu);
109			*xfpusave = (char *)(get_pcb_user_save_td(td) + 1);
110		}
111	}
(kgdb) disassemble /m
Dump of assembler code for function ia32_get_mcontext:
98		mcp->mc_ownedfp = fpugetregs(td);
   0xffffffff80a5fe9d <+301>:	mov    %r14,%rdi
   0xffffffff80a5fea0 <+304>:	call   0xffffffff80a16ca0 <fpugetregs>
   0xffffffff80a5fea5 <+309>:	mov    %eax,0x58(%rbx)

99		bcopy(get_pcb_user_save_td(td), &mcp->mc_fpstate[0],
   0xffffffff80a5fea8 <+312>:	lea    0x60(%rbx),%r15
   0xffffffff80a5feac <+316>:	mov    %r14,%rdi
   0xffffffff80a5feaf <+319>:	call   0xffffffff80a3bbc0 <get_pcb_user_save_td>
   0xffffffff80a5feb4 <+324>:	mov    $0x200,%edx
   0xffffffff80a5feb9 <+329>:	mov    %r15,%rdi
   0xffffffff80a5febc <+332>:	mov    %rax,%rsi
   0xffffffff80a5febf <+335>:	call   0xffffffff80a35d20 <memmove_std>

100		    sizeof(mcp->mc_fpstate));
101		mcp->mc_fpformat = fpuformat();
   0xffffffff80a5fec4 <+340>:	call   0xffffffff80a169e0 <fpuformat>
   0xffffffff80a5fec9 <+345>:	mov    %eax,0x54(%rbx)

102		if (!use_xsave || cpu_max_ext_state_size <= sizeof(struct savefpu)) {
   0xffffffff80a5fecc <+348>:	cmpl   $0x0,0x53c73d(%rip)        # 0xffffffff80f9c610 <use_xsave>
   0xffffffff80a5fed3 <+355>:	je     0xffffffff80a5fef1 <ia32_get_mcontext+385>
   0xffffffff80a5fed5 <+357>:	mov    0x5274bd(%rip),%eax        # 0xffffffff80f87398 <cpu_max_ext_state_size>
   0xffffffff80a5fedb <+363>:	cmp    $0x200,%eax
   0xffffffff80a5fee0 <+368>:	jbe    0xffffffff80a5fef1 <ia32_get_mcontext+385>

103			*xfpusave_len = 0;
104			*xfpusave = NULL;
105		} else {
106			mcp->mc_flags |= _MC_IA32_HASFPXSTATE;
   0xffffffff80a5fee2 <+370>:	orb    $0x4,0x5c(%rbx)

107			*xfpusave_len = mcp->mc_xfpustate_len =
   0xffffffff80a5feeb <+379>:	mov    %eax,0x26c(%rbx)
=> 0xffffffff80a5fef1 <+385>:	ud2    
   0xffffffff80a5fef3 <+387>:	xor    %eax,%eax
   0xffffffff80a5fef5 <+389>:	mov    $0x140,%edi

108			    cpu_max_ext_state_size - sizeof(struct savefpu);
   0xffffffff80a5fee6 <+374>:	add    $0xfffffe00,%eax

109			*xfpusave = (char *)(get_pcb_user_save_td(td) + 1);
110		}
111	}
112	
113	static int
--Type <RET> for more, q to quit, c to continue without paging--q
Quit
(kgdb)
Comment 13 Cy Schubert freebsd_committer 2021-09-21 20:23:16 UTC
Why the ud2 instruction?

=> 0xffffffff80a5fef1 <+385>:	ud2
Comment 14 Cy Schubert freebsd_committer 2021-09-21 20:24:11 UTC
AMD64 in i386 emulation.
Comment 15 Cy Schubert freebsd_committer 2021-09-21 20:54:25 UTC
This is a different panic and should be a new PR.