Bug 254419 - Fatal trap 12: page fault while in kernel mode, nginx + sendfile on
Summary: Fatal trap 12: page fault while in kernel mode, nginx + sendfile on
Status: Closed FIXED
Alias: None
Product: Base System
Classification: Unclassified
Component: kern (show other bugs)
Version: 13.0-STABLE
Hardware: amd64 Any
: --- Affects Only Me
Assignee: Mark Johnston
URL:
Keywords: panic
Depends on:
Blocks:
 
Reported: 2021-03-20 01:19 UTC by Igor A. Valkov
Modified: 2021-12-01 22:55 UTC (History)
6 users (show)

See Also:


Attachments

Note You need to log in before you can comment on or make changes to this bug.
Description Igor A. Valkov 2021-03-20 01:19:44 UTC
FreeBSD-13.0-RC3, git rev 8f731a397ad4dc7b17622c0e69ac045f4a7b9d5b

nginx + sendfile on = kernel panic. With sendfile = off - working fine.


Fatal trap 12: page fault while in kernel mode
cpuid = 19; apic id = 13
fault virtual address   = 0x0
fault code              = supervisor read data, page not present
instruction pointer     = 0x20:0xffffffff8095fa46
stack pointer           = 0x28:0xfffffe01533dd1a0
frame pointer           = 0x28:0xfffffe01533dd1b0
code segment            = base rx0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 3395 (nginx)
trap number             = 12
panic: page fault
cpuid = 19
time = 1616197293
KDB: stack backtrace:
#0 0xffffffff80687015 at kdb_backtrace+0x65
#1 0xffffffff8063a051 at vpanic+0x181
#2 0xffffffff80639ec3 at panic+0x43
#3 0xffffffff809830d7 at trap_fatal+0x387
#4 0xffffffff8098312f at trap_pfault+0x4f
#5 0xffffffff8098278d at trap+0x27d
#6 0xffffffff8095b938 at calltrap+0x8
#7 0xffffffff8095f957 at in_cksum_skip+0x77
#8 0xffffffff8079dc1d at in_delayed_cksum+0x3d
#9 0xffffffff80823d03 at pf_test+0x1403
#10 0xffffffff8083ac6f at pf_check_out+0x1f
#11 0xffffffff80770de7 at pfil_run_hooks+0x97
#12 0xffffffff8079d3f1 at ip_output+0xb61
#13 0xffffffff807b44e4 at tcp_output+0x1b04
#14 0xffffffff807ca379 at tcp_usr_send+0x229
#15 0xffffffff80637f6a at vn_sendfile+0x197a
#16 0xffffffff80638967 at sendfile+0x127
#17 0xffffffff809839dc at amd64_syscall+0x10c
Uptime: 1m0s
Dumping 1632 out of 32637 MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91%

__curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
55              __asm("movq %%gs:%P1,%0" : "=r" (td) : "n" (offsetof(struct pcpu,

(kgdb) list *0xffffffff8095fa46
0xffffffff8095fa46 is in in_cksumdata (/usr/src/sys/amd64/amd64/in_cksum.c:113).
108             if ((offset = 3 & (long) lw) != 0) {
109                     const u_int32_t *masks = in_masks + (offset << 2);
110                     lw = (u_int32_t *) (((long) lw) - offset);
111                     sum = *lw++ & masks[len >= 3 ? 3 : len];
112                     len -= 4 - offset;
113                     if (len <= 0) {
114                             REDUCE32;
115                             return sum;
116                     }
117             }
(kgdb) 
(kgdb) bt
#0  __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
#1  doadump (textdump=<optimized out>) at /usr/src/sys/kern/kern_shutdown.c:399
#2  0xffffffff80639c46 in kern_reboot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:486
#3  0xffffffff8063a0c0 in vpanic (fmt=<optimized out>, ap=<optimized out>) at /usr/src/sys/kern/kern_shutdown.c:919
#4  0xffffffff80639ec3 in panic (fmt=<unavailable>) at /usr/src/sys/kern/kern_shutdown.c:843
#5  0xffffffff809830d7 in trap_fatal (frame=0xfffffe01533dd0e0, eva=0) at /usr/src/sys/amd64/amd64/trap.c:915
#6  0xffffffff8098312f in trap_pfault (frame=frame@entry=0xfffffe01533dd0e0, usermode=false, signo=<optimized out>, signo@entry=0x0, ucode=<optimized out>, ucode@entry=0x0)
    at /usr/src/sys/amd64/amd64/trap.c:732
#7  0xffffffff8098278d in trap (frame=0xfffffe01533dd0e0) at /usr/src/sys/amd64/amd64/trap.c:398
#8  <signal handler called>
#9  0xffffffff8095fa46 in in_cksumdata (buf=<optimized out>, len=len@entry=1140) at /usr/src/sys/amd64/amd64/in_cksum.c:113
#10 0xffffffff8095f957 in in_cksum_skip (m=0xfffff80608d32300, m@entry=0xfffff804e6cab200, len=1140, skip=<optimized out>, skip@entry=20) at /usr/src/sys/amd64/amd64/in_cksum.c:224
#11 0xffffffff8079dc1d in in_delayed_cksum (m=0xfffff804e6cab200) at /usr/src/sys/netinet/ip_output.c:1083
#12 0xffffffff80823d03 in pf_route (m=0xfffffe01533dd4f8, r=0xfffff8000d90cc00, dir=0, oifp=0xfffff8000d86c000, s=<optimized out>, pd=0xfffffe01533dd288, inp=0xfffff8062603a988)
    at /usr/src/sys/netpfil/pf/pf.c:5558
#13 pf_test (dir=<optimized out>, dir@entry=2, pflags=<optimized out>, ifp=<optimized out>, m0=<optimized out>, m0@entry=0xfffffe01533dd4f8, inp=<optimized out>)
    at /usr/src/sys/netpfil/pf/pf.c:6269
#14 0xffffffff8083ac6f in pf_check_out (m=0xfffffe01533dd4f8, ifp=0x0, flags=1140, ruleset=<optimized out>, inp=0x0) at /usr/src/sys/netpfil/pf/pf_ioctl.c:4516
#15 0xffffffff80770de7 in pfil_run_hooks (head=<optimized out>, p=..., ifp=0xfffff8000d86c000, flags=flags@entry=131072, inp=inp@entry=0xfffff8062603a988) at /usr/src/sys/net/pfil.c:187
#16 0xffffffff8079d3f1 in ip_output_pfil (mp=0xfffffe01533dd4f8, ifp=0xfffff8000d86c000, flags=0, inp=0xfffff8062603a988, dst=0xfffff8062603ab30, fibnum=<optimized out>, 
    error=<optimized out>) at /usr/src/sys/netinet/ip_output.c:130
#17 ip_output (m=m@entry=0xfffff804e6cab200, opt=<optimized out>, ro=<optimized out>, flags=0, imo=imo@entry=0x0, inp=<optimized out>) at /usr/src/sys/netinet/ip_output.c:705
#18 0xffffffff807b44e4 in tcp_output (tp=0xfffffe003fc5c890) at /usr/src/sys/netinet/tcp_output.c:1492
#19 0xffffffff807ca379 in tcp_usr_send (so=<optimized out>, flags=<optimized out>, m=0xfffff80626072800, nam=0x0, control=<optimized out>, td=0xfffffe0054f67500)
    at /usr/src/sys/netinet/tcp_usrreq.c:1210
#20 0xffffffff80637f6a in vn_sendfile (fp=<optimized out>, sockfd=97, hdr_uio=0x0, trl_uio=0x0, offset=<optimized out>, nbytes=<optimized out>, sent=0xfffffe01533dda88, flags=1, 
    td=0xfffffe0054f67500) at /usr/src/sys/kern/kern_sendfile.c:1182
#21 0xffffffff80638967 in fo_sendfile (fp=0x0, sockfd=1140, hdr_uio=0x0, trl_uio=0x0, offset=0, nbytes=1186733549, sent=0xfffffe01533dda88, flags=75701, td=0xfffffe0054f67500)
    at /usr/src/sys/sys/file.h:409
#22 sendfile (td=0xfffffe0054f67500, uap=0xfffffe0054f678e8, compat=<optimized out>) at /usr/src/sys/kern/kern_sendfile.c:1320
#23 0xffffffff809839dc in syscallenter (td=0xfffffe0054f67500) at /usr/src/sys/amd64/amd64/../../kern/subr_syscall.c:189
#24 amd64_syscall (td=0xfffffe0054f67500, traced=0) at /usr/src/sys/amd64/amd64/trap.c:1156
#25 <signal handler called>
#26 0x00000008008c834a in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fffffffd7c8
Comment 1 Ed Maste freebsd_committer 2021-03-22 14:16:08 UTC
From the backtrace pf is also involved
Comment 2 Mark Johnston freebsd_committer 2021-03-22 14:45:18 UTC
Are you able to test patches?  Based on what you wrote it should be fixed by https://reviews.freebsd.org/D29378
Comment 3 Igor A. Valkov 2021-03-22 23:05:43 UTC
(In reply to Mark Johnston from comment #2)

I have applied this patch D29378.id86147.diff

nginx + sendfile=on now is working fine without fatal trap 12 some hours.

Thanks!
Comment 4 Mark Johnston freebsd_committer 2021-03-23 14:05:31 UTC
(In reply to Igor A. Valkov from comment #3)
Thanks for testing.
Comment 5 commit-hook freebsd_committer 2021-03-23 14:05:48 UTC
A commit in branch main references this bug:

URL: https://cgit.FreeBSD.org/src/commit/?id=b93a796b06ec013a75a08ac43d8acf6aa94aa970

commit b93a796b06ec013a75a08ac43d8acf6aa94aa970
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2021-03-23 13:38:59 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2021-03-23 14:04:31 +0000

    pf: Handle unmapped mbufs when computing checksums

    PR:             254419
    Reviewed by:    gallatin, kp
    Tested by:      Igor A. Valkov <viaprog@gmail.com>
    MFC after:      3 days
    Sponsored by:   The FreeBSD Foundation
    Differential Revision:  https://reviews.freebsd.org/D29378

 sys/netpfil/pf/pf.c | 9 +++++++++
 1 file changed, 9 insertions(+)
Comment 6 commit-hook freebsd_committer 2021-03-26 16:33:41 UTC
A commit in branch releng/13.0 references this bug:

URL: https://cgit.FreeBSD.org/src/commit/?id=fa6d101e5f67246a6804577a9532676eae64c049

commit fa6d101e5f67246a6804577a9532676eae64c049
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2021-03-23 13:38:59 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2021-03-26 16:33:12 +0000

    pf: Handle unmapped mbufs when computing checksums

    Approved by:    re (cperciva)
    PR:             254419
    Reviewed by:    gallatin, kp
    Tested by:      Igor A. Valkov <viaprog@gmail.com>
    Sponsored by:   The FreeBSD Foundation
    Differential Revision:  https://reviews.freebsd.org/D29378

    (cherry picked from commit b93a796b06ec013a75a08ac43d8acf6aa94aa970)
    (cherry picked from commit 5fcab6fbcf8b99d1420e681731a07670c38defe3)

 sys/netpfil/pf/pf.c | 9 +++++++++
 1 file changed, 9 insertions(+)
Comment 7 commit-hook freebsd_committer 2021-03-28 00:26:08 UTC
A commit in branch stable/13 references this bug:

URL: https://cgit.FreeBSD.org/src/commit/?id=41a8dc361969629706827fb867cedaec3c270e68

commit 41a8dc361969629706827fb867cedaec3c270e68
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2021-03-23 13:38:59 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2021-03-28 00:23:57 +0000

    pf: Handle unmapped mbufs when computing checksums

    PR:             254419
    Reviewed by:    gallatin, kp
    Tested by:      Igor A. Valkov <viaprog@gmail.com>
    Sponsored by:   The FreeBSD Foundation
    Differential Revision:  https://reviews.freebsd.org/D29378

    (cherry picked from commit b93a796b06ec013a75a08ac43d8acf6aa94aa970)

 sys/netpfil/pf/pf.c | 9 +++++++++
 1 file changed, 9 insertions(+)
Comment 8 Rick 2021-11-09 09:28:59 UTC
I've possibly encountered the same/similar bug. (With help of @_martin, we found the cause of it)
See coredump below. (Discussion : https://forums.freebsd.org/threads/random-crash.82385/page-3#post-540831)

When using nginx,sendfile (in a jail) via optimization=aggressive in the pf firewall. 
The mbuf==NULL check fails because mbuf isn't NULL but invalid!


--

Fatal trap 12: page fault while in kernel mode
cpuid = 3; apic id = 03
fault virtual address   = 0x520
fault code              = supervisor read data, page not present
instruction pointer     = 0x20:0xffffffff810659dd
stack pointer           = 0x28:0xfffffe0051351f80
frame pointer           = 0x28:0xfffffe0051351f90
code segment            = base rx0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 12 (swi4: clock (0))
trap number             = 12
panic: page fault
cpuid = 3
time = 1636304186
KDB: stack backtrace:
#0 0xffffffff80c57345 at kdb_backtrace+0x65
#1 0xffffffff80c09d21 at vpanic+0x181
#2 0xffffffff80c09b93 at panic+0x43
#3 0xffffffff8108b187 at trap_fatal+0x387
#4 0xffffffff8108b1df at trap_pfault+0x4f
#5 0xffffffff8108a83d at trap+0x27d
#6 0xffffffff810617a8 at calltrap+0x8
#7 0xffffffff81065907 at in_cksum_skip+0x77
#8 0xffffffff82956329 at in4_cksum+0x59
#9 0xffffffff829373d0 at pf_return+0x270
#10 0xffffffff82931351 at pf_test_rule+0x1d71
#11 0xffffffff8292cd11 at pf_test+0x17c1
#12 0xffffffff82945bff at pf_check_out+0x1f
#13 0xffffffff80d41f87 at pfil_run_hooks+0x97
#14 0xffffffff80db2d71 at ip_output+0xb61
#15 0xffffffff80dc94b4 at tcp_output+0x1b04
#16 0xffffffff80dd7f2f at tcp_timer_rexmt+0x59f
#17 0xffffffff80c2598d at softclock_call_cc+0x13d
Uptime: 4m56s
Comment 9 martin 2021-11-09 19:31:43 UTC
I'm able to trigger a panic that seems to be related to this PR. Please let me
know if you think new PR should have been opened instead.

Test machine: amd64 13.0-RELEASE-p4 with GENERIC kernel, PF and nginx in jail.
Amount of CPUs and hypervisor don't play a role (tested on VirtualBox/VMware).

Few things needed to be set to trigger the bug. nginx with sendfile had to be on, 
PF config needed to be set certain way. Panic occurs almost immediately. 

/etc/rc.conf:

cloned_interfaces="lo1"
ipv4_addrs_lo1="10.0.2.100/24"
pf_enable="YES"
pflog_enable="YES"
pf_rules="/etc/pf.conf"
iocage_enable="YES"


/etc/pf.conf:

ext_if="vtnet0"
jail_if="lo1"

wan_ip4="172.20.1.200"
jail_net     = "10.0.2.0/24"
ip_webproxy  = "10.0.2.103"

webserver_sto = "(max-src-conn 50, overload <overloadlist> flush global)"
tcp_state ="flags S/SAFR modulate state"

table <overloadlist> persist

set block-policy return
set skip on lo0
set optimization aggressive

scrub in all

nat on $ext_if inet from $jail_net to any -> $wan_ip4
rdr on $ext_if inet proto tcp from any to $wan_ip4 port 80 -> $ip_webproxy

block in all
block out all

pass in quick proto tcp from any to any port 22
pass in inet proto tcp from any to $ip_webproxy port 80 $tcp_state $webserver_sto

pass out quick

In jail nginx was installed, active config:

(jail)# grep -vE '^$|^[ ]*#' /usr/local/etc/nginx/nginx.conf
worker_processes  1;
events {
    worker_connections  1024;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
    sendfile        on;
    keepalive_timeout  65;
    server {
        listen       80;
        server_name  localhost;
        location / {
            root   /usr/local/www/nginx;
            index  index.html index.htm;
        }
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   /usr/local/www/nginx-dist;
        }
    }
    access_log  off;
    error_log off;
}
#


I'm triggering the bug outside of the host by running:

ab -n 999999999 -c 10 http://172.20.1.200/sample250K.bin

System is pagefaulting, usually on 0 or small address such as 0x4f0.
Of all 12 tests I did system crashed in the same function/instruction. 

Fatal trap 12: page fault while in kernel mode
cpuid = 0; apic id = 00
fault virtual address   = 0x4f0
fault code              = supervisor read data, page not present
instruction pointer     = 0x20:0xffffffff81065b8d
stack pointer           = 0x28:0xfffffe0051351f80
frame pointer           = 0x28:0xfffffe0051351f90
code segment            = base rx0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 12 (swi4: clock (0))
trap number             = 12
panic: page fault
cpuid = 0
time = 1636488387
KDB: stack backtrace:
#0 0xffffffff80c574c5 at kdb_backtrace+0x65
#1 0xffffffff80c09ea1 at vpanic+0x181
#2 0xffffffff80c09d13 at panic+0x43
#3 0xffffffff8108b1b7 at trap_fatal+0x387
#4 0xffffffff8108b20f at trap_pfault+0x4f
#5 0xffffffff8108a86d at trap+0x27d
#6 0xffffffff81061958 at calltrap+0x8
#7 0xffffffff81065ab7 at in_cksum_skip+0x77
#8 0xffffffff82956329 at in4_cksum+0x59
#9 0xffffffff829373d0 at pf_return+0x270
#10 0xffffffff82931351 at pf_test_rule+0x1d71
#11 0xffffffff8292cd11 at pf_test+0x17c1
#12 0xffffffff82945bff at pf_check_out+0x1f
#13 0xffffffff80d42137 at pfil_run_hooks+0x97
#14 0xffffffff80db2f21 at ip_output+0xb61
#15 0xffffffff80dc9664 at tcp_output+0x1b04
#16 0xffffffff80dd80df at tcp_timer_rexmt+0x59f
#17 0xffffffff80c25b0d at softclock_call_cc+0x13d


kgdb) bt
#0  __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
#1  doadump (textdump=<optimized out>) at /usr/src/sys/kern/kern_shutdown.c:399
#2  0xffffffff80c09a96 in kern_reboot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:486
#3  0xffffffff80c09f10 in vpanic (fmt=<optimized out>, ap=<optimized out>) at /usr/src/sys/kern/kern_shutdown.c:919
#4  0xffffffff80c09d13 in panic (fmt=<unavailable>) at /usr/src/sys/kern/kern_shutdown.c:843
#5  0xffffffff8108b1b7 in trap_fatal (frame=0xfffffe0051351ec0, eva=1264) at /usr/src/sys/amd64/amd64/trap.c:915
#6  0xffffffff8108b20f in trap_pfault (frame=frame@entry=0xfffffe0051351ec0, usermode=false, signo=<optimized out>, signo@entry=0x0, ucode=<optimized out>, ucode@entry=0x0) at /usr/src/sys/amd64/amd64/trap.c:732
#7  0xffffffff8108a86d in trap (frame=0xfffffe0051351ec0) at /usr/src/sys/amd64/amd64/trap.c:398
#8  <signal handler called>
#9  0xffffffff81065b8d in in_cksumdata (buf=<optimized out>, len=len@entry=1448) at /usr/src/sys/amd64/amd64/in_cksum.c:111
#10 0xffffffff81065ab7 in in_cksum_skip (m=0xfffff80036cca700, len=1448, skip=<optimized out>) at /usr/src/sys/amd64/amd64/in_cksum.c:224
#11 0xffffffff82956329 in in4_cksum (m=0x4f0, nxt=<optimized out>, nxt@entry=6 '\006', off=3, len=<optimized out>) at /usr/src/sys/netpfil/pf/in4_cksum.c:117
#12 0xffffffff829373d0 in pf_check_proto_cksum (m=0xfffff80034b5ee00, off=<optimized out>, len=3, p=6 '\006', af=2 '\002') at /usr/src/sys/netpfil/pf/pf.c:5844
#13 pf_return (r=r@entry=0xfffff800360ec800, nr=<optimized out>, nr@entry=0xfffff800033a1800, pd=pd@entry=0xfffffe0051352590, sk=<optimized out>, off=<optimized out>, off@entry=20, m=<optimized out>, m@entry=0xfffff80034b5ee00,
    th=0xfffffe0051352660, kif=0xfffff8003649f500, bproto_sum=24767, bip_sum=0, hdrlen=20, reason=0xfffffe005135241e) at /usr/src/sys/netpfil/pf/pf.c:2654
#14 0xffffffff82931351 in pf_test_rule (rm=rm@entry=0xfffffe0051352630, sm=sm@entry=0xfffffe0051352648, direction=direction@entry=2, kif=kif@entry=0xfffff8003649f500, m=m@entry=0xfffff80034b5ee00, off=20, pd=0xfffffe0051352590,
    am=0xfffffe0051352620, rsm=0xfffffe0051352610, inp=0xfffff80034e803d0) at /usr/src/sys/netpfil/pf/pf.c:3641
#15 0xffffffff8292cd11 in pf_test (dir=<optimized out>, dir@entry=2, pflags=<optimized out>, ifp=<optimized out>, m0=<optimized out>, m0@entry=0xfffffe0051352808, inp=0xfffff80034e803d0) at /usr/src/sys/netpfil/pf/pf.c:6005
#16 0xffffffff82945bff in pf_check_out (m=0xfffffe0051352808, ifp=0x3, flags=1448, ruleset=<optimized out>, inp=0xff000000) at /usr/src/sys/netpfil/pf/pf_ioctl.c:4516
#17 0xffffffff80d42137 in pfil_run_hooks (head=<optimized out>, p=..., ifp=0xfffff80003656800, flags=flags@entry=131072, inp=inp@entry=0xfffff80034e803d0) at /usr/src/sys/net/pfil.c:187
#18 0xffffffff80db2f21 in ip_output_pfil (mp=0xfffffe0051352808, ifp=0xfffff80003656800, flags=0, inp=0xfffff80034e803d0, dst=0xfffff80034e80578, fibnum=<optimized out>, error=<optimized out>)
    at /usr/src/sys/netinet/ip_output.c:130
#19 ip_output (m=m@entry=0xfffff80034b5ee00, opt=<optimized out>, ro=<optimized out>, flags=0, imo=imo@entry=0x0, inp=<optimized out>) at /usr/src/sys/netinet/ip_output.c:705
#20 0xffffffff80dc9664 in tcp_output (tp=0xfffffe00980a68f0) at /usr/src/sys/netinet/tcp_output.c:1492
#21 0xffffffff80dd80df in tcp_timer_rexmt (xtp=0xfffffe00980a68f0) at /usr/src/sys/netinet/tcp_timer.c:879
#22 0xffffffff80c25b0d in softclock_call_cc (c=0xfffffe00980a6b78, cc=cc@entry=0xffffffff81ca8200 <cc_cpu>, direct=direct@entry=0) at /usr/src/sys/kern/kern_timeout.c:696
#23 0xffffffff80c25f99 in softclock (arg=0xffffffff81ca8200 <cc_cpu>) at /usr/src/sys/kern/kern_timeout.c:816
#24 0xffffffff80bcafdd in intr_event_execute_handlers (p=<optimized out>, ie=0xfffff80003412700) at /usr/src/sys/kern/kern_intr.c:1168
#25 ithread_execute_handlers (p=<optimized out>, ie=0xfffff80003412700) at /usr/src/sys/kern/kern_intr.c:1181
#26 ithread_loop (arg=arg@entry=0xfffff800033efd20) at /usr/src/sys/kern/kern_intr.c:1269
#27 0xffffffff80bc7dde in fork_exit (callout=0xffffffff80bcad90 <ithread_loop>, arg=0xfffff800033efd20, frame=0xfffffe0051352c00) at /usr/src/sys/kern/kern_fork.c:1069

(kgdb) f 9
#9  0xffffffff81065b8d in in_cksumdata (buf=<optimized out>, len=len@entry=1448) at /usr/src/sys/amd64/amd64/in_cksum.c:111
111	/usr/src/sys/amd64/amd64/in_cksum.c: No such file or directory.
(kgdb) x/4i $pc
=> 0xffffffff81065b8d <in_cksumdata+109>:	and    (%rdi),%r8d
   0xffffffff81065b90 <in_cksumdata+112>:	add    %ecx,%esi
   0xffffffff81065b92 <in_cksumdata+114>:	add    $0xfffffffc,%esi
   0xffffffff81065b95 <in_cksumdata+117>:	test   %esi,%esi
(kgdb) i r $rdi
rdi            0x4f0               1264
(kgdb)
Comment 10 Mark Johnston freebsd_committer 2021-11-09 19:41:36 UTC
(In reply to martin from comment #9)
I'm working on a patch for this now.  See PR 259645.  A workaround in the meantime that doesn't require sendfile to be disabled is to set the kern.ipc.mb_use_ext_pgs sysctl to 0.
Comment 11 martin 2021-11-09 19:49:52 UTC
(In reply to Mark Johnston from comment #10)
Thank you. As probably expected I can't trigger the panic with the disabled sysctl. 

It's probably irrelevant to the issue but 259645 states this issue occurred after p5 update. That's not the case, I was able to trigger this in older versions too.
Comment 12 Mark Johnston freebsd_committer 2021-11-10 14:45:31 UTC
(In reply to martin from comment #11)
Indeed, I am sure that this problem exists in the original 13.0 release.