Created attachment 196354 [details] Text dump of newnfs_request -> witness panic I recently got a few similar panics, seemingly originating from newnfs_request. The panic goes like this: shared lock of (lockmgr) ufs @ /usr/src/sys/kern/vfs_lookup.c:671 while exclusively locked from /usr/src/sys/kern/vfs_subr.c:2590 panic: excl->share cpuid = 2 time = 1534686985 ... #1 doadump (textdump=<optimized out>) at /usr/src/sys/kern/kern_shutdown.c:366 #2 0xffffffff8044906b in db_dump (dummy=<optimized out>, dummy2=<unavailable>, dummy3=<unavailable>, dummy4=<unavailable>) at /usr/src/sys/ddb/db_command.c:574 #3 0xffffffff80448e39 in db_command (last_cmdp=<optimized out>, cmd_table=<optimized out>, dopager=1) at /usr/src/sys/ddb/db_command.c:481 #4 0xffffffff80448bb4 in db_command_loop () at /usr/src/sys/ddb/db_command.c:534 #5 0xffffffff8044bd7f in db_trap (type=<optimized out>, code=<optimized out>) at /usr/src/sys/ddb/db_main.c:252 #6 0xffffffff80be79d5 in kdb_trap (type=3, code=0, tf=<optimized out>) at /usr/src/sys/kern/subr_kdb.c:693 #7 0xffffffff81084c3a in trap (frame=0xfffffe0000582fc0) at /usr/src/sys/amd64/amd64/trap.c:605 #8 <signal handler called> #9 kdb_enter (why=0xffffffff8131179d "panic", msg=<optimized out>) at /usr/src/sys/kern/subr_kdb.c:479 #10 0xffffffff80b9d7b1 in vpanic (fmt=<optimized out>, ap=0xfffffe0000583130) at /usr/src/sys/kern/kern_shutdown.c:852 #11 0xffffffff80b9d550 in kassert_panic (fmt=0xffffffff812a3ed5 "excl->share") at /usr/src/sys/kern/kern_shutdown.c:749 #12 0xffffffff80c083a8 in witness_checkorder (lock=0xfffff80004190ba8, flags=1, file=0xffffffff8130d3f0 "/usr/src/sys/kern/vfs_lookup.c", line=<optimized out>, interlock=0xffffffff81f64e2c <w_locklistdata+264684>) at /usr/src/sys/kern/subr_witness.c:1176 #13 0xffffffff80b70c3e in lockmgr_slock_hard (lk=<optimized out>, flags=2106368, ilk=0xfffff80004190bd8, file=<optimized out>, line=<optimized out>, lwa=<optimized out>) at /usr/src/sys/kern/kern_lock.c:567 #14 0xffffffff80b71d5b in __lockmgr_args (lk=<optimized out>, flags=<optimized out>, ilk=<optimized out>, wmesg=<optimized out>, pri=<optimized out>, timo=<optimized out>, file=0xffffffff8130d3f0 "/usr/src/sys/kern/vfs_lookup.c", line=671) at /usr/src/sys/kern/kern_lock.c:1195 #15 0xffffffff80ec6a05 in _lockmgr_args (lk=0xfffff80004190ba8, flags=2106368, ilk=<optimized out>, wmesg=<optimized out>, prio=0, timo=0, file=<optimized out>, line=18) at /usr/src/sys/sys/lockmgr.h:104 #16 ffs_lock (ap=0xfffffe00005833c8) at /usr/src/sys/ufs/ffs/ffs_vnops.c:428 #17 0xffffffff81208059 in VOP_LOCK1_APV ( vop=0xffffffff81b62d50 <ffs_vnodeops2>, a=0xfffffe00005833c8) at vnode_if.c:2087 #18 0xffffffff80c86927 in VOP_LOCK1 (vp=<optimized out>, flags=2106368, file=<optimized out>, line=671) at ./vnode_if.h:859 #19 _vn_lock (vp=0xfffff80004190b40, flags=2106368, file=0xffffffff8130d3f0 "/usr/src/sys/kern/vfs_lookup.c", line=671) at /usr/src/sys/kern/vfs_vnops.c:1531 #20 0xffffffff80c68fe6 in lookup (ndp=0xfffffe00005835a0) at /usr/src/sys/kern/vfs_lookup.c:669 #21 0xffffffff80c68aad in namei (ndp=0xfffffe00005835a0) at /usr/src/sys/kern/vfs_lookup.c:450 #22 0xffffffff80c49b85 in unp_connectat (fd=<optimized out>, so=<optimized out>, nam=<optimized out>, td=0xfffff800036f2000) at /usr/src/sys/kern/uipc_usrreq.c:1554 #23 0xffffffff80c3bc98 in soconnectat (fd=<optimized out>, so=<optimized out>, nam=0xfffff8000367d800, td=0xffffffff80b7d390 <_mtx_init+144>) at /usr/src/sys/kern/uipc_socket.c:1230 #24 0xffffffff80e611b9 in clnt_vc_create (so=0xfffff80004c316d0, raddr=0xfffff800035fc020, prog=553713921, vers=1, sendsz=4096, recvsz=4096, intrflag=0) at /usr/src/sys/rpc/clnt_vc.c:159 #25 0xffffffff80e60439 in clnt_reconnect_connect (cl=0xfffff80003372840) at /usr/src/sys/rpc/clnt_rc.c:193 #26 clnt_reconnect_call (cl=0xfffff80003372840, ext=0xfffffe0000583ab0, proc=1, args=0xfffff800048dda00, resultsp=0xfffffe0000583c28, utimeout=...) at /usr/src/sys/rpc/clnt_rc.c:265 #27 0xffffffff80a637ec in newnfs_request (nd=0xfffffe0000583c28, nmp=0x0, clp=0x0, nrp=0xffffffff82021a18 <nfsrv_nfsuserdsock>, vp=0x0, td=0x0, cred=0xfffff80003e65000, prog=553713921, vers=1, retsum=0x0, toplevel=0, xidp=0x0, dssep=0x0) at /usr/src/sys/fs/nfs/nfs_commonkrpc.c:818 #28 0xffffffff80a6d5f9 in nfsrv_getuser (procnum=1, uid=<optimized out>, gid=<optimized out>, name=0x0, p=0xfffffe0000582ce0) at /usr/src/sys/fs/nfs/nfs_commonsubs.c:3616 #29 0xffffffff80a6d71a in nfsrv_getgrpscred (oldcred=0xfffff80003deec00) at /usr/src/sys/fs/nfs/nfs_commonsubs.c:3150 #30 0xffffffff80acf53e in nfsd_excred (nd=0xfffffe0000583ff8, exp=<optimized out>, credanon=0xfffffe0000582f80) at /usr/src/sys/fs/nfsserver/nfs_nfsdport.c:2932 #31 0xffffffff80aa8104 in nfsrvd_compound (nd=<optimized out>, isdgram=0, tag=0x10 <error: Cannot access memory at address 0x10>, taglen=<optimized out>, minorvers=<optimized out>, p=<optimized out>) at /usr/src/sys/fs/nfsserver/nfs_nfsdsocket.c:1008 #32 nfsrvd_dorpc (nd=0xfffffe0000583ff8, isdgram=0, tag=0x10 <error: Cannot access memory at address 0x10>, taglen=7, minorvers=<optimized out>, p=0xfffff800036f2000) at /usr/src/sys/fs/nfsserver/nfs_nfsdsocket.c:560 #33 0xffffffff80abc3a7 in nfs_proc (xid=<optimized out>, xprt=<optimized out>, nd=<optimized out>, rpp=<optimized out>) at /usr/src/sys/fs/nfsserver/nfs_nfsdkrpc.c:387 #34 nfssvc_program (rqst=0xfffff8004d1d5800, xprt=0xfffff800035fb600) at /usr/src/sys/fs/nfsserver/nfs_nfsdkrpc.c:272 #35 0xffffffff80e68499 in svc_executereq (rqstp=<optimized out>) at /usr/src/sys/rpc/svc.c:1031 #36 svc_run_internal (grp=<optimized out>, ismaster=1) at /usr/src/sys/rpc/svc.c:1306 #37 0xffffffff80e6785e in svc_run (pool=<optimized out>) at /usr/src/sys/rpc/svc.c:1385 #38 0xffffffff80abca06 in nfsrvd_nfsd (td=<optimized out>, args=0xfffffe0000584510) at /usr/src/sys/fs/nfsserver/nfs_nfsdkrpc.c:542 #39 0xffffffff80ad298b in nfssvc_nfsd (td=0xfffff800036f2000, uap=<optimized out>) at /usr/src/sys/fs/nfsserver/nfs_nfsdport.c:3451 #40 0xffffffff80e45eeb in sys_nfssvc (td=0xfffff800036f2000, uap=0xfffff800036f23c0) at /usr/src/sys/nfs/nfs_nfssvc.c:111 #41 0xffffffff810859ef in syscallenter (td=0xfffff800036f2000) at /usr/src/sys/amd64/amd64/../../kern/subr_syscall.c:135 #42 amd64_syscall (td=0xfffff800036f2000, traced=0) at /usr/src/sys/amd64/amd64/trap.c:1029 #43 <signal handler called> #44 0x00000008002dee8a in ?? () I'm adding core.txt.2 for reference. Full core dump available on request.
Is there a specific situation happening when these panic()s occur? (such as startup, shutdown or ??) Basically, the panic() happens when the kernel RPC code is doing a connect on an AF_LOCAL socket created by the nfsuserd daemon to do an upcall. I'll admit I haven't tested this myself recently, but I'll try running nfsuserd on current. (prior to FreeBSD12, nfsuserd uses a UDP socket, so this panic() wouldn't happen) (Since unp_connectat() does a straightforward namei(), I don't know why there would be an "excl->shared" panic? The code is using LK_SHARED for the lookup, but that happens all the time.) I am wondering if somehow the AF_LOCAL socket got deleted and that confuses the namei()/lookup(). (The socket is /var/run/nfsuserd.sock.) If the panic()s are causing you grief, you can add the -use-udpsock command line option to nfsuserd to make it use the UDP socket, like FreeBSD-11.
Do you have /var/run NFS mounted by any chance?
Sorry, to clarify...I was wondering if the machine that panic()'d had its /var/run exported to NFS client(s) that would have it mounted via NFS. (If that was the case, it could be a case where the nfsd thread is doing a Getattr on /var/run/nfsuserd.sock and then it tries to connect to /var/run/nfsuserd.sock to do the upcall to the nfsuserd.) --> I think I'll need to make it still use UDP sockets by default (since this could happen when the socket is in the namespace).
(In reply to Rick Macklem from comment #3) > Sorry, to clarify...I was wondering if the machine that panic()'d > had its /var/run exported to NFS client(s) that would have it mounted via > NFS. > (If that was the case, it could be a case where the nfsd thread is doing > a Getattr on /var/run/nfsuserd.sock and then it tries to connect to > /var/run/nfsuserd.sock to do the upcall to the nfsuserd.) > --> I think I'll need to make it still use UDP sockets by default (since > this could happen when the socket is in the namespace). No, it's only exporting other directories. /etc/exports looks like this: /archive -maproot=nfsnobody:nfsnobody -network 192.168.0.1/24 /share -maproot=nfsnobody:nfsnobody -network 192.168.0.1/24 /usr/obj /usr/src -maproot=nfsnobody:nfsnobody -network 192.168.0.1/24 V4: / -network 192.168.0.1/24 The following nfs related settings are in /etc/rc.conf: nfs_client_enable="NO" # This host is an NFS client (or NO). nfs_server_enable="YES" # This host is an NFS server (or NO). nfsv4_server_enable="YES" # Enable support for NFSv4 nfscbd_enable="NO" # NFSv4 client side callback daemon nfsuserd_enable="YES" # NFSv4 user/group name mapping daemon
Hmm. I notice you do have the "V4: / …" line in your /etc/exports. Do you know if there are clients doing an NFSv4 mount from "/"? Something like: mount -t nfs -o nfsv4 nfs-client:/ /mnt I think I will end up reverting the "use AF_LOCAL" socket patch, since that is what is causing the soconnect() on the socket and should definitely make the panic()s go away. (It did fix a problem with using UDP when jails were enabled, but that can be fixed less elegantly with a patch that adds a command line argument for an alternate IP# instead of 127.0.0.1.)
A commit references this bug: Author: rmacklem Date: Wed Aug 22 12:20:10 UTC 2018 New revision: 338192 URL: https://svnweb.freebsd.org/changeset/base/338192 Log: Revert r320757 since it can cause "excl->shared" panics. PR#230752 shows a panic where an nfsd thread tries to do soconnect() on the AF_LOCAL socket used by the nfsuserd while already holding an exclusive lock on it. I am not 100% sure how this happens, but since an AF_LOCAL socket is in the file system namespace it is conceivable that it could lock it and then attempt an upcall to the nfsuserd. However, reverting r320757 stops the nfsuserd from using an AF_LOCAL socket, so it should avoid any such panic(). r320757 did fix a problem with running the nfsuserd when jails were enabled, but that can be dealt with less elegantly by allowing the use of an alternate address instead of 127.0.0.1. The gssd daemon also uses an AF_LOCAL socket, but it will do upcalls before the nfsd thread processes the RPC, so I think it should not be suseptible to this problem. PR: 230752 Changes: head/usr.sbin/nfsuserd/nfsuserd.c
A commit references this bug: Author: rmacklem Date: Wed Aug 22 12:26:18 UTC 2018 New revision: 338193 URL: https://svnweb.freebsd.org/changeset/base/338193 Log: Revert r320758, which was the man page update for r320757 just reverted. This is a content change. PR: 230752 Changes: head/usr.sbin/nfsuserd/nfsuserd.8
(In reply to Rick Macklem from comment #5) > Hmm. I notice you do have the "V4: / …" line in your /etc/exports. > Do you know if there are clients doing an NFSv4 mount from "/"? > Something like: > mount -t nfs -o nfsv4 nfs-client:/ /mnt Yes, most clients use NFSv4, via /etc/fstab lines: nfssrv:/archive /archive nfs rw,intr,soft,nfsv4 0 0 nfssrv:/share /share nfs rw,intr,soft,nfsv4 0 0 I migrated most of them from NFSv3 a month or two ago. This panic started appearing relatively recently, for example when doing buildworlds from /share (where freebsd sources are). > I think I will end up reverting the "use AF_LOCAL" socket patch, > since that is what is causing the soconnect() on the socket and > should definitely make the panic()s go away. > (It did fix a problem with using UDP when jails were enabled, but > that can be fixed less elegantly with a patch that adds a command > line argument for an alternate IP# instead of 127.0.0.1.) I'll check it out, thanks.
If you add the "-use-udpsock" command line option to nfsuserd it has the same effect as upgrading to r338192 or later. Btw, using "soft,intr" is not recommended for NFSv4 mounts. (I think it's in the BUGS section of mount_nfs(8).)
Unfortunately I had a crash again, this time with the clang700-import branch at r338392, which has been sync'd with head as of r338391, so after r338193. Unread portion of the kernel message buffer: shared lock of (lockmgr) ufs @ /usr/src/sys/kern/vfs_lookup.c:671 while exclusively locked from /usr/src/sys/kern/vfs_subr.c:2590 panic: excl->share cpuid = 3 time = 1535629184 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe000059c0e0 vpanic() at vpanic+0x1b4/frame 0xfffffe000059c140 panic() at panic+0x43/frame 0xfffffe000059c1a0 witness_checkorder() at witness_checkorder+0xbc4/frame 0xfffffe000059c230 lockmgr_slock_hard() at lockmgr_slock_hard+0x6e/frame 0xfffffe000059c2c0 __lockmgr_args() at __lockmgr_args+0x758/frame 0xfffffe000059c360 ffs_lock() at ffs_lock+0xa5/frame 0xfffffe000059c3b0 VOP_LOCK1_APV() at VOP_LOCK1_APV+0x92/frame 0xfffffe000059c3e0 _vn_lock() at _vn_lock+0x65/frame 0xfffffe000059c440 lookup() at lookup+0x100/frame 0xfffffe000059c4e0 namei() at namei+0x4bd/frame 0xfffffe000059c5a0 unp_connectat() at unp_connectat+0x155/frame 0xfffffe000059c830 soconnectat() at soconnectat+0xe8/frame 0xfffffe000059c880 clnt_vc_create() at clnt_vc_create+0x349/frame 0xfffffe000059c9c0 clnt_reconnect_call() at clnt_reconnect_call+0x20b/frame 0xfffffe000059ca70 newnfs_request() at newnfs_request+0x8fc/frame 0xfffffe000059cbe0 nfsrv_getuser() at nfsrv_getuser+0x129/frame 0xfffffe000059cd80 nfsrv_getgrpscred() at nfsrv_getgrpscred+0xca/frame 0xfffffe000059cdd0 nfsd_excred() at nfsd_excred+0x10e/frame 0xfffffe000059cdf0 nfsrvd_dorpc() at nfsrvd_dorpc+0x130f/frame 0xfffffe000059cfd0 nfssvc_program() at nfssvc_program+0x527/frame 0xfffffe000059d190 svc_run_internal() at svc_run_internal+0x9f9/frame 0xfffffe000059d2d0 svc_run() at svc_run+0x1ee/frame 0xfffffe000059d330 nfsrvd_nfsd() at nfsrvd_nfsd+0x356/frame 0xfffffe000059d490 nfssvc_nfsd() at nfssvc_nfsd+0x57a/frame 0xfffffe000059d960 sys_nfssvc() at sys_nfssvc+0xcf/frame 0xfffffe000059d980 amd64_syscall() at amd64_syscall+0x28a/frame 0xfffffe000059dab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe000059dab0 --- syscall (155, FreeBSD ELF64, sys_nfssvc), rip = 0x8002dee8a, rsp = 0x7fffffffe518, rbp = 0x7fffffffe7b0 --- KDB: enter: panic __curthread () at ./machine/pcpu.h:230 230 ./machine/pcpu.h: No such file or directory. (kgdb) bt #0 __curthread () at ./machine/pcpu.h:230 #1 doadump (textdump=<optimized out>) at /usr/src/sys/kern/kern_shutdown.c:366 #2 0xffffffff8044f16b in db_dump (dummy=<optimized out>, dummy2=<unavailable>, dummy3=<unavailable>, dummy4=<unavailable>) at /usr/src/sys/ddb/db_command.c:574 #3 0xffffffff8044ef39 in db_command (last_cmdp=<optimized out>, cmd_table=<optimized out>, dopager=1) at /usr/src/sys/ddb/db_command.c:481 #4 0xffffffff8044ecb4 in db_command_loop () at /usr/src/sys/ddb/db_command.c:534 #5 0xffffffff80451e6f in db_trap (type=<optimized out>, code=<optimized out>) at /usr/src/sys/ddb/db_main.c:252 #6 0xffffffff80be47f5 in kdb_trap (type=3, code=0, tf=<optimized out>) at /usr/src/sys/kern/subr_kdb.c:693 #7 0xffffffff8106d0c5 in trap (frame=0xfffffe000059c010) at /usr/src/sys/amd64/amd64/trap.c:619 #8 <signal handler called> #9 kdb_enter (why=0xffffffff812f5de0 "panic", msg=<optimized out>) at /usr/src/sys/kern/subr_kdb.c:479 #10 0xffffffff80b9c5c1 in vpanic (fmt=<optimized out>, ap=0xfffffe000059c180) at /usr/src/sys/kern/kern_shutdown.c:861 #11 0xffffffff80b9c353 in panic (fmt=0xffffffff81e8e998 <cnputs_mtx> "B\267+\201\377\377\377\377") at /usr/src/sys/kern/kern_shutdown.c:799 #12 0xffffffff80c04114 in witness_checkorder (lock=<optimized out>, flags=<optimized out>, file=<optimized out>, line=671, interlock=<optimized out>) at /usr/src/sys/kern/subr_witness.c:1176 #13 0xffffffff80b70fae in lockmgr_slock_hard (lk=<optimized out>, flags=2106368, ilk=0xfffff8000422c278, file=<optimized out>, line=<optimized out>, lwa=<optimized out>) at /usr/src/sys/kern/kern_lock.c:567 #14 0xffffffff80b71f48 in __lockmgr_args (lk=<optimized out>, flags=<optimized out>, ilk=0xfffff8000422c278, wmesg=<optimized out>, pri=<optimized out>, timo=<optimized out>, file=0xffffffff812f19eb "/usr/src/sys/kern/vfs_lookup.c", line=671) at /usr/src/sys/kern/kern_lock.c:1195 #15 0xffffffff80eb46e5 in _lockmgr_args (lk=0xfffff8000422c248, flags=2106368, ilk=<optimized out>, wmesg=<optimized out>, prio=0, timo=0, file=<optimized out>, line=18) at /usr/src/sys/sys/lockmgr.h:104 #16 ffs_lock (ap=0xfffffe000059c3f0) at /usr/src/sys/ufs/ffs/ffs_vnops.c:428 #17 0xffffffff811ec2c2 in VOP_LOCK1_APV (vop=0xffffffff81b635e0 <ffs_vnodeops2>, a=0xfffffe000059c3f0) at vnode_if.c:2087 #18 0xffffffff80c7e645 in VOP_LOCK1 (vp=<optimized out>, flags=2106368, file=0xffffffff812f19eb "/usr/src/sys/kern/vfs_lookup.c", line=671) at ./vnode_if.h:859 #19 _vn_lock (vp=0xfffff8000422c1e0, flags=2106368, file=<optimized out>, line=<optimized out>) at /usr/src/sys/kern/vfs_vnops.c:1531 #20 0xffffffff80c613c0 in lookup (ndp=0xfffffe000059c5b0) at /usr/src/sys/kern/vfs_lookup.c:669 #21 0xffffffff80c60ead in namei (ndp=0xfffffe000059c5b0) at /usr/src/sys/kern/vfs_lookup.c:450 #22 0xffffffff80c431b5 in unp_connectat (fd=<optimized out>, so=<optimized out>, nam=<optimized out>, td=0xfffff80003316580) at /usr/src/sys/kern/uipc_usrreq.c:1554 #23 0xffffffff80c358b8 in soconnectat (fd=<optimized out>, so=<optimized out>, nam=0x1030000, td=0xffffffff80c0327a <witness_init+138>) at /usr/src/sys/kern/uipc_socket.c:1230 #24 0xffffffff80e507a9 in clnt_vc_create (so=0xfffff80004cb96d0, raddr=0xfffff800041c7420, prog=553713921, vers=1, sendsz=4096, recvsz=4096, intrflag=0) at /usr/src/sys/rpc/clnt_vc.c:159 #25 0xffffffff80e4fa3b in clnt_reconnect_connect (cl=0xfffff8000386f5c0) at /usr/src/sys/rpc/clnt_rc.c:193 #26 clnt_reconnect_call (cl=0xfffff8000386f5c0, ext=0xfffffe000059cac0, proc=1, args=0xfffff8000480a200, resultsp=<optimized out>, utimeout=...) at /usr/src/sys/rpc/clnt_rc.c:265 #27 0xffffffff80a65cac in newnfs_request (nd=0xfffffe000059cc38, nmp=0x0, clp=0x0, nrp=0xffffffff82080298 <nfsrv_nfsuserdsock>, vp=0x0, td=0x0, cred=0xfffff80032c74700, prog=553713921, vers=1, retsum=0x0, toplevel=0, xidp=0x0, dssep=0x0) at /usr/src/sys/fs/nfs/nfs_commonkrpc.c:818 #28 0xffffffff80a6fa79 in nfsrv_getuser (procnum=1, uid=<optimized out>, gid=<optimized out>, name=0x0, p=0x0) at /usr/src/sys/fs/nfs/nfs_commonsubs.c:3616 #29 0xffffffff80a6fb9a in nfsrv_getgrpscred (oldcred=0xfffff80032c74800) at /usr/src/sys/fs/nfs/nfs_commonsubs.c:3150 #30 0xffffffff80ad16be in nfsd_excred (nd=0xfffffe000059d008, exp=<optimized out>, credanon=0xfffffe000059bfd0) at /usr/src/sys/fs/nfsserver/nfs_nfsdport.c:2932 #31 0xffffffff80aaa79f in nfsrvd_compound (nd=<optimized out>, isdgram=<optimized out>, tag=0x10 <error: Cannot access memory at address 0x10>, taglen=<optimized out>, minorvers=<optimized out>, p=<optimized out>) at /usr/src/sys/fs/nfsserver/nfs_nfsdsocket.c:1008 #32 nfsrvd_dorpc (nd=0xfffffe000059d008, isdgram=0, tag=0x10 <error: Cannot access memory at address 0x10>, taglen=7, minorvers=2, p=0xfffff80003316580) at /usr/src/sys/fs/nfsserver/nfs_nfsdsocket.c:560 #33 0xffffffff80abe597 in nfs_proc (xid=<optimized out>, xprt=<optimized out>, nd=<optimized out>, rpp=<optimized out>) at /usr/src/sys/fs/nfsserver/nfs_nfsdkrpc.c:387 #34 nfssvc_program (rqst=0xfffff80096076800, xprt=0xfffff8000369a800) at /usr/src/sys/fs/nfsserver/nfs_nfsdkrpc.c:272 #35 0xffffffff80e57789 in svc_executereq (rqstp=<optimized out>) at /usr/src/sys/rpc/svc.c:1031 #36 svc_run_internal (grp=<optimized out>, ismaster=1) at /usr/src/sys/rpc/svc.c:1306 #37 0xffffffff80e56cce in svc_run (pool=<optimized out>) at /usr/src/sys/rpc/svc.c:1385 #38 0xffffffff80abebf6 in nfsrvd_nfsd (td=<optimized out>, args=0xfffffe000059d520) at /usr/src/sys/fs/nfsserver/nfs_nfsdkrpc.c:542 #39 0xffffffff80ad4b0a in nfssvc_nfsd (td=0xfffff80003316580, uap=<optimized out>) at /usr/src/sys/fs/nfsserver/nfs_nfsdport.c:3451 #40 0xffffffff80e3572f in sys_nfssvc (td=0xfffff80003316580, uap=0xfffff80003316940) at /usr/src/sys/nfs/nfs_nfssvc.c:111 #41 0xffffffff8106de6a in syscallenter (td=0xfffff80003316580) at /usr/src/sys/amd64/amd64/../../kern/subr_syscall.c:135 #42 amd64_syscall (td=0xfffff80003316580, traced=0) at /usr/src/sys/amd64/amd64/trap.c:1043 #43 <signal handler called> #44 0x00000008002dee8a in ?? () Backtrace stopped: Cannot access memory at address 0x7fffffffe518
Created attachment 196722 [details] Text dump of newnfs_request -> witness panic 2
Are you sure you replaced the nfsuserd binary in /usr/sbin after the post-r338193 upgrade? (It was changes to nfsuserd.c that got reverted, not anything in the kernel.) rick
Fixed by reverting the change that made nfsuserd use an AF_LOCAL socket instead of a UDP one.