Bug 230752 - panic: excl->share in newnfs_request
Summary: panic: excl->share in newnfs_request
Status: In Progress
Alias: None
Product: Base System
Classification: Unclassified
Component: kern (show other bugs)
Version: CURRENT
Hardware: amd64 Any
: --- Affects Only Me
Assignee: freebsd-fs mailing list
URL:
Keywords: panic
Depends on:
Blocks:
 
Reported: 2018-08-19 17:22 UTC by Dimitry Andric
Modified: 2018-08-30 22:29 UTC (History)
3 users (show)

See Also:


Attachments
Text dump of newnfs_request -> witness panic (88.70 KB, text/plain)
2018-08-19 17:22 UTC, Dimitry Andric
no flags Details
Text dump of newnfs_request -> witness panic 2 (96.14 KB, text/plain)
2018-08-30 21:33 UTC, Dimitry Andric
no flags Details

Note You need to log in before you can comment on or make changes to this bug.
Description Dimitry Andric freebsd_committer 2018-08-19 17:22:13 UTC
Created attachment 196354 [details]
Text dump of newnfs_request -> witness panic

I recently got a few similar panics, seemingly originating from newnfs_request.  The panic goes like this:

shared lock of (lockmgr) ufs @ /usr/src/sys/kern/vfs_lookup.c:671
while exclusively locked from /usr/src/sys/kern/vfs_subr.c:2590
panic: excl->share
cpuid = 2
time = 1534686985
...
#1  doadump (textdump=<optimized out>)
    at /usr/src/sys/kern/kern_shutdown.c:366
#2  0xffffffff8044906b in db_dump (dummy=<optimized out>, 
    dummy2=<unavailable>, dummy3=<unavailable>, dummy4=<unavailable>)
    at /usr/src/sys/ddb/db_command.c:574
#3  0xffffffff80448e39 in db_command (last_cmdp=<optimized out>, 
    cmd_table=<optimized out>, dopager=1) at /usr/src/sys/ddb/db_command.c:481
#4  0xffffffff80448bb4 in db_command_loop ()
    at /usr/src/sys/ddb/db_command.c:534
#5  0xffffffff8044bd7f in db_trap (type=<optimized out>, code=<optimized out>)
    at /usr/src/sys/ddb/db_main.c:252
#6  0xffffffff80be79d5 in kdb_trap (type=3, code=0, tf=<optimized out>)
    at /usr/src/sys/kern/subr_kdb.c:693
#7  0xffffffff81084c3a in trap (frame=0xfffffe0000582fc0)
    at /usr/src/sys/amd64/amd64/trap.c:605
#8  <signal handler called>
#9  kdb_enter (why=0xffffffff8131179d "panic", msg=<optimized out>)
    at /usr/src/sys/kern/subr_kdb.c:479
#10 0xffffffff80b9d7b1 in vpanic (fmt=<optimized out>, ap=0xfffffe0000583130)
    at /usr/src/sys/kern/kern_shutdown.c:852
#11 0xffffffff80b9d550 in kassert_panic (fmt=0xffffffff812a3ed5 "excl->share")
    at /usr/src/sys/kern/kern_shutdown.c:749
#12 0xffffffff80c083a8 in witness_checkorder (lock=0xfffff80004190ba8, 
    flags=1, file=0xffffffff8130d3f0 "/usr/src/sys/kern/vfs_lookup.c", 
    line=<optimized out>, 
    interlock=0xffffffff81f64e2c <w_locklistdata+264684>)
    at /usr/src/sys/kern/subr_witness.c:1176
#13 0xffffffff80b70c3e in lockmgr_slock_hard (lk=<optimized out>, 
    flags=2106368, ilk=0xfffff80004190bd8, file=<optimized out>, 
    line=<optimized out>, lwa=<optimized out>)
    at /usr/src/sys/kern/kern_lock.c:567
#14 0xffffffff80b71d5b in __lockmgr_args (lk=<optimized out>, 
    flags=<optimized out>, ilk=<optimized out>, wmesg=<optimized out>, 
    pri=<optimized out>, timo=<optimized out>, 
    file=0xffffffff8130d3f0 "/usr/src/sys/kern/vfs_lookup.c", line=671)
    at /usr/src/sys/kern/kern_lock.c:1195
#15 0xffffffff80ec6a05 in _lockmgr_args (lk=0xfffff80004190ba8, 
    flags=2106368, ilk=<optimized out>, wmesg=<optimized out>, prio=0, 
    timo=0, file=<optimized out>, line=18) at /usr/src/sys/sys/lockmgr.h:104
#16 ffs_lock (ap=0xfffffe00005833c8) at /usr/src/sys/ufs/ffs/ffs_vnops.c:428
#17 0xffffffff81208059 in VOP_LOCK1_APV (
    vop=0xffffffff81b62d50 <ffs_vnodeops2>, a=0xfffffe00005833c8)
    at vnode_if.c:2087
#18 0xffffffff80c86927 in VOP_LOCK1 (vp=<optimized out>, flags=2106368, 
    file=<optimized out>, line=671) at ./vnode_if.h:859
#19 _vn_lock (vp=0xfffff80004190b40, flags=2106368, 
    file=0xffffffff8130d3f0 "/usr/src/sys/kern/vfs_lookup.c", line=671)
    at /usr/src/sys/kern/vfs_vnops.c:1531
#20 0xffffffff80c68fe6 in lookup (ndp=0xfffffe00005835a0)
    at /usr/src/sys/kern/vfs_lookup.c:669
#21 0xffffffff80c68aad in namei (ndp=0xfffffe00005835a0)
    at /usr/src/sys/kern/vfs_lookup.c:450
#22 0xffffffff80c49b85 in unp_connectat (fd=<optimized out>, 
    so=<optimized out>, nam=<optimized out>, td=0xfffff800036f2000)
    at /usr/src/sys/kern/uipc_usrreq.c:1554
#23 0xffffffff80c3bc98 in soconnectat (fd=<optimized out>, 
    so=<optimized out>, nam=0xfffff8000367d800, 
    td=0xffffffff80b7d390 <_mtx_init+144>)
    at /usr/src/sys/kern/uipc_socket.c:1230
#24 0xffffffff80e611b9 in clnt_vc_create (so=0xfffff80004c316d0, 
    raddr=0xfffff800035fc020, prog=553713921, vers=1, sendsz=4096, 
    recvsz=4096, intrflag=0) at /usr/src/sys/rpc/clnt_vc.c:159
#25 0xffffffff80e60439 in clnt_reconnect_connect (cl=0xfffff80003372840)
    at /usr/src/sys/rpc/clnt_rc.c:193
#26 clnt_reconnect_call (cl=0xfffff80003372840, ext=0xfffffe0000583ab0, 
    proc=1, args=0xfffff800048dda00, resultsp=0xfffffe0000583c28, 
    utimeout=...) at /usr/src/sys/rpc/clnt_rc.c:265
#27 0xffffffff80a637ec in newnfs_request (nd=0xfffffe0000583c28, nmp=0x0, 
    clp=0x0, nrp=0xffffffff82021a18 <nfsrv_nfsuserdsock>, vp=0x0, td=0x0, 
    cred=0xfffff80003e65000, prog=553713921, vers=1, retsum=0x0, toplevel=0, 
    xidp=0x0, dssep=0x0) at /usr/src/sys/fs/nfs/nfs_commonkrpc.c:818
#28 0xffffffff80a6d5f9 in nfsrv_getuser (procnum=1, uid=<optimized out>, 
    gid=<optimized out>, name=0x0, p=0xfffffe0000582ce0)
    at /usr/src/sys/fs/nfs/nfs_commonsubs.c:3616
#29 0xffffffff80a6d71a in nfsrv_getgrpscred (oldcred=0xfffff80003deec00)
    at /usr/src/sys/fs/nfs/nfs_commonsubs.c:3150
#30 0xffffffff80acf53e in nfsd_excred (nd=0xfffffe0000583ff8, 
    exp=<optimized out>, credanon=0xfffffe0000582f80)
    at /usr/src/sys/fs/nfsserver/nfs_nfsdport.c:2932
#31 0xffffffff80aa8104 in nfsrvd_compound (nd=<optimized out>, isdgram=0, 
    tag=0x10 <error: Cannot access memory at address 0x10>, 
    taglen=<optimized out>, minorvers=<optimized out>, p=<optimized out>)
    at /usr/src/sys/fs/nfsserver/nfs_nfsdsocket.c:1008
#32 nfsrvd_dorpc (nd=0xfffffe0000583ff8, isdgram=0, 
    tag=0x10 <error: Cannot access memory at address 0x10>, taglen=7, 
    minorvers=<optimized out>, p=0xfffff800036f2000)
    at /usr/src/sys/fs/nfsserver/nfs_nfsdsocket.c:560
#33 0xffffffff80abc3a7 in nfs_proc (xid=<optimized out>, 
    xprt=<optimized out>, nd=<optimized out>, rpp=<optimized out>)
    at /usr/src/sys/fs/nfsserver/nfs_nfsdkrpc.c:387
#34 nfssvc_program (rqst=0xfffff8004d1d5800, xprt=0xfffff800035fb600)
    at /usr/src/sys/fs/nfsserver/nfs_nfsdkrpc.c:272
#35 0xffffffff80e68499 in svc_executereq (rqstp=<optimized out>)
    at /usr/src/sys/rpc/svc.c:1031
#36 svc_run_internal (grp=<optimized out>, ismaster=1)
    at /usr/src/sys/rpc/svc.c:1306
#37 0xffffffff80e6785e in svc_run (pool=<optimized out>)
    at /usr/src/sys/rpc/svc.c:1385
#38 0xffffffff80abca06 in nfsrvd_nfsd (td=<optimized out>, 
    args=0xfffffe0000584510) at /usr/src/sys/fs/nfsserver/nfs_nfsdkrpc.c:542
#39 0xffffffff80ad298b in nfssvc_nfsd (td=0xfffff800036f2000, 
    uap=<optimized out>) at /usr/src/sys/fs/nfsserver/nfs_nfsdport.c:3451
#40 0xffffffff80e45eeb in sys_nfssvc (td=0xfffff800036f2000, 
    uap=0xfffff800036f23c0) at /usr/src/sys/nfs/nfs_nfssvc.c:111
#41 0xffffffff810859ef in syscallenter (td=0xfffff800036f2000)
    at /usr/src/sys/amd64/amd64/../../kern/subr_syscall.c:135
#42 amd64_syscall (td=0xfffff800036f2000, traced=0)
    at /usr/src/sys/amd64/amd64/trap.c:1029
#43 <signal handler called>
#44 0x00000008002dee8a in ?? ()

I'm adding core.txt.2 for reference.  Full core dump available on request.
Comment 1 Rick Macklem freebsd_committer 2018-08-19 23:25:24 UTC
Is there a specific situation happening when these panic()s occur?
(such as startup, shutdown or ??)

Basically, the panic() happens when the kernel RPC code is doing a connect
on an AF_LOCAL socket created by the nfsuserd daemon to do an upcall.
I'll admit I haven't tested this myself recently, but I'll try running
nfsuserd on current. (prior to FreeBSD12, nfsuserd uses a UDP socket,
so this panic() wouldn't happen)
(Since unp_connectat() does a straightforward namei(), I don't know why
 there would be an "excl->shared" panic? The code is using LK_SHARED
 for the lookup, but that happens all the time.)

I am wondering if somehow the AF_LOCAL socket got deleted and that
confuses the namei()/lookup(). (The socket is /var/run/nfsuserd.sock.)

If the panic()s are causing you grief, you can add the -use-udpsock
command line option to nfsuserd to make it use the UDP socket, like FreeBSD-11.
Comment 2 Rick Macklem freebsd_committer 2018-08-21 23:16:45 UTC
Do you have /var/run NFS mounted by any chance?
Comment 3 Rick Macklem freebsd_committer 2018-08-21 23:27:55 UTC
Sorry, to clarify...I was wondering if the machine that panic()'d
had its /var/run exported to NFS client(s) that would have it mounted via NFS.
(If that was the case, it could be a case where the nfsd thread is doing
 a Getattr on /var/run/nfsuserd.sock and then it tries to connect to
 /var/run/nfsuserd.sock to do the upcall to the nfsuserd.)
--> I think I'll need to make it still use UDP sockets by default (since
    this could happen when the socket is in the namespace).
Comment 4 Dimitry Andric freebsd_committer 2018-08-22 10:56:16 UTC
(In reply to Rick Macklem from comment #3)
> Sorry, to clarify...I was wondering if the machine that panic()'d
> had its /var/run exported to NFS client(s) that would have it mounted via
> NFS.
> (If that was the case, it could be a case where the nfsd thread is doing
>  a Getattr on /var/run/nfsuserd.sock and then it tries to connect to
>  /var/run/nfsuserd.sock to do the upcall to the nfsuserd.)
> --> I think I'll need to make it still use UDP sockets by default (since
>     this could happen when the socket is in the namespace).

No, it's only exporting other directories.  /etc/exports looks like this:

/archive -maproot=nfsnobody:nfsnobody -network 192.168.0.1/24
/share -maproot=nfsnobody:nfsnobody -network 192.168.0.1/24
/usr/obj /usr/src -maproot=nfsnobody:nfsnobody -network 192.168.0.1/24
V4: / -network 192.168.0.1/24

The following nfs related settings are in /etc/rc.conf:

nfs_client_enable="NO"		# This host is an NFS client (or NO).
nfs_server_enable="YES"		# This host is an NFS server (or NO).
nfsv4_server_enable="YES"	# Enable support for NFSv4
nfscbd_enable="NO"		# NFSv4 client side callback daemon
nfsuserd_enable="YES"		# NFSv4 user/group name mapping daemon
Comment 5 Rick Macklem freebsd_committer 2018-08-22 11:51:04 UTC
Hmm. I notice you do have the "V4: / …" line in your /etc/exports.
Do you know if there are clients doing an NFSv4 mount from "/"?
Something like:
  mount -t nfs -o nfsv4 nfs-client:/ /mnt

I think I will end up reverting the "use AF_LOCAL" socket patch,
since that is what is causing the soconnect() on the socket and
should definitely make the panic()s go away.
(It did fix a problem with using UDP when jails were enabled, but
 that can be fixed less elegantly with a patch that adds a command
 line argument for an alternate IP# instead of 127.0.0.1.)
Comment 6 commit-hook freebsd_committer 2018-08-22 12:20:20 UTC
A commit references this bug:

Author: rmacklem
Date: Wed Aug 22 12:20:10 UTC 2018
New revision: 338192
URL: https://svnweb.freebsd.org/changeset/base/338192

Log:
  Revert r320757 since it can cause "excl->shared" panics.

  PR#230752 shows a panic where an nfsd thread tries to do soconnect() on
  the AF_LOCAL socket used by the nfsuserd while already holding an
  exclusive lock on it. I am not 100% sure how this happens, but since an
  AF_LOCAL socket is in the file system namespace it is conceivable that it
  could lock it and then attempt an upcall to the nfsuserd.
  However, reverting r320757 stops the nfsuserd from using an AF_LOCAL
  socket, so it should avoid any such panic().
  r320757 did fix a problem with running the nfsuserd when jails were
  enabled, but that can be dealt with less elegantly by allowing the
  use of an alternate address instead of 127.0.0.1.
  The gssd daemon also uses an AF_LOCAL socket, but it will do upcalls
  before the nfsd thread processes the RPC, so I think it should not
  be suseptible to this problem.

  PR:		230752

Changes:
  head/usr.sbin/nfsuserd/nfsuserd.c
Comment 7 commit-hook freebsd_committer 2018-08-22 12:26:27 UTC
A commit references this bug:

Author: rmacklem
Date: Wed Aug 22 12:26:18 UTC 2018
New revision: 338193
URL: https://svnweb.freebsd.org/changeset/base/338193

Log:
  Revert r320758, which was the man page update for r320757 just reverted.

  This is a content change.

  PR:		230752

Changes:
  head/usr.sbin/nfsuserd/nfsuserd.8
Comment 8 Dimitry Andric freebsd_committer 2018-08-22 17:11:50 UTC
(In reply to Rick Macklem from comment #5)
> Hmm. I notice you do have the "V4: / …" line in your /etc/exports.
> Do you know if there are clients doing an NFSv4 mount from "/"?
> Something like:
>   mount -t nfs -o nfsv4 nfs-client:/ /mnt

Yes, most clients use NFSv4, via /etc/fstab lines:

nfssrv:/archive /archive	 nfs rw,intr,soft,nfsv4 0 0
nfssrv:/share /share nfs rw,intr,soft,nfsv4 0 0

I migrated most of them from NFSv3 a month or two ago.  This panic started appearing relatively recently, for example when doing buildworlds from /share (where freebsd sources are).


> I think I will end up reverting the "use AF_LOCAL" socket patch,
> since that is what is causing the soconnect() on the socket and
> should definitely make the panic()s go away.
> (It did fix a problem with using UDP when jails were enabled, but
>  that can be fixed less elegantly with a patch that adds a command
>  line argument for an alternate IP# instead of 127.0.0.1.)

I'll check it out, thanks.
Comment 9 Rick Macklem freebsd_committer 2018-08-22 23:35:03 UTC
If you add the "-use-udpsock" command line option to nfsuserd it
has the same effect as upgrading to r338192 or later.

Btw, using "soft,intr" is not recommended for NFSv4 mounts.
(I think it's in the BUGS section of mount_nfs(8).)
Comment 10 Dimitry Andric freebsd_committer 2018-08-30 21:27:35 UTC
Unfortunately I had a crash again, this time with the clang700-import branch at r338392, which has been sync'd with head as of r338391, so after r338193.

Unread portion of the kernel message buffer:
shared lock of (lockmgr) ufs @ /usr/src/sys/kern/vfs_lookup.c:671
while exclusively locked from /usr/src/sys/kern/vfs_subr.c:2590
panic: excl->share
cpuid = 3
time = 1535629184
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe000059c0e0
vpanic() at vpanic+0x1b4/frame 0xfffffe000059c140
panic() at panic+0x43/frame 0xfffffe000059c1a0
witness_checkorder() at witness_checkorder+0xbc4/frame 0xfffffe000059c230
lockmgr_slock_hard() at lockmgr_slock_hard+0x6e/frame 0xfffffe000059c2c0
__lockmgr_args() at __lockmgr_args+0x758/frame 0xfffffe000059c360
ffs_lock() at ffs_lock+0xa5/frame 0xfffffe000059c3b0
VOP_LOCK1_APV() at VOP_LOCK1_APV+0x92/frame 0xfffffe000059c3e0
_vn_lock() at _vn_lock+0x65/frame 0xfffffe000059c440
lookup() at lookup+0x100/frame 0xfffffe000059c4e0
namei() at namei+0x4bd/frame 0xfffffe000059c5a0
unp_connectat() at unp_connectat+0x155/frame 0xfffffe000059c830
soconnectat() at soconnectat+0xe8/frame 0xfffffe000059c880
clnt_vc_create() at clnt_vc_create+0x349/frame 0xfffffe000059c9c0
clnt_reconnect_call() at clnt_reconnect_call+0x20b/frame 0xfffffe000059ca70
newnfs_request() at newnfs_request+0x8fc/frame 0xfffffe000059cbe0
nfsrv_getuser() at nfsrv_getuser+0x129/frame 0xfffffe000059cd80
nfsrv_getgrpscred() at nfsrv_getgrpscred+0xca/frame 0xfffffe000059cdd0
nfsd_excred() at nfsd_excred+0x10e/frame 0xfffffe000059cdf0
nfsrvd_dorpc() at nfsrvd_dorpc+0x130f/frame 0xfffffe000059cfd0
nfssvc_program() at nfssvc_program+0x527/frame 0xfffffe000059d190
svc_run_internal() at svc_run_internal+0x9f9/frame 0xfffffe000059d2d0
svc_run() at svc_run+0x1ee/frame 0xfffffe000059d330
nfsrvd_nfsd() at nfsrvd_nfsd+0x356/frame 0xfffffe000059d490
nfssvc_nfsd() at nfssvc_nfsd+0x57a/frame 0xfffffe000059d960
sys_nfssvc() at sys_nfssvc+0xcf/frame 0xfffffe000059d980
amd64_syscall() at amd64_syscall+0x28a/frame 0xfffffe000059dab0
fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe000059dab0
--- syscall (155, FreeBSD ELF64, sys_nfssvc), rip = 0x8002dee8a, rsp = 0x7fffffffe518, rbp = 0x7fffffffe7b0 ---
KDB: enter: panic

__curthread () at ./machine/pcpu.h:230
230     ./machine/pcpu.h: No such file or directory.
(kgdb) bt
#0  __curthread () at ./machine/pcpu.h:230
#1  doadump (textdump=<optimized out>) at /usr/src/sys/kern/kern_shutdown.c:366
#2  0xffffffff8044f16b in db_dump (dummy=<optimized out>, dummy2=<unavailable>, dummy3=<unavailable>, dummy4=<unavailable>) at /usr/src/sys/ddb/db_command.c:574
#3  0xffffffff8044ef39 in db_command (last_cmdp=<optimized out>, cmd_table=<optimized out>, dopager=1) at /usr/src/sys/ddb/db_command.c:481
#4  0xffffffff8044ecb4 in db_command_loop () at /usr/src/sys/ddb/db_command.c:534
#5  0xffffffff80451e6f in db_trap (type=<optimized out>, code=<optimized out>) at /usr/src/sys/ddb/db_main.c:252
#6  0xffffffff80be47f5 in kdb_trap (type=3, code=0, tf=<optimized out>) at /usr/src/sys/kern/subr_kdb.c:693
#7  0xffffffff8106d0c5 in trap (frame=0xfffffe000059c010) at /usr/src/sys/amd64/amd64/trap.c:619
#8  <signal handler called>
#9  kdb_enter (why=0xffffffff812f5de0 "panic", msg=<optimized out>) at /usr/src/sys/kern/subr_kdb.c:479
#10 0xffffffff80b9c5c1 in vpanic (fmt=<optimized out>, ap=0xfffffe000059c180) at /usr/src/sys/kern/kern_shutdown.c:861
#11 0xffffffff80b9c353 in panic (fmt=0xffffffff81e8e998 <cnputs_mtx> "B\267+\201\377\377\377\377") at /usr/src/sys/kern/kern_shutdown.c:799
#12 0xffffffff80c04114 in witness_checkorder (lock=<optimized out>, flags=<optimized out>, file=<optimized out>, line=671, interlock=<optimized out>) at /usr/src/sys/kern/subr_witness.c:1176
#13 0xffffffff80b70fae in lockmgr_slock_hard (lk=<optimized out>, flags=2106368, ilk=0xfffff8000422c278, file=<optimized out>, line=<optimized out>, lwa=<optimized out>) at /usr/src/sys/kern/kern_lock.c:567
#14 0xffffffff80b71f48 in __lockmgr_args (lk=<optimized out>, flags=<optimized out>, ilk=0xfffff8000422c278, wmesg=<optimized out>, pri=<optimized out>, timo=<optimized out>,
    file=0xffffffff812f19eb "/usr/src/sys/kern/vfs_lookup.c", line=671) at /usr/src/sys/kern/kern_lock.c:1195
#15 0xffffffff80eb46e5 in _lockmgr_args (lk=0xfffff8000422c248, flags=2106368, ilk=<optimized out>, wmesg=<optimized out>, prio=0, timo=0, file=<optimized out>, line=18) at /usr/src/sys/sys/lockmgr.h:104
#16 ffs_lock (ap=0xfffffe000059c3f0) at /usr/src/sys/ufs/ffs/ffs_vnops.c:428
#17 0xffffffff811ec2c2 in VOP_LOCK1_APV (vop=0xffffffff81b635e0 <ffs_vnodeops2>, a=0xfffffe000059c3f0) at vnode_if.c:2087
#18 0xffffffff80c7e645 in VOP_LOCK1 (vp=<optimized out>, flags=2106368, file=0xffffffff812f19eb "/usr/src/sys/kern/vfs_lookup.c", line=671) at ./vnode_if.h:859
#19 _vn_lock (vp=0xfffff8000422c1e0, flags=2106368, file=<optimized out>, line=<optimized out>) at /usr/src/sys/kern/vfs_vnops.c:1531
#20 0xffffffff80c613c0 in lookup (ndp=0xfffffe000059c5b0) at /usr/src/sys/kern/vfs_lookup.c:669
#21 0xffffffff80c60ead in namei (ndp=0xfffffe000059c5b0) at /usr/src/sys/kern/vfs_lookup.c:450
#22 0xffffffff80c431b5 in unp_connectat (fd=<optimized out>, so=<optimized out>, nam=<optimized out>, td=0xfffff80003316580) at /usr/src/sys/kern/uipc_usrreq.c:1554
#23 0xffffffff80c358b8 in soconnectat (fd=<optimized out>, so=<optimized out>, nam=0x1030000, td=0xffffffff80c0327a <witness_init+138>) at /usr/src/sys/kern/uipc_socket.c:1230
#24 0xffffffff80e507a9 in clnt_vc_create (so=0xfffff80004cb96d0, raddr=0xfffff800041c7420, prog=553713921, vers=1, sendsz=4096, recvsz=4096, intrflag=0) at /usr/src/sys/rpc/clnt_vc.c:159
#25 0xffffffff80e4fa3b in clnt_reconnect_connect (cl=0xfffff8000386f5c0) at /usr/src/sys/rpc/clnt_rc.c:193
#26 clnt_reconnect_call (cl=0xfffff8000386f5c0, ext=0xfffffe000059cac0, proc=1, args=0xfffff8000480a200, resultsp=<optimized out>, utimeout=...) at /usr/src/sys/rpc/clnt_rc.c:265
#27 0xffffffff80a65cac in newnfs_request (nd=0xfffffe000059cc38, nmp=0x0, clp=0x0, nrp=0xffffffff82080298 <nfsrv_nfsuserdsock>, vp=0x0, td=0x0, cred=0xfffff80032c74700, prog=553713921, vers=1, retsum=0x0,
    toplevel=0, xidp=0x0, dssep=0x0) at /usr/src/sys/fs/nfs/nfs_commonkrpc.c:818
#28 0xffffffff80a6fa79 in nfsrv_getuser (procnum=1, uid=<optimized out>, gid=<optimized out>, name=0x0, p=0x0) at /usr/src/sys/fs/nfs/nfs_commonsubs.c:3616
#29 0xffffffff80a6fb9a in nfsrv_getgrpscred (oldcred=0xfffff80032c74800) at /usr/src/sys/fs/nfs/nfs_commonsubs.c:3150
#30 0xffffffff80ad16be in nfsd_excred (nd=0xfffffe000059d008, exp=<optimized out>, credanon=0xfffffe000059bfd0) at /usr/src/sys/fs/nfsserver/nfs_nfsdport.c:2932
#31 0xffffffff80aaa79f in nfsrvd_compound (nd=<optimized out>, isdgram=<optimized out>, tag=0x10 <error: Cannot access memory at address 0x10>, taglen=<optimized out>, minorvers=<optimized out>,
    p=<optimized out>) at /usr/src/sys/fs/nfsserver/nfs_nfsdsocket.c:1008
#32 nfsrvd_dorpc (nd=0xfffffe000059d008, isdgram=0, tag=0x10 <error: Cannot access memory at address 0x10>, taglen=7, minorvers=2, p=0xfffff80003316580) at /usr/src/sys/fs/nfsserver/nfs_nfsdsocket.c:560
#33 0xffffffff80abe597 in nfs_proc (xid=<optimized out>, xprt=<optimized out>, nd=<optimized out>, rpp=<optimized out>) at /usr/src/sys/fs/nfsserver/nfs_nfsdkrpc.c:387
#34 nfssvc_program (rqst=0xfffff80096076800, xprt=0xfffff8000369a800) at /usr/src/sys/fs/nfsserver/nfs_nfsdkrpc.c:272
#35 0xffffffff80e57789 in svc_executereq (rqstp=<optimized out>) at /usr/src/sys/rpc/svc.c:1031
#36 svc_run_internal (grp=<optimized out>, ismaster=1) at /usr/src/sys/rpc/svc.c:1306
#37 0xffffffff80e56cce in svc_run (pool=<optimized out>) at /usr/src/sys/rpc/svc.c:1385
#38 0xffffffff80abebf6 in nfsrvd_nfsd (td=<optimized out>, args=0xfffffe000059d520) at /usr/src/sys/fs/nfsserver/nfs_nfsdkrpc.c:542
#39 0xffffffff80ad4b0a in nfssvc_nfsd (td=0xfffff80003316580, uap=<optimized out>) at /usr/src/sys/fs/nfsserver/nfs_nfsdport.c:3451
#40 0xffffffff80e3572f in sys_nfssvc (td=0xfffff80003316580, uap=0xfffff80003316940) at /usr/src/sys/nfs/nfs_nfssvc.c:111
#41 0xffffffff8106de6a in syscallenter (td=0xfffff80003316580) at /usr/src/sys/amd64/amd64/../../kern/subr_syscall.c:135
#42 amd64_syscall (td=0xfffff80003316580, traced=0) at /usr/src/sys/amd64/amd64/trap.c:1043
#43 <signal handler called>
#44 0x00000008002dee8a in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fffffffe518
Comment 11 Dimitry Andric freebsd_committer 2018-08-30 21:33:43 UTC
Created attachment 196722 [details]
Text dump of newnfs_request -> witness panic 2
Comment 12 Rick Macklem freebsd_committer 2018-08-30 22:29:08 UTC
Are you sure you replaced the nfsuserd binary in /usr/sbin
after the post-r338193 upgrade?
(It was changes to nfsuserd.c that got reverted, not anything in
 the kernel.)

rick