FreeBSD dumped core - see /var/crash/vmcore.3

Mon May  9 14:01:04 CEST 2016

FreeBSD FreeBSD 10.3-RELEASE-p2 FreeBSD 10.3-RELEASE-p2 #0: Wed May  4 06:03:51 UTC 2016     root@amd64-builder.daemonology.net:/usr/obj/usr/src/sys/GENERIC  amd64

panic: spin lock held too long

GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for details.
This GDB was configured as "amd64-marcel-freebsd"...

Unread portion of the kernel message buffer:
spin lock 0xffffffff816a6200 (smp rendezvous) held by 0xfffff80066083960 (tid 100086) too long
panic: spin lock held too long
cpuid = 2
KDB: stack backtrace:
#0 0xffffffff8098e390 at kdb_backtrace+0x60
#1 0xffffffff80951066 at vpanic+0x126
#2 0xffffffff80950f33 at panic+0x43
#3 0xffffffff80937097 at _mtx_lock_spin_cookie+0x287
#4 0xffffffff80d44cd1 at smp_tlb_shootdown+0x91
#5 0xffffffff80d46b7c at pmap_invalidate_range+0x2fc
#6 0xffffffff809df4c0 at allocbuf+0x5d0
#7 0xffffffff809e0360 at getnewbuf+0x510
#8 0xffffffff809dd6b1 at getblk+0x571
#9 0xffffffff809de19d at breadn_flags+0x2d
#10 0xffffffff80b860f1 at ffs_update+0x101
#11 0xffffffff80baef67 at ffs_sync+0x577
#12 0xffffffff809fb426 at sync_fsync+0x136
#13 0xffffffff80e81af7 at VOP_FSYNC_APV+0xa7
#14 0xffffffff809fbe1b at sched_sync+0x3ab
#15 0xffffffff8091a4ea at fork_exit+0x9a
#16 0xffffffff80d3be0e at fork_trampoline+0xe
Uptime: 39m20s
Dumping 251 out of 2023 MB:..7%..13%..26%..32%..45%..51%..64%..77%..83%..96%

#0  doadump (textdump=<value optimized out>) at pcpu.h:219
219     pcpu.h: No such file or directory.
        in pcpu.h
(kgdb) #0  doadump (textdump=<value optimized out>) at pcpu.h:219
#1  0xffffffff80950cc2 in kern_reboot (howto=260)
    at /usr/src/sys/kern/kern_shutdown.c:486
#2  0xffffffff809510a5 in vpanic (fmt=<value optimized out>, 
    ap=<value optimized out>) at /usr/src/sys/kern/kern_shutdown.c:889
#3  0xffffffff80950f33 in panic (fmt=0x0)
    at /usr/src/sys/kern/kern_shutdown.c:818
#4  0xffffffff80937097 in _mtx_lock_spin_cookie (c=<value optimized out>, 
    tid=<value optimized out>, opts=<value optimized out>, 
    file=<value optimized out>, line=<value optimized out>)
    at /usr/src/sys/kern/kern_mutex.c:568
#5  0xffffffff80d44cd1 in smp_tlb_shootdown (vector=246, 
    pmap=0xffffffff816cd5e0, addr1=18446741876820213760, 
    addr2=18446741876820217856) at /usr/src/sys/amd64/amd64/mp_machdep.c:1164
#6  0xffffffff80d46b7c in pmap_invalidate_range (pmap=<value optimized out>, 
    sva=<value optimized out>, eva=<value optimized out>)
    at /usr/src/sys/amd64/amd64/pmap.c:1515
#7  0xffffffff809df4c0 in allocbuf (bp=0xfffffe007b9f9848, size=0)
    at /usr/src/sys/kern/vfs_bio.c:4334
#8  0xffffffff809e0360 in getnewbuf (maxsize=<value optimized out>, 
    gbflags=<value optimized out>) at /usr/src/sys/kern/vfs_bio.c:2198
#9  0xffffffff809dd6b1 in getblk (vp=0xfffff800029eace8, blkno=3847040, 
    size=32768, slpflag=0, slptimeo=0, flags=<value optimized out>)
    at /usr/src/sys/kern/vfs_bio.c:3275
#10 0xffffffff809de19d in breadn_flags (vp=0xfffff800029eace8, blkno=0, 
    size=0, rablkno=0x0, rabsize=0x0, cnt=0, cred=0xfffffe0093505868, 
    flags=0, bpp=0xfffffe0093505868) at /usr/src/sys/kern/vfs_bio.c:1130
#11 0xffffffff80b860f1 in ffs_update (vp=0xfffff80002f47938, waitfor=0)
    at /usr/src/sys/ufs/ffs/ffs_inode.c:111
#12 0xffffffff80baef67 in ffs_sync (mp=0xfffff800029d1330, waitfor=Cannot access memory at address 0x1
)
    at /usr/src/sys/ufs/ffs/ffs_vfsops.c:1461
#13 0xffffffff809fb426 in sync_fsync (ap=<value optimized out>)
    at /usr/src/sys/kern/vfs_subr.c:3857
#14 0xffffffff80e81af7 in VOP_FSYNC_APV (vop=<value optimized out>, 
    a=<value optimized out>) at vnode_if.c:1330
#15 0xffffffff809fbe1b in sched_sync () at vnode_if.h:549
#16 0xffffffff8091a4ea in fork_exit (callout=0xffffffff809fba70 <sched_sync>, 
    arg=0x0, frame=0xfffffe0093505ac0) at /usr/src/sys/kern/kern_fork.c:1027
#17 0xffffffff80d3be0e in fork_trampoline ()
    at /usr/src/sys/amd64/amd64/exception.S:611
#18 0x0000000000000000 in ?? ()
Current language:  auto; currently minimal
(kgdb) 

------------------------------------------------------------------------
ps -axlww

UID   PID  PPID CPU PRI NI   VSZ               RSS MWCHAN   STAT TT     TIME COMMAND
  0     0     0   0  -8  0     0 18014389993764064 -        DLs   -  0:00.02 [kernel]
  0     1     0   0  20  0  9492 17979214314216448 wait     DLs   -  0:00.01 [init]
  0     2     0   0 -16  0     0 18014389993764064 -        DL    -  0:00.00 [cam]
  0     3     0   0 -16  0     0 18014389993764064 waiting_ DL    -  0:00.00 [sctp_iterator]
  0     4     0   0 -16  0     0 18014389993764064 umarcl   DL    -  0:00.00 [pagedaemon]
  0     5     0   0 -16  0     0 18014389993764064 psleep   DL    -  0:00.00 [vmdaemon]
  0     6     0   0 155  0     0 18014389993764064 -        RL    -  0:00.00 [pagezero]
  0     7     0   0  20  0     0 18014389993764064 -        RL    -  0:00.00 [bufdaemon]
  0     8     0   0 -16  0     0 18014389993764064 vlruwt   DL    -  0:00.00 [vnlru]
  0     9     0   0  20  0     0 18014389993764064 -        RL    -  0:00.00 [syncer]
  0    10     0   0 -16  0     0 18014389993764064 audit_wo DL    -  0:00.00 [audit]
  0    11     0   0 155  0     0 18014389993764064 -        RL    -  0:44.90 [idle]
  0    12     0   0 -84  0     0 18014389993764064 -        WL    -  0:00.12 [intr]
  0    13     0   0  -8  0     0 18014389993764064 -        DL    -  0:00.03 [geom]
  0    14     0   0 -16  0     0 18014389993764064 -        DL    -  0:00.01 [rand_harvestq]
  0   279     1   0  20  0 13628 17979214316326400 select   Ds    -  0:00.00 [devd]
  0   353     1   0  20  0 14520                 0 -        Rs    -  0:00.01 [syslogd]
  0   546     1   0  52  0 61316 17979214320012800 select   Ds    -  0:00.00 [sshd]
  0   549     1   0  20  0 24156 17979214293337856 -        Rs    -  0:00.01 [sendmail]
 25   552     1   0  20  0 24156 17979220983913472 pause    Ds    -  0:00.00 [sendmail]
  0   556     1   0  20  0 16624                 0 nanslp   Ds    -  0:00.00 [cron]
  0   601     1   0  52  0 14520 17979214314205440 ttyin    Ds+   -  0:00.01 [getty]
  0   602     1   0  52  0 14520 17979214317587968 ttyin    Ds+   -  0:00.01 [getty]
  0   603     1   0  52  0 14520                 0 ttyin    Ds+   -  0:00.01 [getty]
  0   604     1   0  52  0 14520 17979214314185472 ttyin    Ds+   -  0:00.01 [getty]
  0   605     1   0  52  0 14520 17979214315914240 ttyin    Ds+   -  0:00.01 [getty]
  0   606     1   0  52  0 14520 17979220984702976 ttyin    Ds+   -  0:00.01 [getty]
  0   607     1   0  52  0 14520 17979214336314880 ttyin    Ds+   -  0:00.01 [getty]
  0   608     1   0  52  0 14520 17979214336311808 ttyin    Ds+   -  0:00.01 [getty]
  0   609     1   0  20  0 47756 17979214322250752 wait     Ds    -  0:00.01 [login]
  0   613   609   0  20  0 23600 17979214314201856 pause    D     -  0:00.02 [csh]
  0   623   613   0  52  0 17096 17979214314220032 wait     D+    -  0:00.00 [sh]
  0  4209   623   0  20  0 12364 17979214319719936 pipewr   D+    -  0:00.00 [cat]
  0  4210   623   0  52  0 17096 17979214317556992 wait     D+    -  0:00.00 [sh]
  0 36316  4210   0  75  0 30060 17979214293359616 -        R+    -  0:00.00 [bsdtar]
  0 36317   556   0  20  0 16624 17979214338635776 -        R     -  0:00.00 [cron]
  0 36318 36317   0  20  0 16624 17979214338635776 -        RVs   -  0:00.00 [cron]
  0 36319   556   0  20  0 16624 17979221008731648 -        R     -  0:00.00 [cron]
  0 36320 36319   0  20  0 16624 17979221008731648 ufs      DVs   -  0:00.00 [cron]
  0 36321   556   0  20  0 16624 17979214314018304 ppwait   D     -  0:00.00 [cron]
  0 36322 36321   0  20  0 16624 17979214314018304 ufs      DVs   -  0:00.00 [cron]
  0 36323   556   0  20  0 16624 17979214338625024 -        R     -  0:00.00 [cron]
  0 36324 36323   0  20  0 16624 17979214338625024 ufs      DVs   -  0:00.00 [cron]
  0 36325   556   0  20  0 16624                 0 -        R     -  0:00.00 [cron]
  0 36326 36325   0  20  0 16624                 0 ufs      DVs   -  0:00.00 [cron]
  0 36327   556   0  20  0 16624 17979214319737856 -        R     -  0:00.00 [cron]
  0 36328 36327   0  20  0 16624 17979214319737856 ufs      DVs   -  0:00.00 [cron]
  0 36329   549   0  20  0 24156                 0 -        R     -  0:00.00 [sendmail]
 25 36330   552   0  20  0 24156                 0 -        R     -  0:00.00 [sendmail]
  0 36331   556   0  20  0 16624 17979214293332224 -        R     -  0:00.00 [cron]
  0 36332 36331   0  20  0 16624 17979214293332224 ufs      DVs   -  0:00.00 [cron]
  0 36333   556   0  20  0 16624 17979214293346816 -        R     -  0:00.00 [cron]
  0 36334   556   0  20  0 16624 17979214293364992 -        R     -  0:00.00 [cron]
  0 36335 36333   0  20  0 16624 17979214293346816 ufs      DVs   -  0:00.00 [cron]
  0 36336 36334   0  20  0 16624 17979214293364992 ufs      DVs   -  0:00.00 [cron]

------------------------------------------------------------------------
vmstat -s

   570294 cpu context switches
  5835517 device interrupts
    89010 software interrupts
  3700077 traps
  5735779 system calls
       14 kernel threads created
     1199  fork() calls
    35123 vfork() calls
        0 rfork() calls
        0 swap pager pageins
        0 swap pager pages paged in
        0 swap pager pageouts
        0 swap pager pages paged out
      510 vnode pager pageins
     3478 vnode pager pages paged in
        0 vnode pager pageouts
        0 vnode pager pages paged out
        0 page daemon wakeups
        0 pages examined by the page daemon
        0 pages reactivated
   997656 copy-on-write faults
      169 copy-on-write optimized faults
  2726779 zero fill pages zeroed
        0 zero fill pages prezeroed
        9 intransit blocking page faults
  3749493 total VM faults taken
      393 page faults requiring I/O
        0 pages affected by kernel thread creation
    43231 pages affected by  fork()
  1229398 pages affected by vfork()
        0 pages affected by rfork()
        0 pages cached
  4589913 pages freed
        0 pages freed by daemon
        0 pages freed by exiting processes
     3784 pages active
    81020 pages inactive
        0 pages in VM cache
    48296 pages wired down
   371514 pages free
     4096 bytes per page
  2147680 total name lookups
          cache hits (82% pos + 5% neg) system 0% per-directory
          deletions 5%, falsehits 0%, toolong 0%

------------------------------------------------------------------------
vmstat -m

         Type InUse MemUse HighUse Requests  Size(s)
    CAM queue     2     1K       -        5  16,512
         cdev     4     1K       -        4  256
     acpitask     1    16K       -        1  16384
     filedesc    55   110K       -    36337  2048
        sigio     1     1K       -        1  64
      kdtrace   161    30K       -    36443  64,256
         kenv    75    11K       -       77  16,32,64,128,8192
       kqueue     0     0K       -       34  256,2048
    proc-args    31     2K       -    36238  16,32,64,128,256
        hhook     2     1K       -        2  256
      ithread    92    16K       -       92  32,128,256
       KTRACE   100    13K       -      100  128
       linker   144    10K       -      146  16,32,64,512
      acpisem    17     3K       -       17  128
        lockf    17     2K       -       61  64,128
   loginclass     3     1K       -       15  64
       devbuf  1293  1256K       -     1313  16,32,64,128,256,512,1024,2048,4096,65536
         temp     9     1K       -    72924  16,32,64,128,256,1024,2048,16384
       ip6ndp     3     1K       -        3  64,128
       module   480    60K       -      481  128
     mtx_pool     2    16K       -        2  8192
     pmchooks     1     1K       -        1  128
         pgrp    27     4K       -       33  128
      session    25     4K       -       29  128
         proc     2    16K       -        2  8192
      subproc   128   257K       -    36410  512,4096
         cred    48     8K       -   306524  64,256
       feeder     7     1K       -        7  32
       plimit    14     4K       -       20  256
      uidinfo     3     3K       -        6  128,2048
CAM dev queue     1     1K       -        1  64
       sysctl     0     0K       -    12050  16,32,64
    sysctloid  1561    79K       -     1594  16,32,64,128
    sysctltmp     0     0K       -      119  16,32,64,128
      tidhash     1    16K       -        1  16384
      callout     5  2184K       -        5  
         umtx   234    30K       -      234  128
     p1003.1b     1     1K       -        1  16
         SWAP     2   141K       -        2  64
          bus   966    76K       -     2378  16,32,64,128,256,1024
       bus-sc    28    23K       -      908  16,32,64,128,256,512,1024,2048,4096,8192,16384,32768,65536
      devstat     6    13K       -        6  32,4096
 eventhandler    84     7K       -       84  64,128
         kobj   326  1304K       -      518  4096
      Per-cpu     1     1K       -        1  32
       DEVFS3    90    23K       -      109  256
       DEVFS1    76    38K       -       85  512
         rman    97    12K       -      513  32,128
         sbuf     0     0K       -      590  16,32,64,128,256,512,1024,2048,4096,8192,32768
       sglist     3     2K       -        3  32,128,1024
   DEVFS_RULE    55    26K       -       55  64,512
        DEVFS    10     1K       -       11  16,128
    taskqueue    19     3K       -       19  16,32,256
       Unitno    21     2K       -     1647  32,64
         vmem     3    38K       -        9  1024,2048,4096,8192,16384,32768
     ioctlops     0     0K       -       76  256,512,1024
       select    10     2K       -       10  128
          iov     2     1K       -   278282  64,128,256,512
          msg     4    30K       -        4  2048,4096,8192,16384
          sem     4   106K       -        4  2048,4096
          shm     1    32K       -        1  32768
          tty    19    19K       -       19  1024
     mbuf_tag     0     0K       -        9  32
        shmfd     1     8K       -        1  8192
       soname     4     1K       -     5287  16,32,128
          pcb    17   407K       -       49  16,32,128,1024,2048,8192
     vfscache     1  1024K       -        1  
   cl_savebuf     0     0K       -       84  64
     vfs_hash     1   512K       -        1  
       vnodes     1     1K       -        1  256
        mount    16     1K       -       86  16,32,64,128,256
  vnodemarker     1     1K       -       52  512
          BPF     2     1K       -        2  128
        ifnet     3     5K       -        3  128,2048
       ifaddr    28    12K       -       28  32,64,128,256,512,2048,4096
  ether_multi    17     1K       -       18  16,32,64
        clone     7     1K       -        7  128
       arpcom     1     1K       -        1  16
      lltable     6     3K       -        6  256,512
      acpidev    17     2K       -       17  64
NFSD V4client     1     1K       -        1  256
 NFSD lckfile     1     1K       -        1  256
     routetbl    32     6K       -       96  32,64,128,256,512
         igmp     2     1K       -        2  256
     in_multi     2     1K       -        2  256
    sctp_a_it     0     0K       -        3  16
     sctp_vrf     1     1K       -        1  64
     sctp_ifa     4     1K       -        4  128
     sctp_ifn     2     1K       -        2  128
    sctp_iter     0     0K       -        3  256
    hostcache     1    32K       -        1  32768
     syncache     1    64K       -        1  65536
    in6_multi    15     2K       -       15  32,256
          mld     2     1K       -        2  128
          rpc     2    16K       -        2  8192
audit_evclass   187     6K       -      231  32
      pagedep  1085   399K       -    18781  256
     inodedep  5477  3250K       -    86709  512
    bmsafemap     2     9K       -    65594  256,8192
       newblk  5096  2298K       -    69009  256
     indirdep     1     1K       -       71  128,32768
     freefrag     0     0K       -      347  128
     freeblks     0     0K       -    22785  256
     freefile     0     0K       -    22794  64
       diradd  5475   685K       -    66113  128
        mkdir  2499   313K       -    28190  128
       dirrem     0     0K       -    23377  128
    newdirblk  1080    68K       -    14159  64
     freework     1     1K       -    24070  16,128
      freedep     0     0K       -       24  64
      jaddref   155    20K       -    94306  128
      jremref     0     0K       -    32447  128
       jmvref     0     0K       -       20  128
      jnewblk   123    16K       -    69008  128
    jfreefrag     0     0K       -      347  128
         jseg    44     6K       -     5136  128
      jsegdep 10622   664K       -   196108  64
        sbdep     0     0K       -       21  64
     savedino     0     0K       -   113089  256
      jblocks     2     1K       -        2  128,256
      softdep     1     1K       -        1  512
  ufs_dirhash    78    40K       -      153  16,32,64,128,256,512,8192,16384
    ufs_quota     1   512K       -        1  
    ufs_mount     3    13K       -        3  512,4096,8192
    vm_pgdata     2   257K       -        2  128
      UMAHash     4     7K       -        9  512,1024,2048,4096
 NFSD session     1     1K       -        1  1024
     atkbddev     2     1K       -        2  64
    pfs_nodes    21     6K       -       21  256
         GEOM    49     9K       -      622  16,32,64,128,256,512,1024,2048,8192,16384
      CAM SIM     1     1K       -        1  256
      CAM XPT    10     1K       -       13  32,64,128
         UART     6     5K       -        6  16,1024
    raid_data     0     0K       -       90  32,128,256
      CAM DEV     1     2K       -        2  2048
     pci_link    16     2K       -       16  64,128
     CAM path     1     1K       -       11  32
       kbdmux     6    22K       -        6  16,512,1024,2048,16384
       apmdev     1     1K       -        1  128
   madt_table     0     0K       -        1  4096
      io_apic     1     2K       -        1  2048
     acpiintr     1     1K       -        1  64
       acpica   378    35K       -    19510  16,32,64,128,256,512,1024
          MCA     4     1K       -        4  128
      entropy  1026    65K       -     1028  32,64,4096
   CAM periph     2     1K       -       12  16,32,64,128,256
          msi    13     2K       -       13  128
     nexusdev     3     1K       -        3  16
       isadev     7     1K       -        7  128

------------------------------------------------------------------------
vmstat -z

ITEM                   SIZE  LIMIT     USED     FREE      REQ FAIL SLEEP

UMA Kegs:               384,      0,     104,       6,     104,   0,   0
UMA Zones:             1152,      0,     104,       1,     104,   0,   0
UMA Slabs:               80,      0,    3278,      22,    3313,   0,   0
UMA RCntSlabs:           88,      0,    1268,      37,    1268,   0,   0
UMA Hash:               256,      0,       4,      11,       8,   0,   0
4 Bucket:                32,      0,      28,    1097,    1087,   0,   0
6 Bucket:                48,      0,       0,       0,       0,   0,   0
8 Bucket:                64,      0,      24,    1030,    6496,  11,   0
12 Bucket:               96,      0,      17,     516,      50,   0,   0
16 Bucket:              128,      0,      51,     476,     481,   0,   0
32 Bucket:              256,      0,     433,     437,    7066,  50,   0
64 Bucket:              512,      0,     218,     222,    9450,  60,   0
128 Bucket:            1024,      0,     218,     110,    5773,   0,   0
256 Bucket:            2048,      0,     155,      57,    1023,   6,   0
vmem btag:               56,      0,    3304,    1311,    7565,  33,   0
VM OBJECT:              256,      0,   75342,     123,  563711,   0,   0
RADIX NODE:             144,      0,    5418,     144,  764024,   0,   0
MAP:                    240,      0,       3,      61,       3,   0,   0
KMAP ENTRY:             128,      0,       4,     151,       4,   0,   0
MAP ENTRY:              128,      0,     729,     542, 1348222,   0,   0
VMSPACE:                448,      0,      32,     175,   36315,   0,   0
fakepg:                 104,      0,       0,       0,       0,   0,   0
mt_zone:               4112,      0,     363,       0,     363,   0,   0
16:                      16,      0,     876,    1132,   22204,   0,   0
32:                      32,      0,     960,    1165,    6145,   0,   0
64:                      64,      0,   14922,   17008,  761700,   0,   0
128:                    128,      0,   10315,   16562,  362054,   0,   0
256:                    256,      0,    6598,    8477,  455664,   0,   0
512:                    512,      0,    5748,   11020,   87506,   0,   0
1024:                  1024,      0,     105,      91,   37627,   0,   0
2048:                  2048,      0,      70,      16,   36420,   0,   0
4096:                  4096,      0,     403,       0,   36948,   0,   0
8192:                  8192,      0,      14,       2,     143,   0,   0
16384:                16384,      0,       6,       3,     137,   0,   0
32768:                32768,      0,       3,       6,      88,   0,   0
65536:                65536,      0,       4,       0,      34,   0,   0
SLEEPQUEUE:              80,      0,     118,     409,     118,   0,   0
64 pcpu:                  8,      0,    1532,     516,    1532,   0,   0
Files:                   80,      0,     102,     798,  281304,   0,   0
TURNSTILE:              136,      0,     118,     202,     118,   0,   0
rl_entry:                40,      0,      59,    1041,      59,   0,   0
umtx pi:                 96,      0,       0,       0,       0,   0,   0
MAC labels:              40,      0,       0,       0,       0,   0,   0
PROC:                  1264,      0,      54,      18,   36336,   0,   0
THREAD:                1192,      0,     105,      12,     105,   0,   0
cpuset:                  72,      0,      48,     227,      48,   0,   0
audit_record:          1248,      0,       0,       0,       0,   0,   0
mbuf_packet:            256, 807390,    1024,    1517,    6698,   0,   0
mbuf:                   256, 807390,      17,     753,    2947,   0,   0
mbuf_cluster:          2048, 126154,    2530,       6,    2530,   0,   0
mbuf_jumbo_page:       4096,  63076,       0,       0,       0,   0,   0
mbuf_jumbo_9k:         9216,  56067,       0,       0,       0,   0,   0
mbuf_jumbo_16k:       16384,  42048,       0,       0,       0,   0,   0
mbuf_ext_refcnt:          4,      0,       0,       0,       0,   0,   0
DMAR_MAP_ENTRY:         120,      0,       0,       0,       0,   0,   0
ttyoutq:                256,      0,      72,     183,     168,   0,   0
g_bio:                  248,      0,       0,    2352,  362761,   0,   0
ttyinq:                 160,      0,     135,     115,     315,   0,   0
nvme_request:           128,      0,       0,       0,       0,   0,   0
vtnet_tx_hdr:            24,      0,       0,    1169,    2662,   0,   0
FPU_save_area:          832,      0,       0,       0,       0,   0,   0
VNODE:                  472,      0,   74830,      50,   97626,   0,   0
VNODEPOLL:              112,      0,       0,       0,       0,   0,   0
BUF TRIE:               144,      0,      59,   13576,    8514,   0,   0
S VFS Cache:            108,      0,   11410,     245,  128027,   0,   0
STS VFS Cache:          148,      0,       0,       0,       0,   0,   0
L VFS Cache:            328,      0,   11055,      33,   15959,   0,   0
LTS VFS Cache:          368,      0,       0,       0,       0,   0,   0
NAMEI:                 1024,      0,       9,      71,  982759,   0,   0
DIRHASH:               1024,      0,    1236,      40,    1451,   0,   0
NCLNODE:                528,      0,       0,       0,       0,   0,   0
pipe:                   744,      0,      19,      36,     923,   0,   0
procdesc:               128,      0,       0,       0,       0,   0,   0
Mountpoints:            816,      0,       2,      13,       2,   0,   0
ksiginfo:               112,      0,      66,     984,      91,   0,   0
itimer:                 352,      0,       0,       0,       0,   0,   0
KNOTE:                  128,      0,       0,     527,      38,   0,   0
socket:                 696,  64755,      12,      43,    1001,   0,   0
unpcb:                  240,  64768,       7,     249,     888,   0,   0
ipq:                     56,   3976,       0,       0,       0,   0,   0
udp_inpcb:              392,  64760,       2,     118,      96,   0,   0
udpcb:                   24,  64796,       2,    1167,      96,   0,   0
tcp_inpcb:              392,  64760,       3,     117,      12,   0,   0
tcpcb:                 1032,  64755,       3,      33,      12,   0,   0
tcptw:                   88,  12960,       0,     135,       4,   0,   0
syncache:               160,  15375,       0,       0,       0,   0,   0
hostcache:              136,  15370,       1,      86,       1,   0,   0
sackhole:                32,      0,       0,       0,       0,   0,   0
tcpreass:                40,   7900,       0,       0,       0,   0,   0
sctp_ep:               1424,  64756,       0,       0,       0,   0,   0
sctp_asoc:             2416,  40000,       0,       0,       0,   0,   0
sctp_laddr:              48,  80012,       0,     332,       3,   0,   0
sctp_raddr:             728,  80000,       0,       0,       0,   0,   0
sctp_chunk:             136, 400026,       0,       0,       0,   0,   0
sctp_readq:             104, 400026,       0,       0,       0,   0,   0
sctp_stream_msg_out:    104, 400026,       0,       0,       0,   0,   0
sctp_asconf:             40, 400000,       0,       0,       0,   0,   0
sctp_asconf_ack:         48, 400060,       0,       0,       0,   0,   0
udplite_inpcb:          392,  64760,       0,       0,       0,   0,   0
ripcb:                  392,  64760,       0,       0,       0,   0,   0
rtentry:                200,      0,      13,     187,      13,   0,   0
selfd:                   56,      0,      18,    1047,     842,   0,   0
SWAPMETA:               288, 252317,       0,       0,       0,   0,   0
FFS inode:              176,      0,   74801,     109,   97595,   0,   0
FFS1 dinode:            128,      0,       0,       0,       0,   0,   0
FFS2 dinode:            256,      0,   74801,      49,   97595,   0,   0


------------------------------------------------------------------------
vmstat -i

interrupt                          total       rate
irq4: uart0                        38047        760
irq256: hpet0:t0                 1902731      38054
irq257: hpet0:t1                   60864       1217
irq258: hpet0:t2                 1878551      37571
irq259: hpet0:t3                 1892737      37854
irq265: virtio_pci0                61133       1222
irq267: virtio_pci1                 1454         29
Total                            5835517     116710

------------------------------------------------------------------------
pstat -T

102/64755 files
0M/1023M swap space

------------------------------------------------------------------------
pstat -s

Device          512-blocks     Used    Avail Capacity
/dev/vtbd0p3       2096856        0  2096856     0%

------------------------------------------------------------------------
iostat

iostat: kvm_read(_tk_nin): invalid address (0x0)
iostat: disabling TTY statistics
           vtbd0             cpu
  KB/t tps  MB/s  us ni sy in id
  6.59 2385 15.35   1  0 66  0 33

------------------------------------------------------------------------
ipcs -a

Message Queues:
T           ID          KEY MODE        OWNER    GROUP    CREATOR  CGROUP                 CBYTES                 QNUM               QBYTES        LSPID        LRPID STIME    RTIME    CTIME   

Shared Memory:
T           ID          KEY MODE        OWNER    GROUP    CREATOR  CGROUP         NATTCH        SEGSZ         CPID         LPID ATIME    DTIME    CTIME   

Semaphores:
T           ID          KEY MODE        OWNER    GROUP    CREATOR  CGROUP          NSEMS OTIME    CTIME   


------------------------------------------------------------------------
ipcs -T

msginfo:
        msgmax:        16384    (max characters in a message)
        msgmni:           40    (# of message queues)
        msgmnb:         2048    (max characters in a message queue)
        msgtql:           40    (max # of messages in system)
        msgssz:            8    (size of a message segment)
        msgseg:         2048    (# of message segments in system)

shminfo:
        shmmax:    536870912    (max shared memory segment size)
        shmmin:            1    (min shared memory segment size)
        shmmni:          192    (max number of shared memory identifiers)
        shmseg:          128    (max shared memory segments per process)
        shmall:       131072    (max amount of shared memory in pages)

seminfo:
        semmni:           50    (# of semaphore identifiers)
        semmns:          340    (# of semaphores in system)
        semmnu:          150    (# of undo structures in system)
        semmsl:          340    (max # of semaphores per id)
        semopm:          100    (max # of operations per semop call)
        semume:           50    (max # of undo entries per process)
        semusz:          632    (size in bytes of undo structure)
        semvmx:        32767    (semaphore maximum value)
        semaem:        16384    (adjust on exit max value)


------------------------------------------------------------------------
nfsstat

Client Info:
Rpc Counts:
  Getattr   Setattr    Lookup  Readlink      Read     Write    Create    Remove
        0         0         0         0         0         0         0         0
   Rename      Link   Symlink     Mkdir     Rmdir   Readdir  RdirPlus    Access
        0         0         0         0         0         0         0         0
    Mknod    Fsstat    Fsinfo  PathConf    Commit
        0         0         0         0         0
Rpc Info:
 TimedOut   Invalid X Replies   Retries  Requests
        0         0         0         0         0
Cache Info:
Attr Hits    Misses Lkup Hits    Misses BioR Hits    Misses BioW Hits    Misses
        0         0         0         0         0         0         0         0
BioRLHits    Misses BioD Hits    Misses DirE Hits    Misses Accs Hits    Misses
        0         0         0         0         0         0         0         0

Server Info:
  Getattr   Setattr    Lookup  Readlink      Read     Write    Create    Remove
        0         0         0         0         0         0         0         0
   Rename      Link   Symlink     Mkdir     Rmdir   Readdir  RdirPlus    Access
        0         0         0         0         0         0         0         0
    Mknod    Fsstat    Fsinfo  PathConf    Commit
        0         0         0         0         0
Server Ret-Failed
                0
Server Faults
            0
Server Cache Stats:
   Inprog      Idem  Non-idem    Misses
        0         0         0         0
Server Write Gathering:
 WriteOps  WriteRPC   Opsaved
        0         0         0

------------------------------------------------------------------------
netstat -s

tcp:
        2640 packets sent
                123 data packets (154666 bytes)
                0 data packets (0 bytes) retransmitted
                0 data packets unnecessarily retransmitted
                0 resends initiated by MTU discovery
                2503 ack-only packets (30 delayed)
                0 URG only packets
                0 window probe packets
                0 window update packets
                14 control packets
        5051 packets received
                75 acks (for 154673 bytes)
                17 duplicate acks
                0 acks for unsent data
                4995 packets (6855167 bytes) received in-sequence
                2 completely duplicate packets (0 bytes)
                0 old duplicate packets
                0 packets with some dup. data (0 bytes duped)
                0 out-of-order packets (0 bytes)
                0 packets (0 bytes) of data after window
                0 window probes
                13 window update packets
                0 packets received after close
                0 discarded for bad checksums
                0 discarded for bad header offset fields
                0 discarded because packet too short
                0 discarded due to memory problems
        7 connection requests
        0 connection accepts
        0 bad connection attempts
        0 listen queue overflows
        0 ignored RSTs in the windows
        7 connections established (including accepts)
        9 connections closed (including 0 drops)
                4 connections updated cached RTT on close
                4 connections updated cached RTT variance on close
                0 connections updated cached ssthresh on close
        0 embryonic connections dropped
        75 segments updated rtt (of 53 attempts)
        0 retransmit timeouts
                0 connections dropped by rexmit timeout
        0 persist timeouts
                0 connections dropped by persist timeout
        0 Connections (fin_wait_2) dropped because of timeout
        0 keepalive timeouts
                0 keepalive probes sent
                0 connections dropped by keepalive
        0 correct ACK header predictions
        4944 correct data packet header predictions
        0 syncache entries added
                0 retransmitted
                0 dupsyn
                0 dropped
                0 completed
                0 bucket overflow
                0 cache overflow
                0 reset
                0 stale
                0 aborted
                0 badack
                0 unreach
                0 zone failures
        0 cookies sent
        0 cookies received
        1 hostcache entry added
                0 bucket overflow
        0 SACK recovery episodes
        0 segment rexmits in SACK recovery episodes
        0 byte rexmits in SACK recovery episodes
        0 SACK options (SACK blocks) received
        0 SACK options (SACK blocks) sent
        0 SACK scoreboard overflow
        0 packets with ECN CE bit set
        0 packets with ECN ECT(0) bit set
        0 packets with ECN ECT(1) bit set
        0 successful ECN handshakes
        0 times ECN reduced the congestion window
        0 packets with valid tcp-md5 signature received
        0 packets with invalid tcp-md5 signature received
        0 packets with tcp-md5 signature mismatch
        0 packets with unexpected tcp-md5 signature received
        0 packets without expected tcp-md5 signature received
udp:
        20 datagrams received
        0 with incomplete header
        0 with bad data length field
        0 with bad checksum
        0 with no checksum
        0 dropped due to no socket
        0 broadcast/multicast datagrams undelivered
        0 dropped due to full socket buffers
        0 not for hashed pcb
        20 delivered
        20 datagrams output
        0 times multicast source filter matched
ip:
        5071 total packets received
        0 bad header checksums
        0 with size smaller than minimum
        0 with data size < data length
        0 with ip length > max ip packet size
        0 with header length < data size
        0 with data length < header length
        0 with bad options
        0 with incorrect version number
        0 fragments received
        0 fragments dropped (dup or out of space)
        0 fragments dropped after timeout
        0 packets reassembled ok
        5071 packets for this host
        0 packets for unknown/unsupported protocol
        0 packets forwarded (0 packets fast forwarded)
        0 packets not forwardable
        0 packets received for unknown multicast group
        0 redirects sent
        2660 packets sent from this host
        0 packets sent with fabricated ip header
        0 output packets dropped due to no bufs, etc.
        0 output packets discarded due to no route
        0 output datagrams fragmented
        0 fragments created
        0 datagrams that can't be fragmented
        0 tunneling packets that can't find gif
        0 datagrams with bad address in header
icmp:
        0 calls to icmp_error
        0 errors not generated in response to an icmp message
        0 messages with bad code fields
        0 messages less than the minimum length
        0 messages with bad checksum
        0 messages with bad length
        0 multicast echo requests ignored
        0 multicast timestamp requests ignored
        0 message responses generated
        0 invalid return addresses
        0 no return routes
arp:
        2 ARP requests sent
        0 ARP replies sent
        15 ARP requests received
        1 ARP reply received
        16 ARP packets received
        0 total packets dropped due to no ARP entry
        0 ARP entrys timed out
        0 Duplicate IPs seen
ip6:
        0 total packets received
        0 with size smaller than minimum
        0 with data size < data length
        0 with bad options
        0 with incorrect version number
        0 fragments received
        0 fragments dropped (dup or out of space)
        0 fragments dropped after timeout
        0 fragments that exceeded limit
        0 packets reassembled ok
        0 packets for this host
        0 packets forwarded
        0 packets not forwardable
        0 redirects sent
        0 packets sent from this host
        0 packets sent with fabricated ip header
        0 output packets dropped due to no bufs, etc.
        0 output packets discarded due to no route
        0 output datagrams fragmented
        0 fragments created
        0 datagrams that can't be fragmented
        0 packets that violated scope rules
        0 multicast packets which we don't join
        Mbuf statistics:
                0 one mbuf
                0 one ext mbuf
                0 two or more ext mbuf
        0 packets whose headers are not contiguous
        0 tunneling packets that can't find gif
        0 packets discarded because of too many headers
        0 failures of source address selection
        Source addresses selection rule applied:
icmp6:
        0 calls to icmp6_error
        0 errors not generated in response to an icmp6 message
        0 errors not generated because of rate limitation
        0 messages with bad code fields
        0 messages < minimum length
        0 bad checksums
        0 messages with bad length
        Histogram of error messages to be generated:
                0 no route
                0 administratively prohibited
                0 beyond scope
                0 address unreachable
                0 port unreachable
                0 packet too big
                0 time exceed transit
                0 time exceed reassembly
                0 erroneous header field
                0 unrecognized next header
                0 unrecognized option
                0 redirect
                0 unknown
        0 message responses generated
        0 messages with too many ND options
        0 messages with bad ND options
        0 bad neighbor solicitation messages
        0 bad neighbor advertisement messages
        0 bad router solicitation messages
        0 bad router advertisement messages
        0 bad redirect messages
        0 path MTU changes
rip6:
        0 messages received
        0 checksum calculations on inbound
        0 messages with bad checksum
        0 messages dropped due to no socket
        0 multicast messages dropped due to no socket
        0 messages dropped due to full socket buffers
        0 delivered
        0 datagrams output

------------------------------------------------------------------------
netstat -m

1041/2270/3311 mbufs in use (current/cache/total)
1013/1523/2536/126154 mbuf clusters in use (current/cache/total/max)
1024/1517 mbuf+clusters out of packet secondary zone in use (current/cache)
0/0/0/63076 4k (page size) jumbo clusters in use (current/cache/total/max)
0/0/0/56067 9k jumbo clusters in use (current/cache/total/max)
0/0/0/42048 16k jumbo clusters in use (current/cache/total/max)
2286K/3613K/5899K bytes allocated to network (current/cache/total)
0/0/0 requests for mbufs denied (mbufs/clusters/mbuf+clusters)
0/0/0 requests for mbufs delayed (mbufs/clusters/mbuf+clusters)
0/0/0 requests for jumbo clusters delayed (4k/9k/16k)
0/0/0 requests for jumbo clusters denied (4k/9k/16k)

------------------------------------------------------------------------
netstat -anr

Routing tables

Internet:
Destination        Gateway            Flags      Netif Expire
default            192.168.11.1       UGS      vtnet0
127.0.0.1          link#2             UH          lo0
192.168.11.0/24    link#1             U        vtnet0
192.168.11.240     link#1             UHS         lo0

Internet6:
Destination                       Gateway                       Flags      Netif Expire
::/96                             ::1                           UGRS        lo0
::1                               link#2                        UH          lo0
::ffff:0.0.0.0/96                 ::1                           UGRS        lo0
fe80::/10                         ::1                           UGRS        lo0
fe80::%lo0/64                     link#2                        U           lo0
fe80::1%lo0                       link#2                        UHS         lo0
ff01::%lo0/32                     ::1                           U           lo0
ff02::/16                         ::1                           UGRS        lo0
ff02::%lo0/32                     ::1                           U           lo0

------------------------------------------------------------------------
netstat -anA

Active Internet connections (including servers)
Tcpcb            Proto Recv-Q Send-Q Local Address      Foreign Address    (state)
fffff80066171810 tcp4       0      0 127.0.0.1.25       *.*                LISTEN
fffff8006602e408 tcp4       0      0 *.22               *.*                LISTEN
fffff8006602e810 tcp6       0      0 *.22               *.*                LISTEN
fffff80002be6930 udp4       0      0 *.514              *.*                
fffff80002be6ab8 udp6       0      0 *.514              *.*                
Active UNIX domain sockets
Address  Type   Recv-Q Send-Q    Inode     Conn     Refs  Nextref Addr
fffff80002bf9870 stream      0      0 fffff80002b54b10        0        0        0 /var/run/devd.pipe
fffff800660181e0 dgram       0      0        0 fffff800660182d0        0 fffff80002f3c5a0
fffff80002f22a50 dgram       0      0        0 fffff80002f22b40        0        0
fffff80002f3c5a0 dgram       0      0        0 fffff800660182d0        0        0
fffff800660182d0 dgram     753      0 fffff80002ffcb10        0 fffff800660181e0        0 /var/run/logpriv
fffff80002f22b40 dgram       0      0 fffff80002ffcce8        0 fffff80002f22a50        0 /var/run/log
fffff80002bf9780 seqpac      0      0 fffff80002b54938        0        0        0 /var/run/devd.seqpacket.pipe

------------------------------------------------------------------------
netstat -aL

Current listen queue sizes (qlen/incqlen/maxqlen)
Proto Listen         Local Address         
tcp4  0/0/10         localhost.smtp         
tcp4  0/0/128        *.ssh                  
tcp6  0/0/128        *.ssh                  
unix  0/0/4          /var/run/devd.pipe
unix  0/0/4          /var/run/devd.seqpacket.pipe

------------------------------------------------------------------------
fstat

fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 5 at 0x400000000
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 5 at 0x400000000
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 5 at 0x400000000
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 5 at 0x400000000
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 5 at 0x400000000
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 5 at 0x400000000
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 5 at 0x400000000
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 5 at 0x400000000
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 5 at 0x400000000
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 5 at 0x1a00000000
fstat: can't read file 7 at 0x200007fffffffff
fstat: can't read file 8 at 0x4000000001fffff
fstat: can't read file 10 at 0x780000ffff
fstat: can't read file 5 at 0xe400000000
fstat: can't read file 11 at 0x8000000000
fstat: can't read file 17 at 0x8000000000
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 5 at 0x600000000
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 5 at 0x400000000
fstat: can't read file 7 at 0x200007fffffffff
fstat: can't read file 8 at 0x4000000001fffff
fstat: can't read file 10 at 0x780000ffff
fstat: can't read file 11 at 0x400000000
fstat: can't read file 13 at 0x200007fffffffff
fstat: can't read file 14 at 0x4000000001fffff
fstat: can't read file 16 at 0x780000ffff
fstat: can't read file 17 at 0x400000000
fstat: can't read file 19 at 0x200007fffffffff
fstat: can't read file 1 at 0x200007fffffffff
fstat: can't read file 2 at 0x4000000001fffff
fstat: can't read file 4 at 0x780000ffff
fstat: can't read file 5 at 0x800000000
USER     CMD          PID   FD MOUNT      INUM MODE         SZ|DV R/W
root     cron       36336 root /             2 drwxr-xr-x    1024  r
root     cron       36336   wd /        1123585 drwxr-x---     512  r
root     cron       36336 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36336    0* pipe fffff80002b0dba0 <-> fffff80002b0dd00      0 rw
root     cron       36335 root /             2 drwxr-xr-x    1024  r
root     cron       36335   wd /        1123585 drwxr-x---     512  r
root     cron       36335 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36335    0* pipe fffff80002ad6000 <-> fffff80002ad6160      0 rw
root     cron       36334 root /             2 drwxr-xr-x    1024  r
root     cron       36334   wd /        1123585 drwxr-x---     512  r
root     cron       36334 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36334    0 /dev         13 crw-rw-rw-    null rw
root     cron       36334    6 /dev         13 crw-rw-rw-    null rw
root     cron       36333 root /             2 drwxr-xr-x    1024  r
root     cron       36333   wd /        1123585 drwxr-x---     512  r
root     cron       36333 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36333    0 /dev         13 crw-rw-rw-    null rw
root     cron       36333    6 /dev         13 crw-rw-rw-    null rw
root     cron       36332 root /             2 drwxr-xr-x    1024  r
root     cron       36332   wd /        1123585 drwxr-x---     512  r
root     cron       36332 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36332    0* pipe fffff80002b7e000 <-> fffff80002b7e160      0 rw
root     cron       36331 root /             2 drwxr-xr-x    1024  r
root     cron       36331   wd /        1123585 drwxr-x---     512  r
root     cron       36331 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36331    0 /dev         13 crw-rw-rw-    null rw
root     cron       36331    6 /dev         13 crw-rw-rw-    null rw
smmsp    sendmail   36330 root /             2 drwxr-xr-x    1024  r
smmsp    sendmail   36330   wd /        1124045 drwxrwx---     512  r
smmsp    sendmail   36330 text /        2410709 -r-xr-sr-x  696896  r
smmsp    sendmail   36330    0 /dev         13 crw-rw-rw-    null  r
root     sendmail   36329 root /             2 drwxr-xr-x    1024  r
root     sendmail   36329   wd /        1124048 drwxr-xr-x     512  r
root     sendmail   36329 text /        2410709 -r-xr-sr-x  696896  r
root     sendmail   36329    0 /dev         13 crw-rw-rw-    null  r
root     cron       36328 root /             2 drwxr-xr-x    1024  r
root     cron       36328   wd /        1123585 drwxr-x---     512  r
root     cron       36328 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36328    0* pipe fffff80002ad7ba0 <-> fffff80002ad7d00      0 rw
root     cron       36327 root /             2 drwxr-xr-x    1024  r
root     cron       36327   wd /        1123585 drwxr-x---     512  r
root     cron       36327 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36327    0 /dev         13 crw-rw-rw-    null rw
root     cron       36327    6 /dev         13 crw-rw-rw-    null rw
root     cron       36326 root /             2 drwxr-xr-x    1024  r
root     cron       36326   wd /        1123585 drwxr-x---     512  r
root     cron       36326 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36326    0* pipe fffff80002add5d0 <-> fffff80002add730      0 rw
root     cron       36325 root /             2 drwxr-xr-x    1024  r
root     cron       36325   wd /        1123585 drwxr-x---     512  r
root     cron       36325 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36325    0 /dev         13 crw-rw-rw-    null rw
root     cron       36325    6 /dev         13 crw-rw-rw-    null rw
root     cron       36324 root /             2 drwxr-xr-x    1024  r
root     cron       36324   wd /        1123585 drwxr-x---     512  r
root     cron       36324 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36324    0* pipe fffff80002add000 <-> fffff80002add160      0 rw
root     cron       36323 root /             2 drwxr-xr-x    1024  r
root     cron       36323   wd /        1123585 drwxr-x---     512  r
root     cron       36323 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36323    0 /dev         13 crw-rw-rw-    null rw
root     cron       36323    6 /dev         13 crw-rw-rw-    null rw
root     cron       36322 root /             2 drwxr-xr-x    1024  r
root     cron       36322   wd /        1123585 drwxr-x---     512  r
root     cron       36322 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36322    0* pipe fffff80002b7f5d0 <-> fffff80002b7f730      0 rw
root     cron       36321 root /             2 drwxr-xr-x    1024  r
root     cron       36321   wd /        1123585 drwxr-x---     512  r
root     cron       36321 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36321    0 /dev         13 crw-rw-rw-    null rw
root     cron       36321    6 /dev         13 crw-rw-rw-    null rw
root     cron       36320 root /             2 drwxr-xr-x    1024  r
root     cron       36320   wd /        1123585 drwxr-x---     512  r
root     cron       36320 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36320    0* pipe fffff80002bb5000 <-> fffff80002bb5160      0 rw
root     cron       36319 root /             2 drwxr-xr-x    1024  r
root     cron       36319   wd /        1123585 drwxr-x---     512  r
root     cron       36319 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36319    0 /dev         13 crw-rw-rw-    null rw
root     cron       36319    6 /dev         13 crw-rw-rw-    null rw
root     cron       36318 root /             2 drwxr-xr-x    1024  r
root     cron       36318   wd /        1123585 drwxr-x---     512  r
root     cron       36318 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36318    0* pipe fffff80002ad75d0 <-> fffff80002ad7730      0 rw
root     cron       36317 root /             2 drwxr-xr-x    1024  r
root     cron       36317   wd /        1123585 drwxr-x---     512  r
root     cron       36317 text /        2329942 -r-xr-xr-x   41392  r
root     cron       36317    0 /dev         13 crw-rw-rw-    null rw
root     cron       36317    6 /dev         13 crw-rw-rw-    null rw
root     bsdtar     36316 root /             2 drwxr-xr-x    1024  r
root     bsdtar     36316   wd /        579116 drwxr-xr-x     512  r
root     bsdtar     36316 text /        2328578 -r-xr-xr-x   58392  r
root     bsdtar     36316 ctty /dev         39 crw-------   ttyu0 rw
root     bsdtar     36316    0* pipe fffff80002b572e8 <-> fffff80002b57448  40509 rw
root     sh          4210 root /             2 drwxr-xr-x    1024  r
root     sh          4210   wd /        1123613 drwxr-xr-x     512  r
root     sh          4210 text /        401873 -r-xr-xr-x  145760  r
root     sh          4210 ctty /dev         39 crw-------   ttyu0 rw
root     sh          4210    0* pipe fffff80002b572e8 <-> fffff80002b57448  40509 rw
root     cat         4209 root /             2 drwxr-xr-x    1024  r
root     cat         4209   wd /        1123613 drwxr-xr-x     512  r
root     cat         4209 text /        401589 -r-xr-xr-x   11960  r
root     cat         4209 ctty /dev         39 crw-------   ttyu0 rw
root     cat         4209    0 /dev         39 crw-------   ttyu0 rw
root     sh           623 root /             2 drwxr-xr-x    1024  r
root     sh           623   wd /        1123613 drwxr-xr-x     512  r
root     sh           623 text /        401873 -r-xr-xr-x  145760  r
root     sh           623 ctty /dev         39 crw-------   ttyu0 rw
root     sh           623    0 /dev         39 crw-------   ttyu0 rw
root     sh           623    6 /dev         39 crw-------   ttyu0 rw
root     csh          613 root /             2 drwxr-xr-x    1024  r
root     csh          613   wd /        642048 drwxr-xr-x     512  r
root     csh          613 text /        401845 -r-xr-xr-x  382376  r
root     csh          613 ctty /dev         39 crw-------   ttyu0 rw
root     login        609 root /             2 drwxr-xr-x    1024  r
root     login        609   wd /        642048 drwxr-xr-x     512  r
root     login        609 text /        2328725 -r-sr-xr-x   25904  r
root     login        609 ctty /dev         39 crw-------   ttyu0 rw
root     login        609    0 /dev         39 crw-------   ttyu0 rw
root     getty        608 root /             2 drwxr-xr-x    1024  r
root     getty        608   wd /             2 drwxr-xr-x    1024  r
root     getty        608 text /        2329103 -r-xr-xr-x   28360  r
root     getty        608 ctty /dev         58 crw-------   ttyv7 rw
root     getty        608    0 /dev         58 crw-------   ttyv7 rw
root     getty        607 root /             2 drwxr-xr-x    1024  r
root     getty        607   wd /             2 drwxr-xr-x    1024  r
root     getty        607 text /        2329103 -r-xr-xr-x   28360  r
root     getty        607 ctty /dev         57 crw-------   ttyv6 rw
root     getty        607    0 /dev         57 crw-------   ttyv6 rw
root     getty        606 root /             2 drwxr-xr-x    1024  r
root     getty        606   wd /             2 drwxr-xr-x    1024  r
root     getty        606 text /        2329103 -r-xr-xr-x   28360  r
root     getty        606 ctty /dev         56 crw-------   ttyv5 rw
root     getty        606    0 /dev         56 crw-------   ttyv5 rw
root     getty        605 root /             2 drwxr-xr-x    1024  r
root     getty        605   wd /             2 drwxr-xr-x    1024  r
root     getty        605 text /        2329103 -r-xr-xr-x   28360  r
root     getty        605 ctty /dev         55 crw-------   ttyv4 rw
root     getty        605    0 /dev         55 crw-------   ttyv4 rw
root     getty        604 root /             2 drwxr-xr-x    1024  r
root     getty        604   wd /             2 drwxr-xr-x    1024  r
root     getty        604 text /        2329103 -r-xr-xr-x   28360  r
root     getty        604 ctty /dev         54 crw-------   ttyv3 rw
root     getty        604    0 /dev         54 crw-------   ttyv3 rw
root     getty        603 root /             2 drwxr-xr-x    1024  r
root     getty        603   wd /             2 drwxr-xr-x    1024  r
root     getty        603 text /        2329103 -r-xr-xr-x   28360  r
root     getty        603 ctty /dev         53 crw-------   ttyv2 rw
root     getty        603    0 /dev         53 crw-------   ttyv2 rw
root     getty        602 root /             2 drwxr-xr-x    1024  r
root     getty        602   wd /             2 drwxr-xr-x    1024  r
root     getty        602 text /        2329103 -r-xr-xr-x   28360  r
root     getty        602 ctty /dev         52 crw-------   ttyv1 rw
root     getty        602    0 /dev         52 crw-------   ttyv1 rw
root     getty        601 root /             2 drwxr-xr-x    1024  r
root     getty        601   wd /             2 drwxr-xr-x    1024  r
root     getty        601 text /        2329103 -r-xr-xr-x   28360  r
root     getty        601 ctty /dev         51 crw-------   ttyv0 rw
root     getty        601    0 /dev         51 crw-------   ttyv0 rw
root     cron         556 root /             2 drwxr-xr-x    1024  r
root     cron         556   wd /        1123585 drwxr-x---     512  r
root     cron         556 text /        2329942 -r-xr-xr-x   41392  r
root     cron         556    0 /dev         13 crw-rw-rw-    null rw
smmsp    sendmail     552 root /             2 drwxr-xr-x    1024  r
smmsp    sendmail     552   wd /        1124045 drwxrwx---     512  r
smmsp    sendmail     552 text /        2410709 -r-xr-sr-x  696896  r
smmsp    sendmail     552    0 /dev         13 crw-rw-rw-    null  r
root     sendmail     549 root /             2 drwxr-xr-x    1024  r
root     sendmail     549   wd /        1124048 drwxr-xr-x     512  r
root     sendmail     549 text /        2410709 -r-xr-sr-x  696896  r
root     sendmail     549    0 /dev         13 crw-rw-rw-    null  r
root     sshd         546 root /             2 drwxr-xr-x    1024  r
root     sshd         546   wd /             2 drwxr-xr-x    1024  r
root     sshd         546 text /        2330130 -r-xr-xr-x  304120  r
root     sshd         546    0 /dev         13 crw-rw-rw-    null rw
root     syslogd      353 root /             2 drwxr-xr-x    1024  r
root     syslogd      353   wd /             2 drwxr-xr-x    1024  r
root     syslogd      353 text /        2330131 -r-xr-xr-x   41048  r
root     syslogd      353    0 /dev         13 crw-rw-rw-    null rw
root     syslogd      353    6 /dev         13 crw-rw-rw-    null rw
root     syslogd      353   12 /dev         13 crw-rw-rw-    null rw
root     syslogd      353   18 /        1192255 -rw-------       3  w
root     devd         279 root /             2 drwxr-xr-x    1024  r
root     devd         279   wd /             2 drwxr-xr-x    1024  r
root     devd         279 text /         82890 -r-xr-xr-x  1111728  r
root     devd         279    0 /dev         13 crw-rw-rw-    null rw
root     devd         279    6 /dev         13 crw-rw-rw-    null rw
root     init           1 root /             2 drwxr-xr-x    1024  r
root     init           1   wd /             2 drwxr-xr-x    1024  r
root     init           1 text /         80553 -r-xr-xr-x  1006464  r
root     kernel         0 root /             2 drwxr-xr-x    1024  r
root     kernel         0   wd /             2 drwxr-xr-x    1024  r

------------------------------------------------------------------------
dmesg

Copyright (c) 1992-2016 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 10.3-RELEASE-p2 #0: Wed May  4 06:03:51 UTC 2016
    root@amd64-builder.daemonology.net:/usr/obj/usr/src/sys/GENERIC amd64
FreeBSD clang version 3.4.1 (tags/RELEASE_34/dot1-final 208032) 20140512
CPU: AMD Opteron(tm) Processor 6262 HE               (1599.82-MHz K8-class CPU)
  Origin="AuthenticAMD"  Id=0x600f12  Family=0x15  Model=0x1  Stepping=2
  Features=0x1783fbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,MMX,FXSR,SSE,SSE2,HTT>
  Features2=0x9e982203<SSE3,PCLMULQDQ,SSSE3,CX16,SSE4.1,SSE4.2,POPCNT,AESNI,XSAVE,OSXSAVE,AVX,HV>
  AMD Features=0x26500800<SYSCALL,NX,MMX+,FFXSR,Page1GB,LM>
  AMD Features2=0x1b9fb<LAHF,CMP,ExtAPIC,CR8,ABM,SSE4A,MAS,Prefetch,XOP,SKINIT,WDT,LWP,FMA4>
  TSC: P-state invariant
Hypervisor: Origin = "bhyve bhyve "
real memory  = 2147483648 (2048 MB)
avail memory = 2050363392 (1955 MB)
Event timer "LAPIC" quality 400
ACPI APIC Table: <BHYVE  BVMADT  >
FreeBSD/SMP: Multiprocessor System Detected: 4 CPUs
FreeBSD/SMP: 4 package(s) x 1 core(s)
 cpu0 (BSP): APIC ID:  0
 cpu1 (AP): APIC ID:  1
 cpu2 (AP): APIC ID:  2
 cpu3 (AP): APIC ID:  3
random: <Software, Yarrow> initialized
ioapic0 <Version 1.1> irqs 0-23 on motherboard
module_register_init: MOD_LOAD (vesa, 0xffffffff80dc6500, 0) error 19
kbd1 at kbdmux0
acpi0: <BHYVE BVXSDT> on motherboard
acpi0: Power Button (fixed)
atrtc0: <AT realtime clock> port 0x70-0x71 irq 8 on acpi0
Event timer "RTC" frequency 32768 Hz quality 0
attimer0: <AT timer> port 0x40-0x43 irq 0 on acpi0
Timecounter "i8254" frequency 1193182 Hz quality 0
Event timer "i8254" frequency 1193182 Hz quality 100
hpet0: <High Precision Event Timer> iomem 0xfed00000-0xfed003ff on acpi0
Timecounter "HPET" frequency 10000000 Hz quality 950
Event timer "HPET" frequency 10000000 Hz quality 550
Event timer "HPET1" frequency 10000000 Hz quality 450
Event timer "HPET2" frequency 10000000 Hz quality 450
Event timer "HPET3" frequency 10000000 Hz quality 450
Event timer "HPET4" frequency 10000000 Hz quality 450
Timecounter "ACPI-fast" frequency 3579545 Hz quality 900
acpi_timer0: <32-bit timer at 3.579545MHz> port 0x408-0x40b on acpi0
pcib0: <ACPI Host-PCI bridge> port 0xcf8-0xcff on acpi0
pci0: <ACPI PCI bus> on pcib0
virtio_pci0: <VirtIO PCI Block adapter> port 0x2000-0x203f mem 0xc0000000-0xc0001fff irq 16 at device 4.0 on pci0
vtblk0: <VirtIO Block Adapter> on virtio_pci0
vtblk0: 20480MB (41943040 512 byte sectors)
virtio_pci1: <VirtIO PCI Network adapter> port 0x2040-0x205f mem 0xc0002000-0xc0003fff irq 17 at device 5.0 on pci0
vtnet0: <VirtIO Networking Adapter> on virtio_pci1
vtnet0: Ethernet address: 00:a0:98:2e:e1:16
virtio_pci2: <VirtIO PCI Entropy adapter> port 0x2060-0x207f mem 0xc0004000-0xc0005fff irq 18 at device 6.0 on pci0
isab0: <PCI-ISA bridge> at device 31.0 on pci0
isa0: <ISA bus> on isab0
uart0: <16550 or compatible> port 0x3f8-0x3ff irq 4 flags 0x10 on acpi0
uart0: console (9600,n,8,1)
uart1: <16550 or compatible> port 0x2f8-0x2ff irq 3 on acpi0
sc0: <System console> at flags 0x100 on isa0
sc0: MDA <16 virtual consoles, flags=0x100>
vga0: <Generic ISA VGA> at port 0x3b0-0x3bb iomem 0xb0000-0xb7fff on isa0
atkbdc0: <Keyboard controller (i8042)> at port 0x60,0x64 on isa0
atkbd0: <AT Keyboard> irq 1 on atkbdc0
kbd0 at atkbd0
atkbd0: [GIANT-LOCKED]
ppc0: cannot reserve I/O port range
Timecounters tick every 1.000 msec
random: unblocking device.
SMP: AP CPU #2 Launched!
SMP: AP CPU #1 Launched!
SMP: AP CPU #3 Launched!
Trying to mount root from ufs:/dev/vtbd0p2 [rw]...
Setting hostuuid: f7968130-d31f-11e5-aed0-00a09856a692.
Setting hostid: 0x7c2ee71a.
Entropy harvesting: interrupts ethernet point_to_point swi.
Starting file system checks:
/dev/vtbd0p2: FILE SYSTEM CLEAN; SKIPPING CHECKS
/dev/vtbd0p2: clean, 3104075 free (59571 frags, 380563 blocks, 1.2% fragmentation)
Mounting local file systems:.
Writing entropy file:.
Setting hostname: FreeBSD.
vtnet0: link state changed to UP
Starting Network: lo0 vtnet0.
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
        options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
        inet6 ::1 prefixlen 128 
        inet6 fe80::1%lo0 prefixlen 64 scopeid 0x2 
        inet 127.0.0.1 netmask 0xff000000 
        nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
vtnet0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
        options=80028<VLAN_MTU,JUMBO_MTU,LINKSTATE>
        ether 00:a0:98:2e:e1:16
        inet 192.168.11.240 netmask 0xffffff00 broadcast 192.168.11.255 
        nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
        media: Ethernet 10Gbase-T <full-duplex>
        status: active
Starting devd.
add net default: gateway 192.168.11.1
add net fe80::: gateway ::1
add net ff02::: gateway ::1
add net ::ffff:0.0.0.0: gateway ::1
add net ::0.0.0.0: gateway ::1
Creating and/or trimming log files.
Starting syslogd.
No core dumps found.
ELF ldconfig path: /lib /usr/lib /usr/lib/compat
32-bit compatibility ldconfig path: /usr/lib32
Clearing /tmp (X related).
Updating motd:.
Mounting late file systems:.
Configuring syscons: keymap blanktime.
Performing sanity check on sshd configuration.
Starting sshd.
Starting sendmail_submit.
Starting sendmail_msp_queue.
Starting cron.
Starting background file system checks in 60 seconds.

Mon May  9 13:18:42 CEST 2016
May  9 13:20:15 FreeBSD login: ROOT LOGIN (root) ON ttyu0
spin lock 0xffffffff816a6200 (smp rendezvous) held by 0xfffff80066083960 (tid 100086) too long
panic: spin lock held too long
cpuid = 2
KDB: stack backtrace:
#0 0xffffffff8098e390 at kdb_backtrace+0x60
#1 0xffffffff80951066 at vpanic+0x126
#2 0xffffffff80950f33 at panic+0x43
#3 0xffffffff80937097 at _mtx_lock_spin_cookie+0x287
#4 0xffffffff80d44cd1 at smp_tlb_shootdown+0x91
#5 0xffffffff80d46b7c at pmap_invalidate_range+0x2fc
#6 0xffffffff809df4c0 at allocbuf+0x5d0
#7 0xffffffff809e0360 at getnewbuf+0x510
#8 0xffffffff809dd6b1 at getblk+0x571
#9 0xffffffff809de19d at breadn_flags+0x2d
#10 0xffffffff80b860f1 at ffs_update+0x101
#11 0xffffffff80baef67 at ffs_sync+0x577
#12 0xffffffff809fb426 at sync_fsync+0x136
#13 0xffffffff80e81af7 at VOP_FSYNC_APV+0xa7
#14 0xffffffff809fbe1b at sched_sync+0x3ab
#15 0xffffffff8091a4ea at fork_exit+0x9a
#16 0xffffffff80d3be0e at fork_trampoline+0xe
Uptime: 39m20s
Dumping 251 out of 2023 MB:..7%..13%..26%..32%..45%..51%..64%..77%..83%..96%

------------------------------------------------------------------------
kernel config

options CONFIG_AUTOGENERATED
ident   GENERIC
machine amd64
cpu     HAMMER
makeoptions     WITH_CTF=1
makeoptions     DEBUG=-g
options XENHVM
options HYPERV
options USB_DEBUG
options ATH_ENABLE_11N
options AH_AR5416_INTERRUPT_MITIGATION
options AH_SUPPORT_AR5416
options IEEE80211_SUPPORT_MESH
options IEEE80211_AMPDU_AGE
options IEEE80211_DEBUG
options SC_PIXEL_MODE
options VESA
options AHD_REG_PRETTY_PRINT
options AHC_REG_PRETTY_PRINT
options ATA_STATIC_ID
options ACPI_DMAR
options SMP
options KDB_TRACE
options KDB
options RCTL
options RACCT_DEFAULT_TO_DISABLED
options RACCT
options INCLUDE_CONFIG_FILE
options DDB_CTF
options KDTRACE_HOOKS
options KDTRACE_FRAME
options MAC
options PROCDESC
options CAPABILITIES
options CAPABILITY_MODE
options AUDIT
options HWPMC_HOOKS
options KBD_INSTALL_CDEV
options PRINTF_BUFR_SIZE=128
options _KPOSIX_PRIORITY_SCHEDULING
options SYSVSEM
options SYSVMSG
options SYSVSHM
options STACK
options KTRACE
options SCSI_DELAY=5000
options COMPAT_FREEBSD7
options COMPAT_FREEBSD6
options COMPAT_FREEBSD5
options COMPAT_FREEBSD4
options COMPAT_FREEBSD32
options GEOM_LABEL
options GEOM_RAID
options GEOM_PART_GPT
options PSEUDOFS
options PROCFS
options CD9660
options MSDOSFS
options NFS_ROOT
options NFSLOCKD
options NFSD
options NFSCL
options MD_ROOT
options QUOTA
options UFS_GJOURNAL
options UFS_DIRHASH
options UFS_ACL
options SOFTUPDATES
options FFS
options SCTP
options TCP_OFFLOAD
options INET6
options INET
options PREEMPTION
options SCHED_ULE
options NEW_PCIB
options GEOM_PART_MBR
options GEOM_PART_EBR_COMPAT
options GEOM_PART_EBR
options GEOM_PART_BSD
device  isa
device  mem
device  io
device  uart_ns8250
device  cpufreq
device  acpi
device  pci
device  fdc
device  ahci
device  ata
device  mvs
device  siis
device  ahc
device  ahd
device  esp
device  hptiop
device  isp
device  mpt
device  mps
device  mpr
device  sym
device  trm
device  adv
device  adw
device  aic
device  bt
device  isci
device  scbus
device  ch
device  da
device  sa
device  cd
device  pass
device  ses
device  amr
device  arcmsr
device  ciss
device  dpt
device  hptmv
device  hptnr
device  hptrr
device  hpt27xx
device  iir
device  ips
device  mly
device  twa
device  tws
device  aac
device  aacp
device  aacraid
device  ida
device  mfi
device  mlx
device  mrsas
device  twe
device  nvme
device  nvd
device  atkbdc
device  atkbd
device  psm
device  kbdmux
device  vga
device  splash
device  sc
device  vt
device  vt_vga
device  vt_efifb
device  agp
device  cbb
device  pccard
device  cardbus
device  uart
device  ppc
device  ppbus
device  lpt
device  ppi
device  puc
device  bxe
device  de
device  em
device  igb
device  ix
device  ixv
device  ixl
device  ixlv
device  le
device  ti
device  txp
device  vx
device  miibus
device  ae
device  age
device  alc
device  ale
device  bce
device  bfe
device  bge
device  cas
device  dc
device  et
device  fxp
device  gem
device  hme
device  jme
device  lge
device  msk
device  nfe
device  nge
device  pcn
device  re
device  rl
device  sf
device  sge
device  sis
device  sk
device  ste
device  stge
device  tl
device  tx
device  vge
device  vr
device  wb
device  xl
device  cs
device  ed
device  ex
device  ep
device  fe
device  sn
device  xe
device  wlan
device  wlan_wep
device  wlan_ccmp
device  wlan_tkip
device  wlan_amrr
device  an
device  ath
device  ath_pci
device  ath_hal
device  ath_rate_sample
device  ipw
device  iwi
device  iwn
device  malo
device  mwl
device  ral
device  wi
device  wpi
device  loop
device  random
device  padlock_rng
device  rdrand_rng
device  ether
device  vlan
device  tun
device  md
device  gif
device  faith
device  firmware
device  bpf
device  uhci
device  ohci
device  ehci
device  xhci
device  usb
device  ukbd
device  umass
device  sound
device  snd_cmi
device  snd_csa
device  snd_emu10kx
device  snd_es137x
device  snd_hda
device  snd_ich
device  snd_via8233
device  mmc
device  mmcsd
device  sdhci
device  virtio
device  virtio_pci
device  vtnet
device  virtio_blk
device  virtio_scsi
device  virtio_balloon
device  hyperv
device  xenpci
device  vmx

------------------------------------------------------------------------
ddb capture buffer

ddb: ddb_capture: kvm_nlist