Attachment 223285 Details for Bug 254303 – kgdb backtrace

kgdb backtrace

trace_gdb (text/plain), 65.99 KB, created by Aleks on 2021-03-15 09:18:32 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Aleks

Created: 2021-03-15 09:18:32 UTC

Size: 65.99 KB

patch

obsolete

>Fri Mar 12 12:10:27 EET 2021
>
>FreeBSD lv-tet-router1 13.0-BETA3 FreeBSD 13.0-BETA3 #0 releng/13.0-n244525-150b4388d3b: Fri Feb 19 04:04:34 UTC 2021     root@releng1.nyi.freebsd.org:/usr/obj/usr/src/amd64.amd64/sys/GENERIC  amd64
>
>panic: page fault
>
>GNU gdb (GDB) 10.1 [GDB v10.1 for FreeBSD]
>Copyright (C) 2020 Free Software Foundation, Inc.
>License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
>This is free software: you are free to change and redistribute it.
>There is NO WARRANTY, to the extent permitted by law.
>Type "show copying" and "show warranty" for details.
>This GDB was configured as "x86_64-portbld-freebsd13.0".
>Type "show configuration" for configuration details.
>For bug reporting instructions, please see:
><https://www.gnu.org/software/gdb/bugs/>.
>Find the GDB manual and other documentation resources online at:
>    <http://www.gnu.org/software/gdb/documentation/>.
>
>For help, type "help".
>Type "apropos word" to search for commands related to "word"...
>Reading symbols from /boot/kernel/kernel...
>Reading symbols from /usr/lib/debug//boot/kernel/kernel.debug...
>
>Unread portion of the kernel message buffer:
>
>
>Fatal trap 12: page fault while in kernel mode
>cpuid = 3; apic id = 06
>fault virtual address   = 0x58
>fault code              = supervisor write data, page not present
>instruction pointer     = 0x20:0xffffffff80d45e32
>stack pointer           = 0x28:0xfffffe014256b4e0
>frame pointer           = 0x28:0xfffffe014256b520
>code segment            = base 0x0, limit 0xfffff, type 0x1b
>                        = DPL 0, pres 1, long 1, def32 0, gran 1
>processor eflags        = interrupt enabled, resume, IOPL = 0
>current process         = 2889 (zebra)
>trap number             = 12
>panic: page fault
>cpuid = 3
>time = 1615543179
>KDB: stack backtrace:
>#0 0xffffffff80c567b5 at kdb_backtrace+0x65
>#1 0xffffffff80c09381 at vpanic+0x181
>#2 0xffffffff80c091f3 at panic+0x43
>#3 0xffffffff810891a7 at trap_fatal+0x387
>#4 0xffffffff810891ff at trap_pfault+0x4f
>#5 0xffffffff8108885d at trap+0x27d
>#6 0xffffffff8105feb8 at calltrap+0x8
>#7 0xffffffff80d462bb at nhgrp_free+0x5b
>#8 0xffffffff80d49682 at rib_del_route+0x132
>#9 0xffffffff80d4eb84 at route_output+0xc14
>#10 0xffffffff80ca84b3 at sosend_generic+0x633
>#11 0xffffffff80ca8940 at sosend+0x50
>#12 0xffffffff80c7e449 at soo_write+0x49
>#13 0xffffffff80c75a88 at dofilewrite+0x88
>#14 0xffffffff80c755fc at sys_write+0xbc
>#15 0xffffffff81089aac at amd64_syscall+0x10c
>#16 0xffffffff810607de at fast_syscall_common+0xf8
>Uptime: 29m2s
>Dumping 1567 out of 32614 MB:..2%..11%..21%..31%..41%..51%..61%..71%..81%..91%
>
>__curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
>55      /usr/src/sys/amd64/include/pcpu_aux.h: No such file or directory.
>(kgdb) #0  __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
>#1  doadump (textdump=<optimized out>) at /usr/src/sys/kern/kern_shutdown.c:399
>#2  0xffffffff80c08f76 in kern_reboot (howto=260)
>    at /usr/src/sys/kern/kern_shutdown.c:486
>#3  0xffffffff80c093f0 in vpanic (fmt=<optimized out>, ap=<optimized out>)
>    at /usr/src/sys/kern/kern_shutdown.c:919
>#4  0xffffffff80c091f3 in panic (fmt=<unavailable>)
>    at /usr/src/sys/kern/kern_shutdown.c:843
>#5  0xffffffff810891a7 in trap_fatal (frame=0xfffffe014256b420, eva=88)
>    at /usr/src/sys/amd64/amd64/trap.c:915
>#6  0xffffffff810891ff in trap_pfault (frame=frame@entry=0xfffffe014256b420, 
>    usermode=false, signo=<optimized out>, signo@entry=0x0, 
>    ucode=<optimized out>, ucode@entry=0x0)
>    at /usr/src/sys/amd64/amd64/trap.c:732
>#7  0xffffffff8108885d in trap (frame=0xfffffe014256b420)
>    at /usr/src/sys/amd64/amd64/trap.c:398
>#8  <signal handler called>
>#9  0xffffffff80d45e32 in atomic_fcmpset_long (dst=<optimized out>, 
>    expect=<optimized out>, src=18446741880128186368)
>    at /usr/src/sys/amd64/include/atomic.h:233
>#10 unlink_nhgrp (ctl=0x0, key=key@entry=0xfffff805e939c718)
>    at /usr/src/sys/net/route/nhgrp.c:188
>#11 0xffffffff80d462bb in nhgrp_free (nhg=<optimized out>)
>    at /usr/src/sys/net/route/nhgrp_ctl.c:347
>#12 0xffffffff80d48305 in nhop_free_any (nh=0x0)
>    at /usr/src/sys/net/route/nhop_ctl.c:714
>#13 0xffffffff80d49682 in del_route (rnh=0xfffff802bab8dc00, 
>    info=<optimized out>, rc=0xfffffe014256b780)
>    at /usr/src/sys/net/route/route_ctl.c:842
>#14 rib_del_route (fibnum=<optimized out>, info=0xfffffe014256b6b8, 
>    rc=0xfffffe014256b780) at /usr/src/sys/net/route/route_ctl.c:745
>#15 0xffffffff80d49d87 in rib_action (fibnum=fibnum@entry=0, 
>    action=action@entry=2, info=info@entry=0xfffffe014256b6b8, rc=0x1, 
>    rc@entry=0xfffffe014256b780) at /usr/src/sys/net/route/route_ctl.c:1216
>#16 0xffffffff80d4eb84 in route_output (m=0xfffff806696bdc00, 
>    so=0xfffff80399e1e760) at /usr/src/sys/net/rtsock.c:1035
>#17 0xffffffff80ca84b3 in sosend_generic (so=0xfffff80399e1e760, addr=0x0, 
>    uio=<optimized out>, top=0xfffff806696bdc00, control=0x0, flags=0, 
>    td=0xfffffe01445ca800) at /usr/src/sys/kern/uipc_socket.c:1755
>#18 0xffffffff80ca8940 in sosend (so=0x0, so@entry=0xfffff80399e1e760, 
>    addr=0xfffff805e939c718, addr@entry=0x0, uio=0x1, 
>    uio@entry=0xfffffe014256ba60, top=0x1, top@entry=0x0, 
>    control=control@entry=0x0, flags=7, flags@entry=0, td=0xfffffe01445ca800)
>    at /usr/src/sys/kern/uipc_socket.c:1810
>#19 0xffffffff80c7e449 in soo_write (fp=<optimized out>, 
>    uio=0xfffffe014256ba60, active_cred=<optimized out>, 
>    flags=<optimized out>, td=<optimized out>)
>    at /usr/src/sys/kern/sys_socket.c:148
>#20 0xffffffff80c75a88 in fo_write (fp=0xfffff803b25a74b0, 
>    uio=0xfffffe014256ba60, active_cred=0x1, flags=7, td=0xfffffe01445ca800)
>    at /usr/src/sys/sys/file.h:338
>#21 dofilewrite (td=td@entry=0xfffffe01445ca800, fd=fd@entry=8, 
>    fp=0xfffff803b25a74b0, auio=auio@entry=0xfffffe014256ba60, 
>    offset=<optimized out>, offset@entry=-1, flags=<optimized out>, 
>    flags@entry=0) at /usr/src/sys/kern/sys_generic.c:565
>#22 0xffffffff80c755fc in kern_writev (td=0xfffffe01445ca800, fd=8, 
>    auio=0xfffffe014256ba60) at /usr/src/sys/kern/sys_generic.c:492
>#23 sys_write (td=0xfffffe01445ca800, uap=<optimized out>)
>    at /usr/src/sys/kern/sys_generic.c:407
>#24 0xffffffff81089aac in syscallenter (td=0xfffffe01445ca800)
>    at /usr/src/sys/amd64/amd64/../../kern/subr_syscall.c:189
>#25 amd64_syscall (td=0xfffffe01445ca800, traced=0)
>    at /usr/src/sys/amd64/amd64/trap.c:1156
>#26 <signal handler called>
>#27 0x000000080072080a in ?? ()
>Backtrace stopped: Cannot access memory at address 0x7fffdfdfc718
>(kgdb) 
>
>------------------------------------------------------------------------
>ps -axlww
>
> UID  PID PPID  C PRI NI     VSZ     RSS MWCHAN   STAT TT       TIME COMMAND
>   0    0    0 11 -16  0       0       0 swapin   DLs   -    1:41.64 [kernel]
>   0    1    0 11  20  0   11824    1104 wait     DLs   -    0:00.03 [init]
>   0    2    0 10 -16  0       0       0 -        DL    -    0:00.00 [KTLS]
>   0    3    0 11 -16  0       0       0 crypto_w DL    -    0:00.00 [crypto]
>   0    4    0  2 -16  0       0       0 crypto_r DL    -    0:00.00 [crypto returns 0]
>   0    5    0 10 -16  0       0       0 crypto_r DL    -    0:00.00 [crypto returns 1]
>   0    6    0 10 -16  0       0       0 crypto_r DL    -    0:00.00 [crypto returns 2]
>   0    7    0 11 -16  0       0       0 crypto_r DL    -    0:00.00 [crypto returns 3]
>   0    8    0  7 -16  0       0       0 crypto_r DL    -    0:00.00 [crypto returns 4]
>   0    9    0  9 -16  0       0       0 crypto_r DL    -    0:00.00 [crypto returns 5]
>   0   10    0  1 -16  0       0       0 audit_wo DL    -    0:00.00 [audit]
>   0   11    0  0 155  0       0       0 -        RL    -  154:10.57 [idle]
>   0   12    0 -1 -56  0       0       0 -        WL    -    0:02.03 [intr]
>   0   13    0  2  -8  0       0       0 -        DL    -    0:00.01 [geom]
>   0   14    0  9 -16  0       0       0 crypto_r DL    -    0:00.00 [crypto returns 6]
>   0   15    0  0 -16  0       0       0 crypto_r DL    -    0:00.00 [crypto returns 7]
>   0   16    0 11 -16  0       0       0 crypto_r DL    -    0:00.00 [crypto returns 8]
>   0   17    0  1 -16  0       0       0 crypto_r DL    -    0:00.00 [crypto returns 9]
>   0   18    0  8 -16  0       0       0 crypto_r DL    -    0:00.00 [crypto returns 10]
>   0   19    0  9 -16  0       0       0 crypto_r DL    -    0:00.00 [crypto returns 11]
>   0   20    0  2 -16  0       0       0 seqstate DL    -    0:00.00 [sequencer 00]
>   0   21    0  6 -16  0       0       0 -        RL    -    0:00.07 [cam]
>   0   22    0  5 -68  0       0       0 -        DL    -    0:00.07 [usb]
>   0   23    0  2 -16  0       0       0 -        DL    -    0:00.00 [soaiod1]
>   0   24    0  8 -16  0       0       0 -        DL    -    0:00.00 [soaiod2]
>   0   25    0  4 -16  0       0       0 -        DL    -    0:00.00 [soaiod3]
>   0   26    0 11 -16  0       0       0 -        DL    -    0:00.00 [soaiod4]
>   0   27    0  2  -8  0       0       0 t->zthr_ DL    -    0:00.19 [zfskern]
>   0   28    0  2 -16  0       0       0 -        DL    -    0:00.12 [rand_harvestq]
>   0   29    0  7 -16  0       0       0 psleep   DL    -    0:00.05 [pagedaemon]
>   0   30    0  5 -16  0       0       0 psleep   DL    -    0:00.00 [vmdaemon]
>   0   31    0  8 -16  0       0       0 qsleep   DL    -    0:00.02 [bufdaemon]
>   0   32    0  4 -16  0       0       0 vlruwt   DL    -    0:00.00 [vnlru]
>   0   33    0  1  16  0       0       0 syncer   DL    -    0:00.00 [syncer]
>   0  183    1 11  52  0   12768    2172 pause    Ds    -    0:00.00 [adjkerntz]
>   0 2741    1  0  52  0   13100    2452 select   Ds    -    0:00.00 [moused]
>   0 2759    1  0  52  0   13100    2452 select   Ds    -    0:00.00 [moused]
>   0 2772    1  2  20  0   11492    1484 select   Ds    -    0:00.00 [devd]
>   0 2889    1 10  20  0  832504  529852 -        Rs    -    0:41.50 [zebra]
> 168 2892    1  6  20  0 1743252 1360376 select   Ds    -    0:13.02 [bgpd]
> 168 2895    1  8  20  0   69988    6704 select   Ds    -    0:00.01 [staticd]
>   0 3034    1  8  20  0   12920    2792 select   Ds    -    0:00.01 [syslogd]
>   0 3147    1 11  20  0   20952    8132 select   Ds    -    0:00.00 [sshd]
>   0 3159    1 10  20  0   12964    2612 nanslp   Ds    -    0:00.00 [cron]
>   0 3185    1  1  52  0   12892    2336 ttyin    Ds+   -    0:00.00 [getty]
>   0 3186    1  4  52  0   12892    2336 ttyin    Ds+   -    0:00.00 [getty]
>   0 3187    1  5  52  0   12892    2336 ttyin    Ds+   -    0:00.00 [getty]
>   0 3188    1  0  52  0   12892    2336 ttyin    Ds+   -    0:00.00 [getty]
>   0 3189    1  9  52  0   12892    2336 ttyin    Ds+   -    0:00.00 [getty]
>   0 3190    1  7  52  0   12892    2336 ttyin    Ds+   -    0:00.00 [getty]
>   0 3191    1 11  52  0   12892    2336 ttyin    Ds+   -    0:00.00 [getty]
>   0 3192    1 10  52  0   12892    2336 ttyin    Ds+   -    0:00.00 [getty]
>   0 3193 3147  3  21  0   21388    8840 select   Ds    -    0:00.01 [sshd]
>1001 3196 3193  4  20  0   27024   13512 select   D     -    0:05.38 [sshd]
>1001 3197 3196  8  22  0   13624    3348 wait     Ds    -    0:00.00 [sh]
>   0 3200 3197  7  20  0   13708    3140 wait     D     -    0:00.01 [su]
>   0 3201 3200  8  20  0   16540    4128 ttyin    D+    -    0:00.06 [csh]
>   0 3260 3147  7  21  0   21388    8956 select   Ds    -    0:00.01 [sshd]
>1001 3263 3260 11  20  0   27024   13780 select   D     -    0:01.58 [sshd]
>1001 3264 3263  1  21  0   13624    3352 wait     Ds    -    0:00.00 [sh]
>   0 3268 3264  8  20  0   13708    3140 wait     D     -    0:00.01 [su]
>   0 3269 3268  7  20  0   16540    4100 pause    D     -    0:00.02 [csh]
>   0 3319 3269  2  20  0   23736   12312 select   D+    -    0:01.69 [vtysh]
>
>------------------------------------------------------------------------
>vmstat -s
>
> 25289498 cpu context switches
>  9285820 device interrupts
>  3449131 software interrupts
>  1013807 traps
> 41219086 system calls
>       33 kernel threads created
>     2995  fork() calls
>      302 vfork() calls
>       30 rfork() calls
>        0 swap pager pageins
>        0 swap pager pages paged in
>        0 swap pager pageouts
>        0 swap pager pages paged out
>      752 vnode pager pageins
>     5278 vnode pager pages paged in
>        0 vnode pager pageouts
>        0 vnode pager pages paged out
>        0 page daemon wakeups
>        0 pages examined by the page daemon
>        0 clean page reclamation shortfalls
>        0 pages reactivated by the page daemon
>   140564 copy-on-write faults
>      396 copy-on-write optimized faults
>   883022 zero fill pages zeroed
>        0 zero fill pages prezeroed
>      335 intransit blocking page faults
>  1827637 total VM faults taken
>      578 page faults requiring I/O
>        0 pages affected by kernel thread creation
>   295840 pages affected by  fork()
>    10568 pages affected by vfork()
>     1304 pages affected by rfork()
>  1023426 pages freed
>        0 pages freed by daemon
>        0 pages freed by exiting processes
>        0 pages active
>        0 pages inactive
>        0 pages in the laundry queue
>        0 pages wired down
>        0 virtual user pages wired down
>        0 pages free
>        0 bytes per page
>        0 total name lookups
>          cache hits (0% pos + 0% neg) system 0% per-directory
>          deletions 0%, falsehits 0%, toolong 0%
>
>------------------------------------------------------------------------
>vmstat -m
>
>vmstat: memstat_kvm_malloc: 
>         Type InUse MemUse Requests  Size(s)
>
>------------------------------------------------------------------------
>vmstat -z
>
>ITEM                   SIZE  LIMIT     USED     FREE      REQ     FAILSLEEP XDOMAIN
>UMA Kegs:               384,      0,     174,       0,     174,   0,   0,   0
>UMA Zones:             2048,      0,     201,       0,     201,   0,   0,   0
>UMA Slabs 0:             80,      0,  257336,       0,  257336,   0,   0,   0
>UMA Slabs 1:            112,      0,      11,       0,      11,   0,   0,   0
>UMA Hash:               256,      0,       0,       0,       0,   0,   0,   0
>2 Bucket:                32,      0,     541,    1979,   51563,   0,   0,   0
>4 Bucket:                48,      0,     114,    1230,    5074,   0,   0,   0
>8 Bucket:                80,      0,    1397,    1119,   27454,   7,   0,   0
>16 Bucket:              144,      0,     456,    2736,  160585,   0,   0,   0
>32 Bucket:              256,      0,    5838,    1173,   45682,  42,   0,   0
>64 Bucket:              512,      0,     230,     506,   61142,  83,   0,   0
>128 Bucket:            1024,      0,     215,     280,    5663,  17,   0,   0
>256 Bucket:            2048,      0,     572,     154,   79085,32785,   0,   0
>SMR SHARED:              24,      0,       2,      61,       2,   0,   0,   0
>SMR CPU:                 32,      0,       2,      61,       2,   0,   0,   0
>vmem:                  1856,      0,       1,       1,       1,   0,   0,   0
>vmem btag:               56,      0,   16137,     555,   16137, 455,   0,   0
>VM OBJECT:              264,      0,    1422,     348,   66379,   0,   0,   0
>RADIX NODE:             144,      0,   59286,     715,  226091,   0,   0,   0
>KMAP ENTRY:              96,      0,      24,       0,      24,   0,   0,   0
>MAP ENTRY:               96,      0,    1838,    2026,  210820,   0,   0,   0
>VMSPACE:               2544,      0,      30,      66,    3329,   0,   0,   0
>fakepg:                 104,      0,       0,       0,       0,   0,   0,   0
>pcpu-4:                   4,      0,       0,       0,       0,   0,   0,   0
>pcpu-8:                   8,      0,    4193,    1947,    4195,   0,   0,   0
>pcpu-16:                 16,      0,      35,    1243,      35,   0,   0,   0
>pcpu-32:                 32,      0,       0,       0,       0,   0,   0,   0
>pcpu-64:                 64,      0,     455,     185,     455,   0,   0,   0
>malloc-16:               16,      0,   21342,    1338, 1786604,   0,   0,   0
>malloc-32:               32,      0,   92016,    4248, 4342322,   0,   0,   0
>malloc-64:               64,      0,   22598,    7642,16237612,   0,   0,   0
>malloc-128:             128,      0,   47948,    4194, 2867354,   0,   0,   0
>malloc-256:             256,      0,    6998,    1387, 3450671,   0,   0,   0
>malloc-384:             384,      0,    1415,    5215,  108489,   0,   0,   0
>malloc-512:             512,      0,      76,    5764,   19537,   0,   0,   0
>malloc-1024:           1024,      0,    1960,     120, 3438939,   0,   0,   0
>malloc-2048:           2048,      0,     276,     106,  114072,   0,   0,   0
>malloc-4096:           4096,      0,    2723,      29,  151491,   0,   0,   0
>malloc-8192:           8192,      0,     548,      12,    2434,   0,   0,   0
>malloc-16384:         16384,      0,      37,      60,   30752,   0,   0,   0
>malloc-32768:         32768,      0,     145,       6,    2287,   0,   0,   0
>malloc-65536:         65536,      0,      15,      12,    2719,   0,   0,   0
>SLEEPQUEUE:              80,      0,     784,     176,     784,   0,   0,   0
>kenv:                   258,      0,      18,     177,   27320,   0,   0,   0
>Files:                   80,      0,     152,     748,   35942,   0,   0,   0
>filedesc0:             1072,      0,      63,     147,    3361,   0,   0,   0
>PWD:                     32,      0,      14,    1498,     399,   0,   0,   0
>rl_entry:                40,      0,      71,    1141,      71,   0,   0,   0
>TURNSTILE:              136,      0,     784,      77,     784,   0,   0,   0
>rangeset pctrie nodes:    144,      0,       0,       0,       0,   0,   0,   0
>umtx pi:                 96,      0,       0,       0,       0,   0,   0,   0
>umtx_shm:                88,      0,       0,       0,       0,   0,   0,   0
>swrbuf:                2368,    128,       0,       0,       0,   0,   0,   0
>swwbuf:                2368,     64,       0,       0,       0,   0,   0,   0
>nfspbuf:               2368,    128,       0,       0,       0,   0,   0,   0
>mdpbuf:                2368,     25,       0,       0,       0,   0,   0,   0
>vnpbuf:                2368,    512,       0,       0,       0,   0,   0,   0
>clpbuf:                2368,    128,       0,       0,       0,   0,   0,   0
>pbuf:                  2368,     16,       0,       0,       0,   0,   0,   0
>MAC labels:              40,      0,       0,       0,       0,   0,   0,   0
>PROC:                  1320,      0,      62,      55,    3360,   0,   0,   0
>PGRP:                    88,      0,      28,     478,     277,   0,   0,   0
>THREAD:                1792,      0,     722,      60,     764,   0,   0,   0
>cpuset:                 104,      0,      27,     345,      27,   0,   0,   0
>domainset:               40,      0,       0,       0,       0,   0,   0,   0
>audit_record:          1280,      0,       0,       0,       0,   0,   0,   0
>mbuf_packet:            256,      0,      41,    4699, 3569756,   0,   0,   0
>mbuf:                   256, 12994953,   58050,    8595,203287130,   0,   0,   0
>mbuf_cluster:          2048, 2030461,   62083,    4403,  148733,   0,   0,   0
>mbuf_jumbo_page:       4096, 1015230,       0,    3455,   42574,   0,   0,   0
>mbuf_jumbo_9k:         9216, 300809,       0,       0,       0,   0,   0,   0
>mbuf_jumbo_16k:       16384, 169205,       0,       0,       0,   0,   0,   0
>epoch_record pcpu:      256,      0,       4,      12,       4,   0,   0,   0
>ktls_session:           192,      0,       0,       0,       0,   0,   0,   0
>ttyinq:                 160,      0,     240,     210,     705,   0,   0,   0
>ttyoutq:                256,      0,     126,     144,     374,   0,   0,   0
>FPU_save_area:          832,      0,       1,       8,       1,   0,   0,   0
>DEVCTL:                1024,   1000,       0,       0,     587,   0,   0,   0
>IOMMU_MAP_ENTRY:        120,      0,       0,       0,       0,   0,   0,   0
>g_bio:                  376,      0,       0,     440,   59893,   0,   0,   0
>linux_dma_pctrie:       144,      0,       0,       0,       0,   0,   0,   0
>linux_dma_object:        24,      0,       0,       0,       0,   0,   0,   0
>cryptop:                280,      0,       0,       0,       0,   0,   0,   0
>vtnet_tx_hdr:            24,      0,       0,       0,       0,   0,   0,   0
>taskq_zone:             192,      0,       0,     966,    7185,   0,   0,   0
>VNODE:                  488,      0,     733,     235,    1219,   0,   0,   0
>BUF TRIE:               144,      0,       0,       0,       0,   0,   0,   0
>NAMEI:                 1024,      0,       0,      52,   73357,   0,   0,   0
>S VFS Cache:            104,      0,    1214,     346,    1298,   0,   0,   0
>STS VFS Cache:          144,      0,       0,       0,       0,   0,   0,   0
>L VFS Cache:            320,      0,       0,      12,       1,   0,   0,   0
>LTS VFS Cache:          360,      0,       0,       0,       0,   0,   0,   0
>TMPFS node:             224,      0,       0,       0,       0,   0,   0,   0
>DIRHASH:               1024,      0,       0,       0,       0,   0,   0,   0
>NCLNODE:                592,      0,       0,       0,       0,   0,   0,   0
>Mountpoints:           2752,      0,      12,      23,      12,   0,   0,   0
>AIO:                    208,      0,       0,       0,       0,   0,   0,   0
>AIOP:                    32,      0,       0,       0,       0,   0,   0,   0
>AIOCB:                  552,      0,       0,       0,       0,   0,   0,   0
>AIOLIO:                 272,      0,       0,       0,       0,   0,   0,   0
>pipe:                   744,      0,      19,     106,    1340,   0,   0,   0
>zfs_btree_leaf_cache:   4096,      0,      86,      80,   16707,   0,   0,   0
>metaslab_alloc_trace_cache:     72,      0,       0,       0,       0,   0,   0,   0
>ddt_cache:            24840,      0,      13,       0,      13,   0,   0,   0
>ddt_entry_cache:        392,      0,       0,       0,       0,   0,   0,   0
>zio_cache:             1232,      0,      16,    5507,  232128,   0,   0,   0
>zio_link_cache:          48,      0,       0,    8652,  133519,   0,   0,   0
>zio_buf_comb_512:       512,      0,      97,   16263,   63608,   0,   0,   0
>zio_buf_comb_1024:     1024,      0,      78,      34,     157,   0,   0,   0
>zio_buf_comb_1536:     1536,      0,      47,      63,     196,   0,   0,   0
>zio_buf_comb_2048:     2048,      0,      31,      13,      61,   0,   0,   0
>zio_buf_comb_2560:     2560,      0,      23,      25,      45,   0,   0,   0
>zio_buf_comb_3072:     3072,      0,      14,      18,      14,   0,   0,   0
>zio_buf_comb_3584:     3584,      0,      12,      24,     161,   0,   0,   0
>zio_buf_comb_4096:     4096,      0,    1624,      45,    6434,   0,   0,   0
>zio_buf_comb_5120:     5120,      0,       5,      18,      38,   0,   0,   0
>zio_buf_comb_6144:     6144,      0,       5,      14,      45,   0,   0,   0
>zio_buf_comb_7168:     7168,      0,       2,       7,       9,   0,   0,   0
>zio_buf_comb_8192:     8192,      0,       6,      33,    1279,   0,   0,   0
>zio_buf_comb_10240:   10240,      0,       3,      13,      25,   0,   0,   0
>zio_buf_comb_12288:   12288,      0,       5,      29,     719,   0,   0,   0
>zio_buf_comb_14336:   14336,      0,       2,      10,      16,   0,   0,   0
>zio_buf_comb_16384:   16384,      0,     209,      45,    2069,   0,   0,   0
>zio_buf_comb_20480:   20480,      0,       6,      28,     265,   0,   0,   0
>zio_buf_comb_24576:   24576,      0,       4,      20,     114,   0,   0,   0
>zio_buf_comb_28672:   28672,      0,       1,      24,     112,   0,   0,   0
>zio_buf_comb_32768:   32768,      0,       3,      25,     104,   0,   0,   0
>zio_buf_comb_40960:   40960,      0,       8,      21,     149,   0,   0,   0
>zio_buf_comb_49152:   49152,      0,       3,      25,     133,   0,   0,   0
>zio_buf_comb_57344:   57344,      0,       3,      27,     162,   0,   0,   0
>zio_buf_comb_65536:   65536,      0,       2,      25,     183,   0,   0,   0
>zio_buf_comb_81920:   81920,      0,       1,      32,     425,   0,   0,   0
>zio_buf_comb_98304:   98304,      0,       0,      33,     665,   0,   0,   0
>zio_buf_comb_114688: 114688,      0,       2,      31,    1100,   0,   0,   0
>zio_buf_comb_131072: 131072,      0,     418,    9935,   43688,   0,   0,   0
>zio_buf_comb_163840: 163840,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_196608: 196608,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_229376: 229376,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_262144: 262144,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_327680: 327680,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_393216: 393216,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_458752: 458752,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_524288: 524288,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_655360: 655360,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_786432: 786432,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_917504: 917504,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_1048576: 1048576,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_1310720: 1310720,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_1572864: 1572864,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_1835008: 1835008,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_2097152: 2097152,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_2621440: 2621440,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_3145728: 3145728,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_3670016: 3670016,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_4194304: 4194304,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_5242880: 5242880,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_6291456: 6291456,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_7340032: 7340032,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_8388608: 8388608,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_10485760: 10485760,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_12582912: 12582912,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_14680064: 14680064,      0,       0,       0,       0,   0,   0,   0
>zio_buf_comb_16777216: 16777216,      0,       0,       0,       0,   0,   0,   0
>abd_chunk:             4096,      0,   66669,  132988,  940823,   0,   0,   0
>sa_cache:               288,      0,     681,      61,     733,   0,   0,   0
>dnode_t:                816,      0,    1039,      86,    1907,   0,   0,   0
>arc_buf_hdr_t_full:     248,      0,   14312,     104,   45532,   0,   0,   0
>arc_buf_hdr_t_full_crypt:    312,      0,       0,       0,       0,   0,   0,   0
>arc_buf_hdr_t_l2only:     96,      0,       0,       0,       0,   0,   0,   0
>arc_buf_t:               64,      0,     805,    8708,   46048,   0,   0,   0
>dmu_buf_impl_t:         296,      0,    1635,    5333,   46584,   0,   0,   0
>zil_lwb_cache:          360,      0,       3,      85,      30,   0,   0,   0
>zil_zcw_cache:           80,      0,       0,     400,      40,   0,   0,   0
>sio_cache_0:            136,      0,       0,       0,       0,   0,   0,   0
>sio_cache_1:            152,      0,       0,       0,       0,   0,   0,   0
>sio_cache_2:            168,      0,       0,       0,       0,   0,   0,   0
>zfs_znode_cache:        472,      0,     681,     103,     733,   0,   0,   0
>ksiginfo:               112,      0,      76,    1076,     195,   0,   0,   0
>itimer:                 352,      0,       0,       0,       0,   0,   0,   0
>KNOTE:                  160,      0,       0,       0,       0,   0,   0,   0
>socket:                 944, 1043670,      47,      61,    6096,   0,   0,   0
>unpcb:                  256, 1043670,      27,     138,    5719,   0,   0,   0
>ipq:                     56,  51200,       0,       0,       0,   0,   0,   0
>udp_inpcb:              488, 1043670,       2,     166,     292,   0,   0,   0
>udpcb:                   32, 1043670,       2,    1510,     292,   0,   0,   0
>tcp_inpcb:              488, 1043670,      15,      65,      54,   0,   0,   0
>tcpcb:                 1048, 1043670,      15,     106,      54,   0,   0,   0
>tcptw:                   88,  27767,       0,      92,       2,   0,   0,   0
>syncache:               168,  15360,       0,     288,      36,   0,   0,   0
>hostcache:               96,  15360,       1,      41,       1,   0,   0,   0
>sackhole:                32,      0,       0,     252,      40,   0,   0,   0
>tfo:                      4,      0,       0,       0,       0,   0,   0,   0
>tfo_ccache_entries:      80,      0,       0,       0,       0,   0,   0,   0
>tcpreass:                48, 126903,       0,       0,       0,   0,   0,   0
>tcp_log:                408, 5000000,       0,       0,       0,   0,   0,   0
>tcp_log_bucket:         176,      0,       0,       0,       0,   0,   0,   0
>tcp_log_node:           120,      0,       0,       0,       0,   0,   0,   0
>udplite_inpcb:          488, 1043670,       0,       0,       0,   0,   0,   0
>ripcb:                  488, 1043670,       1,      71,      19,   0,   0,   0
>IPsec SA lft_c:          16,      0,       0,       0,       0,   0,   0,   0
>routing nhops:          256,      0,      18,     162, 2535986,   0,   0,   0
>rtentry:                176,      0,  840560,    1332, 2535986,   0,   0,   0
>IPFW counters:           16,      0,      11,    1267,      11,   0,   0,   0
>IPFW dynamic states data:     88,  16384,       0,       0,       0,   0,   0,   0
>IPFW parent dynamic states:     32,   4096,       0,       0,       0,   0,   0,   0
>IPFW IPv4 dynamic states:     40,      0,       0,       0,       0,   0,   0,   0
>IPFW IPv6 dynamic states:     72,      0,       0,       0,       0,   0,   0,   0
>swpctrie:               144, 4060952,       0,       0,       0,   0,   0,   0
>swblk:                  136, 4060928,       0,       0,       0,   0,   0,   0
>
>------------------------------------------------------------------------
>vmstat -i
>
>interrupt                          total       rate
>irq9: acpi0                            4          0
>irq18: ehci0 ehci1                  7152         15
>cpu0:timer                       1746900       3750
>cpu1:timer                         84208        181
>cpu2:timer                         97060        208
>cpu3:timer                         83833        180
>cpu4:timer                        331949        712
>cpu5:timer                        129926        279
>cpu6:timer                         88099        189
>cpu7:timer                         99400        213
>cpu8:timer                         81378        175
>cpu9:timer                        106392        228
>cpu10:timer                        93522        201
>cpu11:timer                       106969        230
>irq56: igb0:rxq0                  492837       1058
>irq57: igb0:rxq1                     300          1
>irq58: igb0:rxq2                     447          1
>irq59: igb0:rxq3                     163          0
>irq60: igb0:rxq4                     208          0
>irq61: igb0:rxq5                     228          0
>irq62: igb0:rxq6                   20671         44
>irq63: igb0:rxq7                     210          0
>irq64: igb0:aq                         1          0
>irq74: mpr0                        17488         38
>irq75: ixl0:aq                         2          0
>irq76: ixl0:rxq0                  400531        860
>irq77: ixl0:rxq1                    2389          5
>irq78: ixl0:rxq2                    2515          5
>irq79: ixl0:rxq3                    1742          4
>irq80: ixl0:rxq4                    1714          4
>irq81: ixl0:rxq5                    2165          5
>irq82: ixl0:rxq6                    2051          4
>irq83: ixl0:rxq7                    1985          4
>irq84: ixl0:rxq8                    1692          4
>irq85: ixl0:rxq9                    1751          4
>irq86: ixl0:rxq10                   9661         21
>irq87: ixl0:rxq11                   5115         11
>irq88: ixl1:aq                         2          0
>irq89: ixl1:rxq0                 1406197       3018
>irq90: ixl1:rxq1                    1649          4
>irq91: ixl1:rxq2                    2083          4
>irq92: ixl1:rxq3                    1720          4
>irq93: ixl1:rxq4                    4493         10
>irq94: ixl1:rxq5                    2060          4
>irq95: ixl1:rxq6                    2024          4
>irq96: ixl1:rxq7                    1789          4
>irq97: ixl1:rxq8                    1755          4
>irq98: ixl1:rxq9                    1589          3
>irq99: ixl1:rxq10                  10716         23
>irq100: ixl1:rxq11                  3820          8
>irq101: ixl2:aq                        2          0
>irq102: ixl2:rxq0                 399723        858
>irq103: ixl2:rxq1                 540396       1160
>irq104: ixl2:rxq2                 319255        685
>irq105: ixl2:rxq3                 263704        566
>irq106: ixl2:rxq4                 186108        399
>irq107: ixl2:rxq5                 442168        949
>irq108: ixl2:rxq6                 249374        535
>irq109: ixl2:rxq7                 248110        533
>irq110: ixl2:rxq8                 249662        536
>irq111: ixl2:rxq9                 249114        535
>irq112: ixl2:rxq10                241445        518
>irq113: ixl2:rxq11                253095        543
>irq114: ixl3:aq                        2          0
>irq115: ixl3:rxq0                 250504        538
>irq116: ixl3:rxq1                 407833        875
>irq117: ixl3:rxq2                 166054        356
>irq118: ixl3:rxq3                 302735        650
>irq119: ixl3:rxq4                 231462        497
>irq120: ixl3:rxq5                 194616        418
>irq121: ixl3:rxq6                 283139        608
>irq122: ixl3:rxq7                 404185        868
>irq123: ixl3:rxq8                 238641        512
>irq124: ixl3:rxq9                 228135        490
>irq125: ixl3:rxq10                326520        701
>irq126: ixl3:rxq11                196919        423
>Total                           12335456      26477
>
>------------------------------------------------------------------------
>pstat -T
>
>152/1043670 files
>0M/0M swap space
>
>------------------------------------------------------------------------
>pstat -s
>
>Device          1K-blocks     Used    Avail Capacity
>
>------------------------------------------------------------------------
>iostat
>
>       tty             da0              da1            pass0             cpu
> tin  tout KB/t  tps  MB/s  KB/t  tps  MB/s  KB/t  tps  MB/s  us ni sy in id
>   2 105499  230    5   1.2  30.3    6   0.2   0.0    0   0.0   0  0  1  0 98
>
>------------------------------------------------------------------------
>ipcs -a
>
>Message Queues:
>T           ID          KEY MODE        OWNER    GROUP    CREATOR  CGROUP                 CBYTES                 QNUM               QBYTES        LSPID        LRPID STIME    RTIME    CTIME   
>
>Shared Memory:
>T           ID          KEY MODE        OWNER    GROUP    CREATOR  CGROUP         NATTCH        SEGSZ         CPID         LPID ATIME    DTIME    CTIME   
>
>Semaphores:
>T           ID          KEY MODE        OWNER    GROUP    CREATOR  CGROUP          NSEMS OTIME    CTIME   
>
>
>------------------------------------------------------------------------
>ipcs -T
>
>msginfo:
>        msgmax:        16384    (max characters in a message)
>        msgmni:           40    (# of message queues)
>        msgmnb:         2048    (max characters in a message queue)
>        msgtql:           40    (max # of messages in system)
>        msgssz:            8    (size of a message segment)
>        msgseg:         2048    (# of message segments in system)
>
>shminfo:
>        shmmax:    536870912    (max shared memory segment size)
>        shmmin:            1    (min shared memory segment size)
>        shmmni:          192    (max number of shared memory identifiers)
>        shmseg:          128    (max shared memory segments per process)
>        shmall:       131072    (max amount of shared memory in pages)
>
>seminfo:
>        semmni:           50    (# of semaphore identifiers)
>        semmns:          340    (# of semaphores in system)
>        semmnu:          150    (# of undo structures in system)
>        semmsl:          340    (max # of semaphores per id)
>        semopm:          100    (max # of operations per semop call)
>        semume:           50    (max # of undo entries per process)
>        semusz:          632    (size in bytes of undo structure)
>        semvmx:        32767    (semaphore maximum value)
>        semaem:        16384    (adjust on exit max value)
>
>
>------------------------------------------------------------------------
>netstat -s
>
>tcp:
>        202902 packets sent
>                186101 data packets (357954563 bytes)
>                439 data packets (699868 bytes) retransmitted
>                2 data packets unnecessarily retransmitted
>                0 resends initiated by MTU discovery
>                16306 ack-only packets (225 delayed)
>                0 URG only packets
>                0 window probe packets
>                18 window update packets
>                38 control packets
>        62060 packets received
>                25295 acks (for 223283389 bytes)
>                372 duplicate acks
>                0 acks for unsent data
>                36319 packets (46844278 bytes) received in-sequence
>                12 completely duplicate packets (12 bytes)
>                0 old duplicate packets
>                0 packets with some dup. data (0 bytes duped)
>                0 out-of-order packets (0 bytes)
>                0 packets (0 bytes) of data after window
>                0 window probes
>                5 window update packets
>                32 packets received after close
>                0 discarded for bad checksums
>                0 discarded for bad header offset fields
>                0 discarded because packet too short
>                0 discarded due to full reassembly queue
>        4 connection requests
>        36 connection accepts
>        0 bad connection attempts
>        0 listen queue overflows
>        0 ignored RSTs in the windows
>        39 connections established (including accepts)
>                34 times used RTT from hostcache
>                34 times used RTT variance from hostcache
>                0 times used slow-start threshold from hostcache
>        39 connections closed (including 0 drops)
>                1 connection updated cached RTT on close
>                1 connection updated cached RTT variance on close
>                0 connections updated cached ssthresh on close
>        1 embryonic connection dropped
>        25259 segments updated rtt (of 17555 attempts)
>        5 retransmit timeouts
>                0 connections dropped by rexmit timeout
>        0 persist timeouts
>                0 connections dropped by persist timeout
>        0 Connections (fin_wait_2) dropped because of timeout
>        0 keepalive timeouts
>                0 keepalive probes sent
>                0 connections dropped by keepalive
>        17976 correct ACK header predictions
>        34860 correct data packet header predictions
>        36 syncache entries added
>                0 retransmitted
>                0 dupsyn
>                0 dropped
>                36 completed
>                0 bucket overflow
>                0 cache overflow
>                0 reset
>                0 stale
>                0 aborted
>                0 badack
>                0 unreach
>                0 zone failures
>        36 cookies sent
>        0 cookies received
>        1 hostcache entry added
>                0 bucket overflow
>        7 SACK recovery episodes
>        393 segment rexmits in SACK recovery episodes
>        560256 byte rexmits in SACK recovery episodes
>        465 SACK options (SACK blocks) received
>        0 SACK options (SACK blocks) sent
>        0 SACK scoreboard overflow
>        0 packets with ECN CE bit set
>        0 packets with ECN ECT(0) bit set
>        0 packets with ECN ECT(1) bit set
>        0 successful ECN handshakes
>        0 times ECN reduced the congestion window
>        0 packets with matching signature received
>        0 packets with bad signature received
>        0 times failed to make signature due to no SA
>        0 times unexpected signature received
>        0 times no signature provided by segment
>        0 Path MTU discovery black hole detection activations
>        0 Path MTU discovery black hole detection min MSS activations
>        0 Path MTU discovery black hole detection failures
>TCP connection count by state:
>        0 connections in CLOSED state
>        10 connections in LISTEN state
>        0 connections in SYN_SENT state
>        0 connections in SYN_RCVD state
>        5 connections in ESTABLISHED state
>        0 connections in CLOSE_WAIT state
>        0 connections in FIN_WAIT_1 state
>        0 connections in CLOSING state
>        0 connections in LAST_ACK state
>        0 connections in FIN_WAIT_2 state
>        0 connections in TIME_WAIT state
>udp:
>        12 datagrams received
>        0 with incomplete header
>        0 with bad data length field
>        0 with bad checksum
>        3 with no checksum
>        8 dropped due to no socket
>        0 broadcast/multicast datagrams undelivered
>        0 dropped due to full socket buffers
>        0 not for hashed pcb
>        4 delivered
>        4 datagrams output
>        0 times multicast source filter matched
>ip:
>        133215601 total packets received
>        0 bad header checksums
>        0 with size smaller than minimum
>        0 with data size < data length
>        0 with ip length > max ip packet size
>        0 with header length < data size
>        0 with data length < header length
>        0 with bad options
>        0 with incorrect version number
>        0 fragments received
>        0 fragments dropped (dup or out of space)
>        0 fragments dropped after timeout
>        0 packets reassembled ok
>        62209 packets for this host
>        0 packets for unknown/unsupported protocol
>        61957844 packets forwarded (61957844 packets fast forwarded)
>        2967 packets not forwardable
>        0 packets received for unknown multicast group
>        0 redirects sent
>        52835403 packets sent from this host
>        0 packets sent with fabricated ip header
>        0 output packets dropped due to no bufs, etc.
>        3029 output packets discarded due to no route
>        0 output datagrams fragmented
>        0 fragments created
>        0 datagrams that can't be fragmented
>        0 tunneling packets that can't find gif
>        0 datagrams with bad address in header
>icmp:
>        4618860 calls to icmp_error
>        2971 errors not generated in response to an icmp message
>        Output histogram:
>                echo reply: 106
>                destination unreachable: 8
>                routing redirect: 61907725
>                time exceeded: 4615881
>        0 messages with bad code fields
>        0 messages less than the minimum length
>        0 messages with bad checksum
>        0 messages with bad length
>        0 multicast echo requests ignored
>        0 multicast timestamp requests ignored
>        Input histogram:
>                echo reply: 2
>                routing redirect: 28
>                echo: 106
>                time exceeded: 1
>        106 message responses generated
>        0 invalid return addresses
>        13891669 no return routes
>arp:
>        11 ARP requests sent
>        4 ARP requests failed to sent
>        1 ARP reply sent
>        1 ARP request received
>        7 ARP replies received
>        8 ARP packets received
>        0 total packets dropped due to no ARP entry
>        2 ARP entrys timed out
>        0 Duplicate IPs seen
>ip6:
>        0 total packets received
>        0 with size smaller than minimum
>        0 with data size < data length
>        0 with bad options
>        0 with incorrect version number
>        0 fragments received
>        0 fragments dropped (dup or out of space)
>        0 fragments dropped after timeout
>        0 fragments that exceeded limit
>        0 atomic fragments
>        0 packets reassembled ok
>        0 packets for this host
>        0 packets forwarded
>        0 packets not forwardable
>        0 redirects sent
>        0 packets sent from this host
>        0 packets sent with fabricated ip header
>        0 output packets dropped due to no bufs, etc.
>        1 output packet discarded due to no route
>        0 output datagrams fragmented
>        0 fragments created
>        0 datagrams that can't be fragmented
>        0 packets that violated scope rules
>        0 multicast packets which we don't join
>        Mbuf statistics:
>                0 one mbuf
>                0 one ext mbuf
>                0 two or more ext mbuf
>        0 packets whose headers are not contiguous
>        0 tunneling packets that can't find gif
>        0 packets discarded because of too many headers
>        0 failures of source address selection
>icmp6:
>        0 calls to icmp6_error
>        0 errors not generated in response to an icmp6 message
>        0 errors not generated because of rate limitation
>        0 messages with bad code fields
>        0 messages < minimum length
>        0 bad checksums
>        0 messages with bad length
>        Histogram of error messages to be generated:
>                0 no route
>                0 administratively prohibited
>                0 beyond scope
>                0 address unreachable
>                0 port unreachable
>                0 packet too big
>                0 time exceed transit
>                0 time exceed reassembly
>                0 erroneous header field
>                0 unrecognized next header
>                0 unrecognized option
>                0 redirect
>                0 unknown
>        0 message responses generated
>        0 messages with too many ND options
>        0 messages with bad ND options
>        0 bad neighbor solicitation messages
>        0 bad neighbor advertisement messages
>        0 bad router solicitation messages
>        0 bad router advertisement messages
>        0 bad redirect messages
>        0 default routers overflows
>        0 prefix overflows
>        0 neighbour entries overflows
>        0 redirect overflows
>        0 messages with invalid hop limit
>        0 path MTU changes
>rip6:
>        0 messages received
>        0 checksum calculations on inbound
>        0 messages with bad checksum
>        0 messages dropped due to no socket
>        0 multicast messages dropped due to no socket
>        0 messages dropped due to full socket buffers
>        0 delivered
>        0 datagrams output
>pfkey:
>        0 requests sent from userland
>        0 bytes sent from userland
>        0 messages with invalid length field
>        0 messages with invalid version field
>        0 messages with invalid message type field
>        0 messages too short
>        0 messages with memory allocation failure
>        0 messages with duplicate extension
>        0 messages with invalid extension type
>        0 messages with invalid sa type
>        0 messages with invalid address extension
>        0 requests sent to userland
>        0 bytes sent to userland
>        0 messages toward single socket
>        0 messages toward all sockets
>        0 messages toward registered sockets
>        0 messages with memory allocation failure
>
>------------------------------------------------------------------------
>netstat -m
>
>netstat: memstat_kvm_all: 
>
>------------------------------------------------------------------------
>netstat -anA
>
>Active Internet connections (including servers)
>Tcpcb            Proto Recv-Q Send-Q Local Address      Foreign Address    (state)    
>fffffe0144efc4d8 tcp4       0      0 *.2616             *.*                LISTEN     
>fffffe0144efc8f0 tcp6       0      0 *.2616             *.*                LISTEN     
>fffffe0144ecaca8 tcp4       0      0 *.179              *.*                LISTEN     
>fffffe0144ecb0c0 tcp6       0      0 *.179              *.*                LISTEN     
>fffffe0144ecb4d8 tcp4       0      0 *.2605             *.*                LISTEN     
>fffffe0144ecb8f0 tcp6       0      0 *.2605             *.*                LISTEN     
>fffffe0144c954d8 tcp4       0      0 *.2601             *.*                LISTEN     
>fffffe0144c958f0 tcp6       0      0 *.2601             *.*                LISTEN     
>fffff803125823d0 udp4       0      0 *.514              *.*                
>fffff803125825b8 udp6       0      0 *.514              *.*                
>fffff80312768d58 icm6       0      0 *.*                *.*                
>Active UNIX domain sockets
>Address          Type   Recv-Q Send-Q            Inode             Conn             Refs          Nextref Addr
>fffff80614326900 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/staticd.vty
>fffff80614326a00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/staticd.vty
>fffff80614326b00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/bgpd.vty
>fffff80614326c00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/bgpd.vty
>fffff80614326d00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/zebra.vty
>fffff802babe3900 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/zebra.vty
>fffff803b12df600 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/zebra.vty
>fffff803b1662d00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/zebra.vty
>fffff803b2408b00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/zebra.vty
>fffff803b118a600 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/zebra.vty
>fffff803b118a800 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/zebra.vty
>fffff802babe3c00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/zebra.vty
>fffff803b1330e00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/zserv.api
>fffff803b12dfc00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/zserv.api
>fffff80399ab9d00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/staticd.vty
>fffff803b1662900 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/zserv.api
>fffff803b1662a00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/zserv.api
>fffff803b1662b00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/zserv.api
>fffff802babe3e00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/zserv.api
>fffff803b2408d00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/bgpd.vty
>fffff803b118ac00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/zebra.vty
>fffff803b243ec00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/frr/zserv.api
>fffff803b12dfe00 stream      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/devd.pipe
>fffff803b1662500 dgram       0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/devd.pipe
>fffff803b1662700 dgram       0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/logpriv
>fffff803b1662800 dgram       0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/log
>fffff803b12dfd00 seqpac      0      0        8006f6410 fffffffffffff9d5           20a8e9        8006f6328 /var/run/devd.seqpacket.pipe
>
>------------------------------------------------------------------------
>netstat -aL
>
>Current listen queue sizes (qlen/incqlen/maxqlen)
>Proto Listen                           Local Address         
>tcp4  0/0/128                          *.22022                
>tcp6  0/0/128                          *.22022                
>tcp4  0/0/3                            *.2616                 
>tcp6  0/0/3                            *.2616                 
>tcp4  0/0/128                          *.bgp                  
>tcp6  0/0/128                          *.bgp                  
>tcp4  0/0/3                            *.nsc-posa             
>tcp6  0/0/3                            *.nsc-posa             
>tcp4  0/0/3                            *.discp-client         
>tcp6  0/0/3                            *.discp-client         
>unix  0/0/5                            /var/run/frr/staticd.vty
>unix  0/0/5                            /var/run/frr/bgpd.vty
>unix  0/0/5                            /var/run/frr/zebra.vty
>unix  0/0/5                            /var/run/frr/zserv.api
>unix  0/0/4                            /var/run/devd.pipe
>unix  0/0/4                            /var/run/devd.seqpacket.pipe
>
>------------------------------------------------------------------------
>fstat
>
>USER     CMD          PID   FD MOUNT      INUM MODE         SZ|DV R/W
>root     vtysh       3319 root /            34 drwxr-xr-x      25  r
>root     vtysh       3319   wd /usr/home      2 drwxr-xr-x       9  r
>root     vtysh       3319 text /         34699 -rwxr-xr-x  749968  r
>root     vtysh       3319 ctty /dev        140 crw--w----   pts/1 rw
>root     vtysh       3319    0 /dev        140 crw--w----   pts/1 rw
>root     vtysh       3319    1 /dev        140 crw--w----   pts/1 rw
>root     vtysh       3319    2 /dev        140 crw--w----   pts/1 rw
>root     vtysh       3319    3* local stream fffff802babe3900 <-> fffff80614326d00
>root     vtysh       3319    4* local stream fffff80614326c00 <-> fffff80614326b00
>root     vtysh       3319    5* local stream fffff80614326a00 <-> fffff80614326900
>root     csh         3269 root /            34 drwxr-xr-x      25  r
>root     csh         3269   wd /usr/home      2 drwxr-xr-x       9  r
>root     csh         3269 text /         15976 -r-xr-xr-x  424656  r
>root     csh         3269 ctty /dev        140 crw--w----   pts/1 rw
>root     csh         3269   15 /dev        140 crw--w----   pts/1 rw
>root     csh         3269   16 /dev        140 crw--w----   pts/1 rw
>root     csh         3269   17 /dev        140 crw--w----   pts/1 rw
>root     csh         3269   18 /dev        140 crw--w----   pts/1 rw
>root     csh         3269   19 /dev        140 crw--w----   pts/1 rw
>root     su          3268 root /            34 drwxr-xr-x      25  r
>root     su          3268   wd /usr/home      2 drwxr-xr-x       9  r
>root     su          3268 text /         15964 -r-sr-xr-x   17232  r
>root     su          3268 ctty /dev        140 crw--w----   pts/1 rw
>root     su          3268    0 /dev        140 crw--w----   pts/1 rw
>root     su          3268    1 /dev        140 crw--w----   pts/1 rw
>root     su          3268    2 /dev        140 crw--w----   pts/1 rw
>iff      sh          3264 root /            34 drwxr-xr-x      25  r
>iff      sh          3264   wd /usr/home      2 drwxr-xr-x       9  r
>iff      sh          3264 text /         16007 -r-xr-xr-x  162616  r
>iff      sh          3264 ctty /dev        140 crw--w----   pts/1 rw
>iff      sh          3264    0 /dev        140 crw--w----   pts/1 rw
>iff      sh          3264    1 /dev        140 crw--w----   pts/1 rw
>iff      sh          3264    2 /dev        140 crw--w----   pts/1 rw
>iff      sh          3264   10 /dev        140 crw--w----   pts/1 rw
>iff      sshd        3263 root /            34 drwxr-xr-x      25  r
>iff      sshd        3263   wd /            34 drwxr-xr-x      25  r
>iff      sshd        3263 text /          2033 -r-xr-xr-x  309704  r
>iff      sshd        3263    0 /dev         42 crw-rw-rw-    null rw
>iff      sshd        3263    1 /dev         42 crw-rw-rw-    null rw
>iff      sshd        3263    2 /dev         42 crw-rw-rw-    null rw
>iff      sshd        3263    3 /dev         74 crw-rw-rw-  crypto rw
>iff      sshd        3263    4* local stream fffff803b1662d00 <-> fffff803b12df600
>iff      sshd        3263    5* internet stream tcp fffffe0144eb9890
>iff      sshd        3263    6 /dev         74 crw-rw-rw-  crypto rw
>iff      sshd        3263    7* pipe fffff803f55bc5d0 <-> fffff803f55bc730      0 rw
>iff      sshd        3263    8* local stream fffff803b2408b00
>iff      sshd        3263    9* pipe fffff803f55bc730 <-> fffff803f55bc5d0      0 rw
>iff      sshd        3263   10* pseudo-terminal master      pts/1 rw
>iff      sshd        3263   12* pseudo-terminal master      pts/1 rw
>iff      sshd        3263   13* pseudo-terminal master      pts/1 rw
>root     sshd        3260 root /            34 drwxr-xr-x      25  r
>root     sshd        3260   wd /            34 drwxr-xr-x      25  r
>root     sshd        3260 text /          2033 -r-xr-xr-x  309704  r
>root     sshd        3260    0 /dev         42 crw-rw-rw-    null  r
>root     sshd        3260    1 /dev         42 crw-rw-rw-    null rw
>root     sshd        3260    2 /dev         42 crw-rw-rw-    null rw
>root     sshd        3260    3 /dev         74 crw-rw-rw-  crypto rw
>root     sshd        3260    4* pseudo-terminal master      pts/1 rw
>root     sshd        3260    5* internet stream tcp fffffe0144eb9890
>root     sshd        3260    6 /dev         74 crw-rw-rw-  crypto rw
>root     sshd        3260    7* local stream fffff803b12df600 <-> fffff803b1662d00
>root     sshd        3260    8* local stream fffff803b2408b00
>root     csh         3201 root /            34 drwxr-xr-x      25  r
>root     csh         3201   wd /usr/home      2 drwxr-xr-x       9  r
>root     csh         3201 text /         15976 -r-xr-xr-x  424656  r
>root     csh         3201 ctty /dev        138 crw--w----   pts/0 rw
>root     csh         3201   15 /dev        138 crw--w----   pts/0 rw
>root     csh         3201   16 /dev        138 crw--w----   pts/0 rw
>root     csh         3201   17 /dev        138 crw--w----   pts/0 rw
>root     csh         3201   18 /dev        138 crw--w----   pts/0 rw
>root     csh         3201   19 /dev        138 crw--w----   pts/0 rw
>root     su          3200 root /            34 drwxr-xr-x      25  r
>root     su          3200   wd /usr/home      2 drwxr-xr-x       9  r
>root     su          3200 text /         15964 -r-sr-xr-x   17232  r
>root     su          3200 ctty /dev        138 crw--w----   pts/0 rw
>root     su          3200    0 /dev        138 crw--w----   pts/0 rw
>root     su          3200    1 /dev        138 crw--w----   pts/0 rw
>root     su          3200    2 /dev        138 crw--w----   pts/0 rw
>iff      sh          3197 root /            34 drwxr-xr-x      25  r
>iff      sh          3197   wd /usr/home      2 drwxr-xr-x       9  r
>iff      sh          3197 text /         16007 -r-xr-xr-x  162616  r
>iff      sh          3197 ctty /dev        138 crw--w----   pts/0 rw
>iff      sh          3197    0 /dev        138 crw--w----   pts/0 rw
>iff      sh          3197    1 /dev        138 crw--w----   pts/0 rw
>iff      sh          3197    2 /dev        138 crw--w----   pts/0 rw
>iff      sh          3197   10 /dev        138 crw--w----   pts/0 rw
>iff      sshd        3196 root /            34 drwxr-xr-x      25  r
>iff      sshd        3196   wd /            34 drwxr-xr-x      25  r
>iff      sshd        3196 text /          2033 -r-xr-xr-x  309704  r
>iff      sshd        3196    0 /dev         42 crw-rw-rw-    null rw
>iff      sshd        3196    1 /dev         42 crw-rw-rw-    null rw
>iff      sshd        3196    2 /dev         42 crw-rw-rw-    null rw
>iff      sshd        3196    3 /dev         74 crw-rw-rw-  crypto rw
>iff      sshd        3196    4* local stream fffff803b118a800 <-> fffff803b118a600
>iff      sshd        3196    5* internet stream tcp fffffe01941fa8f0
>iff      sshd        3196    6 /dev         74 crw-rw-rw-  crypto rw
>iff      sshd        3196    7* pipe fffff803995bc000 <-> fffff803995bc160      0 rw
>iff      sshd        3196    8* local stream fffff802babe3c00
>iff      sshd        3196    9* pipe fffff803995bc160 <-> fffff803995bc000      0 rw
>iff      sshd        3196   10* pseudo-terminal master      pts/0 rw
>iff      sshd        3196   12* pseudo-terminal master      pts/0 rw
>iff      sshd        3196   13* pseudo-terminal master      pts/0 rw
>root     sshd        3193 root /            34 drwxr-xr-x      25  r
>root     sshd        3193   wd /            34 drwxr-xr-x      25  r
>root     sshd        3193 text /          2033 -r-xr-xr-x  309704  r
>root     sshd        3193    0 /dev         42 crw-rw-rw-    null  r
>root     sshd        3193    1 /dev         42 crw-rw-rw-    null rw
>root     sshd        3193    2 /dev         42 crw-rw-rw-    null rw
>root     sshd        3193    3 /dev         74 crw-rw-rw-  crypto rw
>root     sshd        3193    4* pseudo-terminal master      pts/0 rw
>root     sshd        3193    5* internet stream tcp fffffe01941fa8f0
>root     sshd        3193    6 /dev         74 crw-rw-rw-  crypto rw
>root     sshd        3193    7* local stream fffff803b118a600 <-> fffff803b118a800
>root     sshd        3193    8* local stream fffff802babe3c00
>root     getty       3192 root /            34 drwxr-xr-x      25  r
>root     getty       3192   wd /            34 drwxr-xr-x      25  r
>root     getty       3192 text /          6177 -r-xr-xr-x   31016  r
>root     getty       3192 ctty /dev         88 crw-------   ttyv7 rw
>root     getty       3192    0 /dev         88 crw-------   ttyv7 rw
>root     getty       3192    1 /dev         88 crw-------   ttyv7 rw
>root     getty       3192    2 /dev         88 crw-------   ttyv7 rw
>root     getty       3191 root /            34 drwxr-xr-x      25  r
>root     getty       3191   wd /            34 drwxr-xr-x      25  r
>root     getty       3191 text /          6177 -r-xr-xr-x   31016  r
>root     getty       3191 ctty /dev         87 crw-------   ttyv6 rw
>root     getty       3191    0 /dev         87 crw-------   ttyv6 rw
>root     getty       3191    1 /dev         87 crw-------   ttyv6 rw
>root     getty       3191    2 /dev         87 crw-------   ttyv6 rw
>root     getty       3190 root /            34 drwxr-xr-x      25  r
>root     getty       3190   wd /            34 drwxr-xr-x      25  r
>root     getty       3190 text /          6177 -r-xr-xr-x   31016  r
>root     getty       3190 ctty /dev         86 crw-------   ttyv5 rw
>root     getty       3190    0 /dev         86 crw-------   ttyv5 rw
>root     getty       3190    1 /dev         86 crw-------   ttyv5 rw
>root     getty       3190    2 /dev         86 crw-------   ttyv5 rw
>root     getty       3189 root /            34 drwxr-xr-x      25  r
>root     getty       3189   wd /            34 drwxr-xr-x      25  r
>root     getty       3189 text /          6177 -r-xr-xr-x   31016  r
>root     getty       3189 ctty /dev         85 crw-------   ttyv4 rw
>root     getty       3189    0 /dev         85 crw-------   ttyv4 rw
>root     getty       3189    1 /dev         85 crw-------   ttyv4 rw
>root     getty       3189    2 /dev         85 crw-------   ttyv4 rw
>root     getty       3188 root /            34 drwxr-xr-x      25  r
>root     getty       3188   wd /            34 drwxr-xr-x      25  r
>root     getty       3188 text /          6177 -r-xr-xr-x   31016  r
>root     getty       3188 ctty /dev         84 crw-------   ttyv3 rw
>root     getty       3188    0 /dev         84 crw-------   ttyv3 rw
>root     getty       3188    1 /dev         84 crw-------   ttyv3 rw
>root     getty       3188    2 /dev         84 crw-------   ttyv3 rw
>root     getty       3187 root /            34 drwxr-xr-x      25  r
>root     getty       3187   wd /            34 drwxr-xr-x      25  r
>root     getty       3187 text /          6177 -r-xr-xr-x   31016  r
>root     getty       3187 ctty /dev         83 crw-------   ttyv2 rw
>root     getty       3187    0 /dev         83 crw-------   ttyv2 rw
>root     getty       3187    1 /dev         83 crw-------   ttyv2 rw
>root     getty       3187    2 /dev         83 crw-------   ttyv2 rw
>root     getty       3186 root /            34 drwxr-xr-x      25  r
>root     getty       3186   wd /            34 drwxr-xr-x      25  r
>root     getty       3186 text /          6177 -r-xr-xr-x   31016  r
>root     getty       3186 ctty /dev         82 crw-------   ttyv1 rw
>root     getty       3186    0 /dev         82 crw-------   ttyv1 rw
>root     getty       3186    1 /dev         82 crw-------   ttyv1 rw
>root     getty       3186    2 /dev         82 crw-------   ttyv1 rw
>root     getty       3185 root /            34 drwxr-xr-x      25  r
>root     getty       3185   wd /            34 drwxr-xr-x      25  r
>root     getty       3185 text /          6177 -r-xr-xr-x   31016  r
>root     getty       3185 ctty /dev         81 crw-------   ttyv0 rw
>root     getty       3185    0 /dev         81 crw-------   ttyv0 rw
>root     getty       3185    1 /dev         81 crw-------   ttyv0 rw
>root     getty       3185    2 /dev         81 crw-------   ttyv0 rw
>root     cron        3159 root /            34 drwxr-xr-x      25  r
>root     cron        3159   wd /         16019 drwxr-x---       3  r
>root     cron        3159 text /          1955 -r-xr-xr-x   45976  r
>root     cron        3159    0 /dev         42 crw-rw-rw-    null rw
>root     cron        3159    1 /dev         42 crw-rw-rw-    null rw
>root     cron        3159    2 /dev         42 crw-rw-rw-    null rw
>root     cron        3159    3 /         16018 drwxr-xr-x      28  r
>root     cron        3159    4 /         98690 -rw-------       4  w
>root     sshd        3147 root /            34 drwxr-xr-x      25  r
>root     sshd        3147   wd /            34 drwxr-xr-x      25  r
>root     sshd        3147 text /          2033 -r-xr-xr-x  309704  r
>root     sshd        3147    0 /dev         42 crw-rw-rw-    null rw
>root     sshd        3147    1 /dev         42 crw-rw-rw-    null rw
>root     sshd        3147    2 /dev         42 crw-rw-rw-    null rw
>root     sshd        3147    3 /dev         74 crw-rw-rw-  crypto rw
>root     sshd        3147    4* internet6 stream tcp fffffe0144bce0c0
>root     sshd        3147    5* internet stream tcp fffffe0144bcdca8
>root     syslogd     3034 root /            34 drwxr-xr-x      25  r
>root     syslogd     3034   wd /            34 drwxr-xr-x      25  r
>root     syslogd     3034 text /          2022 -r-xr-xr-x   54424  r
>root     syslogd     3034    0 /dev         42 crw-rw-rw-    null rw
>root     syslogd     3034    1 /dev         42 crw-rw-rw-    null rw
>root     syslogd     3034    2 /dev         42 crw-rw-rw-    null rw
>root     syslogd     3034    3* pipe fffff803996648b8 <-> fffff80399664a18      0 rw
>root     syslogd     3034    4* pipe fffff80399664a18 <-> fffff803996648b8      0 rw
>root     syslogd     3034    5 /dev         26 crw-------    klog  r
>root     syslogd     3034    6* internet6 dgram udp fffff80399e235b8
>root     syslogd     3034    7* internet dgram udp fffff80399e237a0
>root     syslogd     3034    8* local dgram fffff803b1662800
>root     sys

Actions: View

Attachments on bug 254303: 223285 | 223541