# uname -Kv FreeBSD 14.1-STABLE f5fabf3dc SUPPORT-14-1 1401501 # kgdb /boot/kernel/kernel /var/crash/vmcore.2 ... Reading symbols from /boot/kernel/kernel... Reading symbols from /usr/lib/debug//boot/kernel/kernel.debug... Unread portion of the kernel message buffer: panic: free: address 0xfffff8000989b900(0xfffff8000989b000) has not been allocated. cpuid = 3 time = 1727914194 KDB: stack backtrace: db_trace_self_wrapper() at 0xffffffff804c959b = db_trace_self_wrapper+0x2b/frame 0xfffffe0083db2bd0 vpanic() at 0xffffffff80c24011 = vpanic+0x131/frame 0xfffffe0083db2d00 panic() at 0xffffffff80c23ed3 = panic+0x43/frame 0xfffffe0083db2d60 free() at 0xffffffff80bf8c3d = free+0xfd/frame 0xfffffe0083db2d90 dbuf_destroy() at 0xffffffff8290a774 = dbuf_destroy+0x64/frame 0xfffffe0083db2dd0 dnode_destroy() at 0xffffffff829379fe = dnode_destroy+0x13e/frame 0xfffffe0083db2e10 dnode_buf_evict_async() at 0xffffffff82938ac5 = dnode_buf_evict_async+0x85/frame 0xfffffe0083db2e40 taskqueue_run_locked() at 0xffffffff80c97332 = taskqueue_run_locked+0x182/frame 0xfffffe0083db2ec0 taskqueue_thread_loop() at 0xffffffff80c985b2 = taskqueue_thread_loop+0xc2/frame 0xfffffe0083db2ef0 fork_exit() at 0xffffffff80bdc06f = fork_exit+0x7f/frame 0xfffffe0083db2f30 fork_trampoline() at 0xffffffff8123b78e = fork_trampoline+0xe/frame 0xfffffe0083db2f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- KDB: enter: panic Reading symbols from /boot/kernel/zfs.ko... Reading symbols from /usr/lib/debug//boot/kernel/zfs.ko.debug... Reading symbols from /boot/kernel/acpi_wmi.ko... Reading symbols from /usr/lib/debug//boot/kernel/acpi_wmi.ko.debug... Reading symbols from /boot/kernel/uhid.ko... Reading symbols from /usr/lib/debug//boot/kernel/uhid.ko.debug... Reading symbols from /boot/kernel/wmt.ko... Reading symbols from /usr/lib/debug//boot/kernel/wmt.ko.debug... Reading symbols from /boot/kernel/mac_ntpd.ko... Reading symbols from /usr/lib/debug//boot/kernel/mac_ntpd.ko.debug... Reading symbols from /boot/kernel/fdescfs.ko... Reading symbols from /usr/lib/debug//boot/kernel/fdescfs.ko.debug... __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:57 57 return (td); (kgdb) bt #0 __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:57 #1 doadump (textdump=0) at /usr/src/sys/kern/kern_shutdown.c:405 #2 0xffffffff804c69b8 in db_fncall_generic (nargs=0, args=0xfffffe0083db2600, addr=<optimized out>, rv=<optimized out>) at /usr/src/sys/ddb/db_command.c:626 #3 db_fncall (dummy1=<optimized out>, dummy2=<optimized out>, dummy3=<optimized out>, dummy4=<optimized out>) at /usr/src/sys/ddb/db_command.c:674 #4 0xffffffff804c642d in db_command (last_cmdp=<optimized out>, cmd_table=<optimized out>, dopager=false) at /usr/src/sys/ddb/db_command.c:504 #5 0xffffffff804c6576 in db_command_script (command=command@entry=0xffffffff8201a722 <db_recursion_data+82> "call doadump") at /usr/src/sys/ddb/db_command.c:569 #6 0xffffffff804cb718 in db_script_exec (scriptname=scriptname@entry=0xfffffe0083db27d0 "kdb.enter.panic", warnifnotfound=warnifnotfound@entry=0) at /usr/src/sys/ddb/db_script.c:302 #7 0xffffffff804cb612 in db_script_kdbenter (eventname=<optimized out>) at /usr/src/sys/ddb/db_script.c:324 #8 0xffffffff804c96e1 in db_trap (type=<optimized out>, code=<optimized out>) at /usr/src/sys/ddb/db_main.c:267 #9 0xffffffff80c73236 in kdb_trap (type=type@entry=3, code=code@entry=0, tf=tf@entry=0xfffffe0083db2b10) at /usr/src/sys/kern/subr_kdb.c:790 #10 0xffffffff81263978 in trap (frame=0xfffffe0083db2b10) at /usr/src/sys/amd64/amd64/trap.c:608 #11 <signal handler called> #12 kdb_enter (why=<optimized out>, msg=<optimized out>) at /usr/src/sys/kern/subr_kdb.c:556 #13 0xffffffff80c24042 in vpanic (fmt=0xffffffff81541553 "free: address %p(%p) has not been allocated.\n", ap=ap@entry=0xfffffe0083db2d40) at /usr/src/sys/kern/kern_shutdown.c:955 #14 0xffffffff80c23ed3 in panic (fmt=0xffffffff81caf7a8 <vt_conswindow+16> "\217?V\201\377\377\377\377") at /usr/src/sys/kern/kern_shutdown.c:891 #15 0xffffffff80bf8c3d in free (addr=0xfffff8000989b900, mtp=0xffffffff82b7e610 <M_SOLARIS>) at /usr/src/sys/kern/kern_malloc.c:919 #16 0xffffffff828b3951 in zfs_kmem_free (buf=0xffffffff81caf7a8 <vt_conswindow+16>, size=size@entry=320) at /usr/src/sys/contrib/openzfs/module/os/freebsd/spl/spl_kmem.c:120 #17 0xffffffff8290a774 in dbuf_destroy (db=0xfffff800b0848be0) at /usr/src/sys/contrib/openzfs/module/zfs/dbuf.c:3024 #18 0xffffffff829379fe in dnode_destroy (dn=0xfffff801fde61650) at /usr/src/sys/contrib/openzfs/module/zfs/dnode.c:666 #19 0xffffffff82938ac5 in dnode_buf_evict_async (dbu=0xfffff800683c4000) at /usr/src/sys/contrib/openzfs/module/zfs/dnode.c:1350 #20 0xffffffff80c97332 in taskqueue_run_locked (queue=queue@entry=0xfffff8001188de00) at /usr/src/sys/kern/subr_taskqueue.c:518 #21 0xffffffff80c985b2 in taskqueue_thread_loop (arg=arg@entry=0xfffff80011898a70) at /usr/src/sys/kern/subr_taskqueue.c:830 #22 0xffffffff80bdc06f in fork_exit (callout=0xffffffff80c984f0 <taskqueue_thread_loop>, arg=0xfffff80011898a70, frame=0xfffffe0083db2f40) at /usr/src/sys/kern/kern_fork.c:1164 #23 <signal handler called>
Проблема
The problem is clearly related to ZFS
# zpool -V zfs-2.2.4-FreeBSD_g256659204 zfs-kmod-2.2.4-FreeBSD_g256659204
What kind of workload was the system running? Is the problem reproducible?
The system is not loaded on average - LA 0.7. But backup synchronization of many small files is started periodically.