Panic when zfs sending to an encrypted dataset. FreeBSD 13.0-RELEASE-p4 tar zcvf panic.tar.gz /boot/kernel /var/crash/*.1 /usr/lib/debug/boot/kernel https://drive.google.com/file/d/1WhzW2tWEQenPPb6Dogd4eS1cXNNODJbD/view?usp=sharing zpool history data | grep -e "zpool import" -B 4 2021-12-22.01:10:20 zfs receive -s -F data/encrypted/zroot/var/audit 2021-12-22.01:10:20 zfs rollback -R data/encrypted/zroot/var/crash@autosnap_2021-12-21_01:01:02_hourly 2021-12-22.01:10:32 zfs receive -s -F data/encrypted/zroot/var/crash 2021-12-22.01:10:32 zfs rollback -R data/encrypted/zroot/var/log@autosnap_2021-12-21_01:01:02_hourly 2021-12-22.01:13:41 zpool import -c /etc/zfs/zpool.cache -a -N -- 2022-01-18.23:09:32 zfs receive -s -F data/encrypted/zroot/var/audit 2022-01-18.23:09:33 zfs rollback -R data/encrypted/zroot/var/crash@autosnap_2022-01-17_22:01:03_hourly 2022-01-18.23:09:42 zfs receive -s -F data/encrypted/zroot/var/crash 2022-01-18.23:09:43 zfs rollback -R data/encrypted/zroot/var/log@autosnap_2022-01-17_22:01:03_hourly 2022-01-18.23:13:13 zpool import -c /etc/zfs/zpool.cache -a -N The only remaining data sets to be sent were very small, so they may not have been successfully received and the history may not reflect that: zroot/var/log 9.47M 1.36T 664K /var/log zroot/var/mail 16.3M 1.36T 12.5M /var/mail zroot/var/tmp 512K 1.36T 96K /var/tmp Unread portion of the kernel message buffer: panic: VERIFY3(0 == dmu_bonus_hold_by_dnode(dn, FTAG, &db, flags)) failed (0 == 5) cpuid = 12 time = 1642565383 KDB: stack backtrace: #0 0xffffffff80c574c5 at kdb_backtrace+0x65 #1 0xffffffff80c09ea1 at vpanic+0x181 #2 0xffffffff8215e07a at spl_panic+0x3a #3 0xffffffff821cd7f1 at receive_object+0x971 #4 0xffffffff821cb9a5 at receive_writer_thread+0x115 #5 0xffffffff80bc7dde at fork_exit+0x7e #6 0xffffffff810629de at fork_trampoline+0xe Uptime: 27d21h56m23s Dumping 6349 out of 65091 MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91% __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55 warning: Source file is more recent than executable. 55 __asm("movq %%gs:%P1,%0" : "=r" (td) : "n" (offsetof(struct pcpu, (kgdb) #0 __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55 #1 doadump (textdump=<optimized out>) at /usr/src/sys/kern/kern_shutdown.c:399 #2 0xffffffff80c09a96 in kern_reboot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:486 #3 0xffffffff80c09f10 in vpanic (fmt=<optimized out>, ap=<optimized out>) at /usr/src/sys/kern/kern_shutdown.c:919 #4 0xffffffff8215e07a in spl_panic (file=<optimized out>, func=<optimized out>, line=<unavailable>, fmt=<unavailable>) at /usr/src/sys/contrib/openzfs/module/os/freebsd/spl/spl_misc.c:107 #5 0xffffffff821cd7f1 in receive_object (rwa=rwa@entry=0xfffff80729ea1900, drro=drro@entry=0xfffff8086864a488, data=0xfffff8082a2a1200) at /usr/src/sys/contrib/openzfs/module/zfs/dmu_recv.c:1799 #6 0xffffffff821cb9a5 in receive_process_record (rwa=0xfffff80729ea1900, rrd=0xfffff8086864a480) at /usr/src/sys/contrib/openzfs/module/zfs/dmu_recv.c:2711 #7 receive_writer_thread (arg=arg@entry=0xfffff80729ea1900) at /usr/src/sys/contrib/openzfs/module/zfs/dmu_recv.c:2806 #8 0xffffffff80bc7dde in fork_exit ( callout=0xffffffff821cb890 <receive_writer_thread>, arg=0xfffff80729ea1900, frame=0xfffffe020ae5abc0) at /usr/src/sys/kern/kern_fork.c:1069 #9 <signal handler called> #10 0x00000008007b888a in ?? () Backtrace stopped: Cannot access memory at address 0x7fffffff5088 (kgdb)
related: https://github.com/openzfs/zfs/issues/12001
We're seeing a similar behaviour on a GENERIC kernel with version: FreeBSD 13.1-STABLE #1 stable/13-n254017-664ccd7e3335: Thu Jan 26 14:22:27 CET 2023 A crash during receive to an encrypted filesystem. Unread portion of the kernel message buffer: panic: VERIFY3(0 == zap_remove(mos, dsobj, spa_feature_table[f].fi_guid, tx)) failed (0 == 2) cpuid = 26 time = 1686583749 KDB: stack backtrace: #0 0xffffffff80c529d5 at kdb_backtrace+0x65 #1 0xffffffff80c05361 at vpanic+0x151 #2 0xffffffff82174e3a at spl_panic+0x3a #3 0xffffffff821fc1e6 at dsl_dataset_deactivate_feature_impl+0xe6 #4 0xffffffff82200cb5 at dsl_dataset_clone_swap_sync_impl+0x135 #5 0xffffffff821e6ff2 at dmu_recv_end_sync+0x2a2 #6 0xffffffff82228244 at dsl_sync_task_sync+0xb4 #7 0xffffffff8221aa6b at dsl_pool_sync+0x42b #8 0xffffffff8224f920 at spa_sync+0xb00 #9 0xffffffff82264271 at txg_sync_thread+0x281 #10 0xffffffff80bc1bee at fork_exit+0x7e #11 0xffffffff81087cbe at fork_trampoline+0xe Uptime: 136d15h27m59s Dumping 16448 out of 130905 MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91% __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55 55 __asm("movq %%gs:%P1,%0" : "=r" (td) : "n" (offsetof(struct pcpu, (kgdb) bt #0 __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55 #1 dump_savectx () at /usr/src/sys/kern/kern_shutdown.c:394 #2 0xffffffff80c04f58 in dumpsys (di=0x0) at /usr/src/sys/x86/include/dump.h:87 #3 doadump (textdump=<optimized out>) at /usr/src/sys/kern/kern_shutdown.c:423 #4 kern_reboot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:497 #5 0xffffffff80c053ce in vpanic (fmt=<optimized out>, ap=ap@entry=0xfffffe02b8c04930) at /usr/src/sys/kern/kern_shutdown.c:930 #6 0xffffffff82174e3a in spl_panic (file=<optimized out>, func=<optimized out>, line=<unavailable>, fmt=<unavailable>) at /usr/src/sys/contrib/openzfs/module/os/freebsd/spl/spl_misc.c:107 #7 0xffffffff821fc1e6 in dsl_dataset_deactivate_feature_impl (ds=ds@entry=0xfffff81ae1019000, f=f@entry=SPA_FEATURE_USEROBJ_ACCOUNTING, tx=tx@entry=0xfffff818a6d9dd00) at /usr/src/sys/contrib/openzfs/module/zfs/dsl_dataset.c:1116 #8 0xffffffff82200cb5 in dsl_dataset_clone_swap_sync_impl (clone=0xfffff816447f0000, origin_head=<unavailable>, tx=<unavailable>, tx@entry=0xfffff818a6d9dd00) at /usr/src/sys/contrib/openzfs/module/zfs/dsl_dataset.c:4083 #9 0xffffffff821e6ff2 in dmu_recv_end_sync (arg=0xfffffe0267bb75b8, tx=0xfffff818a6d9dd00) at /usr/src/sys/contrib/openzfs/module/zfs/dmu_recv.c:3233 #10 0xffffffff82228244 in dsl_sync_task_sync (dst=0xfffffe0267bb73a8, tx=tx@entry=0xfffff818a6d9dd00) at /usr/src/sys/contrib/openzfs/module/zfs/dsl_synctask.c:248 #11 0xffffffff8221aa6b in dsl_pool_sync (dp=dp@entry=0xfffff81576ea8000, txg=txg@entry=39350549) at /usr/src/sys/contrib/openzfs/module/zfs/dsl_pool.c:847 #12 0xffffffff8224f920 in spa_sync_iterate_to_convergence (spa=0xfffffe02a4c77000, tx=0xfffff81cd3091400) at /usr/src/sys/contrib/openzfs/module/zfs/spa.c:9069 #13 spa_sync (spa=spa@entry=0xfffffe02a4c77000, txg=txg@entry=39350549) at /usr/src/sys/contrib/openzfs/module/zfs/spa.c:9287 #14 0xffffffff82264271 in txg_sync_thread (arg=arg@entry=0xfffff81576ea8000) at /usr/src/sys/contrib/openzfs/module/zfs/txg.c:591 #15 0xffffffff80bc1bee in fork_exit (callout=0xffffffff82263ff0 <txg_sync_thread>, arg=0xfffff81576ea8000, frame=0xfffffe02b8c04f40) at /usr/src/sys/kern/kern_fork.c:1093 #16 <signal handler called> #17 mi_startup () at /usr/src/sys/kern/init_main.c:322 #18 0xffffffff80f79919 in swapper () at /usr/src/sys/vm/vm_swapout.c:755 #19 0xffffffff8038a022 in btext () at /usr/src/sys/amd64/amd64/locore.S:80 (kgdb)