The function genl_register_family() returns an error if the name of the family is already in use, but does not release the lock.
The next call to genl_register_family() will block indefinitely.

A side comment, why does the function panic if the maximum number of families are already registered? It could just return an error.

--- netlink_generic.old.c       2025-11-08 08:55:46.012479000 -0300
+++ netlink_generic.c   2025-11-08 09:00:00.541146000 -0300
@@ -366,8 +366,10 @@
        GENL_LOCK();
        for (u_int i = 0; i < MAX_FAMILIES; i++)
                if (families[i].family_name != NULL &&
-                   strcmp(families[i].family_name, family_name) == 0)
+                   strcmp(families[i].family_name, family_name) == 0) {
+                       GENL_UNLOCK();
                        return (0);
+               }

        /* Microoptimization: index 0 is reserved for the control family. */
        gf = NULL;
@@ -376,6 +378,10 @@
                        gf = &families[i];
                        break;
                }
+
+       //
+       //XXX: Why kernel panic? It could return 0 as register group below.
+       //
        KASSERT(gf, ("%s: maximum of %u generic netlink families allocated",
            __func__, MAX_FAMILIES));


//----------------------------------------------------------------------------

To test the bug, just creates a module that register a family.
You load and unload it 3 times: 1st ok, 2nd error, 3rd deadlock.

#include <sys/param.h>
#include <sys/module.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/malloc.h>

#include <netlink/netlink.h>
#include <netlink/netlink_ctl.h>
#include <netlink/netlink_generic.h>


static int
genltest_modevent(module_t mod __unused, int event, void *arg __unused)
{
        int error = 0;
        switch (event) {
        case MOD_LOAD:
                uprintf("Generic netlink load\n");
                uint16_t id = genl_register_family("genltest", 0, 2, 0);
                uprintf("Register: %d\n", id);
                break;
        case MOD_UNLOAD:
                uprintf("Generic netlink unload\n");
                break;
        default:
                error = EOPNOTSUPP;
                break;
        }
        return (error);
}

static moduledata_t genltest_mod = {
        "genltest",
        genltest_modevent,
        NULL
};

DECLARE_MODULE(genltest, genltest_mod, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);