Bug 282744

Summary: setfib does not add the default gateway to the second routing table.
Product: Base System Reporter: Vladyslav V. Prodan <admin>
Component: binAssignee: freebsd-net (Nobody) <net>
Status: New ---    
Severity: Affects Only Me CC: zarychtam
Priority: ---    
Version: 14.1-RELEASE   
Hardware: Any   
OS: Any   

Description Vladyslav V. Prodan 2024-11-13 23:47:09 UTC 
# uname -a
FreeBSD vb-14-1.domain.com 14.1-PRERELEASE FreeBSD 14.1-PRERELEASE #0 stable/14-n267586-19e335596658: Thu May  2 20:41:56 UTC 2024     root@releng1.nyi.freebsd.org:/usr/obj/usr/src/amd64.amd64/sys/GENERIC amd64

# grep fibs /boot/loader.conf
net.fibs=2

root@vb-14-1:~ # ifconfig em2
em2: flags=1008843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
        options=48505bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,LRO,VLAN_HWFILTER,VLAN_HWTSO,HWSTATS,MEXTPG>
        ether 08:00:27:33:a3:63
        inet6 fe80::a00:27ff:fe33:a363%em2 prefixlen 64 scopeid 0x3
        media: Ethernet autoselect (1000baseT <full-duplex>)
        status: active
        nd6 options=23<PERFORMNUD,ACCEPT_RTADV,AUTO_LINKLOCAL>
root@vb-14-1:~ # setfib 1 ifconfig em2 inet 192.168.77.104/24
root@vb-14-1:~ # setfib 1 route add default 192.168.77.2
add net default: gateway 192.168.77.2 fib 1: Invalid argument


root@vb-14-1:~ # ifconfig em2
em2: flags=1008843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
        options=48505bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,LRO,VLAN_HWFILTER,VLAN_HWTSO,HWSTATS,MEXTPG>
        ether 08:00:27:33:a3:63
        inet 192.168.77.104 netmask 0xffffff00 broadcast 192.168.77.255
        inet6 fe80::a00:27ff:fe33:a363%em2 prefixlen 64 scopeid 0x3
        media: Ethernet autoselect (1000baseT <full-duplex>)
        status: active
        nd6 options=23<PERFORMNUD,ACCEPT_RTADV,AUTO_LINKLOCAL>




The same commands work fine in 12.3-STABLE.

root@vb-12-3:~ # sysctl net.fibs
net.fibs: 5
root@vb-12-3:~ # ifconfig em2
em2: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
        options=81009b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,VLAN_HWFILTER>
        ether 08:00:27:51:7f:dd
        inet6 fe80::a00:27ff:fe51:7fdd%em2 prefixlen 64 scopeid 0x3
        media: Ethernet autoselect (1000baseT <full-duplex>)
        status: active
        nd6 options=23<PERFORMNUD,ACCEPT_RTADV,AUTO_LINKLOCAL>
root@vb-12-3:~ # setfib 1 ifconfig em2 inet 192.168.77.202/24
root@vb-12-3:~ # setfib 1 route add default 192.168.77.2
add net default: gateway 192.168.77.2 fib 1
root@vb-12-3:~ # netstat -rnF 1
Routing tables (fib: 1)

Internet:
Destination        Gateway            Flags     Netif Expire
default            192.168.77.2       UGS         em2
127.0.0.1          link#6             UH          lo0
192.168.4.0/24     link#1             U           em0
192.168.56.0/24    link#4             U           em3
192.168.66.0/24    link#2             U           em1
192.168.77.0/24    link#3             U           em2

Internet6:
Destination                       Gateway                       Flags     Netif Expire
::/96                             ::1                           UGRS        lo0
::1                               link#6                        UH          lo0
::ffff:0.0.0.0/96                 ::1                           UGRS        lo0
fda2:223c:f4f::/64                link#1                        U           em0
fe80::/10                         ::1                           UGRS        lo0
fe80::%em0/64                     link#1                        U           em0
fe80::%em1/64                     link#2                        U           em1
fe80::%em2/64                     link#3                        U           em2
fe80::%em3/64                     link#4                        U           em3
fe80::%lo0/64                     link#6                        U           lo0
ff02::/16                         ::1                           UGRS        lo0
root@vb-12-3:~ # netstat -rnF 0
Routing tables

Internet:
Destination        Gateway            Flags     Netif Expire
default            192.168.4.1        UGS         em0
127.0.0.1          link#6             UH          lo0
192.168.4.0/24     link#1             U           em0
192.168.4.234      link#1             UHS         lo0
192.168.56.0/24    link#4             U           em3
192.168.56.107     link#4             UHS         lo0
192.168.66.0/24    link#2             U           em1
192.168.66.109     link#2             UHS         lo0
192.168.77.0/24    link#3             U           em2
192.168.77.202     link#3             UHS         lo0

Internet6:
Destination                       Gateway                       Flags     Netif Expire
::/96                             ::1                           UGRS        lo0
::1                               link#6                        UH          lo0
::ffff:0.0.0.0/96                 ::1                           UGRS        lo0
fda2:223c:f4f::/64                link#1                        U           em0
fda2:223c:f4f:0:a00:27ff:feac:236c link#1                       UHS         lo0
fe80::/10                         ::1                           UGRS        lo0
fe80::%em0/64                     link#1                        U           em0
fe80::a00:27ff:feac:236c%em0      link#1                        UHS         lo0
fe80::%em1/64                     link#2                        U           em1
fe80::a00:27ff:febc:8e4b%em1      link#2                        UHS         lo0
fe80::%em2/64                     link#3                        U           em2
fe80::a00:27ff:fe51:7fdd%em2      link#3                        UHS         lo0
fe80::%em3/64                     link#4                        U           em3
fe80::a00:27ff:fe94:7274%em3      link#4                        UHS         lo0
fe80::%lo0/64                     link#6                        U           lo0
fe80::1%lo0                       link#6                        UHS         lo0
ff02::/16                         ::1                           UGRS        lo0
root@vb-12-3:~ # netstat -rnF 1
Routing tables (fib: 1)

Internet:
Destination        Gateway            Flags     Netif Expire
default            192.168.77.2       UGS         em2
127.0.0.1          link#6             UH          lo0
192.168.4.0/24     link#1             U           em0
192.168.56.0/24    link#4             U           em3
192.168.66.0/24    link#2             U           em1
192.168.77.0/24    link#3             U           em2

Internet6:
Destination                       Gateway                       Flags     Netif Expire
::/96                             ::1                           UGRS        lo0
::1                               link#6                        UH          lo0
::ffff:0.0.0.0/96                 ::1                           UGRS        lo0
fda2:223c:f4f::/64                link#1                        U           em0
fe80::/10                         ::1                           UGRS        lo0
fe80::%em0/64                     link#1                        U           em0
fe80::%em1/64                     link#2                        U           em1
fe80::%em2/64                     link#3                        U           em2
fe80::%em3/64                     link#4                        U           em3
fe80::%lo0/64                     link#6                        U           lo0
ff02::/16                         ::1                           UGRS        lo0
Comment 1 Vladyslav V. Prodan 2024-11-13 23:55:06 UTC 
root@vb-12-3:~ # uname -a
FreeBSD vb-12-3.3 12.3-STABLE FreeBSD 12.3-STABLE #0 r372168M: Thu Jun 23 09:30:              29 EEST 2022     root@vb-12-3:/usr/obj/usr/src/amd64.amd64/sys/universal-12-3  a              md64
Comment 2 Marek Zarychta 2024-11-14 06:12:23 UTC 
(In reply to Vladyslav V. Prodan from comment #0)
What is the setting of the sysctl knob "net.add_addr_allfibs"?

Perhaps you can try to assign the route this way:

# route add -net 192.168.77.0/24 -iface em2 -fib 1
# route add default 192.168.77.2 -fib 1
Comment 3 Marek Zarychta 2024-11-14 06:29:34 UTC 
If interface em2 should or could be moved to fib 1, then a similar effect could still be achievable with the following:
 
# ifconfig em2 inet 192.168.77.104/24 fib 1
# route add default 192.168.77.2 -fib 1
Comment 4 Vladyslav V. Prodan 2024-11-14 19:04:55 UTC 
(In reply to Marek Zarychta from comment #2)


Thank you, it helped.

root@vb-14-1:~ # ifconfig em2 inet 192.168.77.104/24 fib 1 up
root@vb-14-1:~ # ifconfig em2
em2: flags=1008843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 15                                     00
        options=48505bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM                                     ,TSO4,LRO,VLAN_HWFILTER,VLAN_HWTSO,HWSTATS,MEXTPG>
        ether 08:00:27:33:a3:63
        inet 192.168.77.104 netmask 0xffffff00 broadcast 192.168.77.255
        inet6 fe80::a00:27ff:fe33:a363%em2 prefixlen 64 scopeid 0x3
        fib: 1
        media: Ethernet autoselect (1000baseT <full-duplex>)
        status: active
        nd6 options=23<PERFORMNUD,ACCEPT_RTADV,AUTO_LINKLOCAL>
root@vb-14-1:~ # route add default 192.168.77.2 -fib 1
add net default: gateway 192.168.77.2 fib 1

root@vb-14-1:~ # netstat -rnF 1
Routing tables (fib: 1)

Internet:
Destination        Gateway            Flags     Netif Expire
default            192.168.77.2       UGS         em2
127.0.0.1          link#5             UHS         lo0
192.168.77.0/24    link#3             U           em2
192.168.77.104     link#5             UHS         lo0

Internet6:
Destination                       Gateway                       Flags     Netif Exp                                     ire
::/96                             link#5                        URS         lo0
::1                               link#5                        UHS         lo0
::ffff:0.0.0.0/96                 link#5                        URS         lo0
fe80::%lo0/10                     link#5                        URS         lo0
ff02::/16                         link#5                        URS         lo0


But! for ipv6 addresses to appear in fib 1, you need to reboot the system...