Bug 248468 - jail(8) host has no internet access when vnet & non-vnet jails running at same time
Summary: jail(8) host has no internet access when vnet & non-vnet jails running at sam...
Status: New
Alias: None
Product: Base System
Classification: Unclassified
Component: kern (show other bugs)
Version: 12.1-RELEASE
Hardware: Any Any
: --- Affects Many People
Assignee: freebsd-jail (Nobody)
URL:
Keywords:
Depends on:
Blocks:
 
Reported: 2020-08-04 15:06 UTC by joeb1
Modified: 2020-08-12 22:33 UTC (History)
2 users (show)

See Also:


Attachments

Note You need to log in before you can comment on or make changes to this bug.
Description joeb1 2020-08-04 15:06:22 UTC
Equipment. Real hardware, 12.1 release, amd64 dual cpu.

Summery Description;
non-vnet jails and vnet jails using the bridge/epair method can ping the public internet when only non-vnet jails are started at a time or when only vnet jails are started at a time. But when both non-vnet jails and vnet jails are started together then neither one can ping the public internet. The order of the jails definitions in the jail.conf file has no effect on changing what is happening.
The vnet jail doesn't run a firewall inside of it so the hosts firewall is doing the NATing of the private addresses.

Bug description:
When non-vnet jails are started their ip addresses are added to the NIC facing the public AFTER the public ip address and the non-vnet jail has access to the public internet. But when both non-vnet jails and vnet jails are started at the same time then the non-vnet jails ip addresses gets added BEFORE the public ip address of the NIC facing the public internet causing the host to lose all access to the public internet. 

It makes no difference which command method is used to start and stop the jails.
Service jail start jailname   or  jail –cv jailname

The following is a capture of the command sequence showing this bug. Follow the re0 NIC public ip address xx.25.51.0 in the ifconfig -a listing.


Before any jails are started.
/root >ifconfig -a
snip ...
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
    options=8209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
    ether 50:3e:aa:06:11:22
    inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
    media: Ethernet autoselect (1000baseT <full-duplex,master>)
    status: active
    nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
    ether 02:3e:ba:a7:58:00
    id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
    maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
    root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
    member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
            ifmaxaddr 0 port 2 priority 128 path cost 20000
    groups: bridge
    nd6 options=1<PERFORMNUD>


/root >cat /etc/jail.conf

#  non-vnet jail
zdir20 {
host.hostname       =  "zdir20";
path                =  "/usr/jails/zdir20";
mount.fstab         =  "/usr/local/etc/fstab/zdir20";
exec.consolelog     =  "/var/log/zdir20.console.log";
mount.devfs;
ip4.addr            =  10.0.22.5;
interface           =  "re0";
allow.raw_sockets;
devfs_ruleset       =  "4";
exec.start          =  "/bin/sh /etc/rc";
exec.stop           =  "/bin/sh /etc/rc.shutdown";
}

#  vnet jail using the bridge/epair method
v0jail1 {
host.hostname   = "v0jail1";
path            = "/usr/jails/v0jail1";
mount.fstab     = "/usr/local/etc/fstab/v0jail1";
exec.consolelog = "/var/log/v0jail1.console.log";
mount.devfs;
devfs_ruleset   = "4";
vnet            = "new";
vnet.interface  = "epair55b";
exec.prestart   = "ifconfig epair55  create up";
exec.prestart  += "ifconfig bridge0 addm epair55a";
exec.prestart  += "ifconfig epair55a descr vnet-v0jail1";
exec.prestart  += "ifconfig bridge0 inet 10.0.48.2 netmask 255.255.255.0 alias";
exec.start      = "/bin/sh /etc/rc";
exec.start     += "ifconfig epair55b inet 10.0.48.1 netmask 255.255.255.0";
exec.start     += "route add default 10.0.48.2";
exec.prestop    = "ifconfig epair55b -vnet v0jail1";
exec.stop       = "/bin/sh /etc/rc.shutdown";
exec.poststop   = "ifconfig bridge0 deletem epair55a";
exec.poststop  += "sleep 2";
exec.poststop  += "ifconfig epair55a destroy";
exec.poststop  += "ifconfig bridge0 inet 10.0.48.2 -alias";
}


/root >jls
   JID  IP Address      Hostname                      Path

# start only the non-vnet jail
/root >service jail start zdir20
Starting jails: zdir20.

/root >jls
   JID  IP Address      Hostname                      Path
    18  10.0.22.5       zdir20                        /usr/jails/zdir20

# Take notice that the non-vnet jails ip address follows the nic’s
# public ip address.
/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
    options=8209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
    ether 50:3e:aa:06:11:22
    inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
    inet 10.0.22.5 netmask 0xffffffff broadcast 10.0.22.5
    media: Ethernet autoselect (1000baseT <full-duplex,master>)
    status: active
    nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
    ether 02:3e:ba:a7:58:00
    id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
    maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
    root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
    member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
            ifmaxaddr 0 port 2 priority 128 path cost 20000
    groups: bridge
    nd6 options=1<PERFORMNUD>

# login to the non-vnet jail and ping the public
/root >jexec zdir20 login -f root
Last login: Sun Aug  2 11:30:40 on pts/0
FreeBSD 12.1-RELEASE-p6 GENERIC

Welcome to your FreeBSD jail.
zdir20 /root >
zdir20 /root >ping -c 2 freebsd.org
PING freebsd.org (96.47.72.84): 56 data bytes
64 bytes from 96.47.72.84: icmp_seq=0 ttl=48 time=44.426 ms
64 bytes from 96.47.72.84: icmp_seq=1 ttl=48 time=44.481 ms

--- freebsd.org ping statistics ---
2 packets transmitted, 2 packets received, 0.0% packet loss
round-trip min/avg/max/stddev = 44.426/44.453/44.481/0.027 ms
zdir20 /root >exit
logout

# stop the non-vnet jail and show that the network is back to
# starting condition.
/root >service jail stop zdir20
Stopping jails: zdir20.

/root >jls
   JID  IP Address      Hostname                      Path

/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
    options=8209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
    ether 50:3e:aa:06:11:22
    inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
    media: Ethernet autoselect (1000baseT <full-duplex,master>)
    status: active
    nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
    ether 02:3e:ba:a7:58:00
    id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
    maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
    root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
    member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
            ifmaxaddr 0 port 2 priority 128 path cost 20000
    groups: bridge
    nd6 options=1<PERFORMNUD>

# start only the vnet jail and see the bridge0
/root >service jail start v0jail1
Starting jails: v0jail1.
/root >jls
   JID  IP Address      Hostname                      Path
    19                  v0jail1                       /usr/jails/v0jail1

/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
    options=82099<RXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
    ether 50:3e:aa:06:11:22
    inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
    media: Ethernet autoselect (1000baseT <full-duplex>)
    status: active
    nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
    ether 02:3e:ba:a7:58:00
    inet 10.0.48.2 netmask 0xffffff00 broadcast 10.0.48.255
    id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
    maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
    root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
    member: epair55a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
            ifmaxaddr 0 port 5 priority 128 path cost 2000
    member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
            ifmaxaddr 0 port 2 priority 128 path cost 20000
    groups: bridge
    nd6 options=1<PERFORMNUD>
epair55a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
    description: vnet-v0jail1
    options=8<VLAN_MTU>
    ether 02:eb:be:f5:15:0a
    inet6 fe80::eb:beff:fef5:150a%epair55a prefixlen 64 scopeid 0x5
    groups: epair
    media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
    status: active
    nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>

# login to the vnet jail and ping the public internet.
/root >jexec v0jail1 login -f root
Last login: Sun Aug  2 11:29:41 on pts/0
FreeBSD 12.1-RELEASE-p6 GENERIC

Welcome to your FreeBSD jail.
v0jail1 /root >ping -c 2 freebsd.org
PING freebsd.org (96.47.72.84): 56 data bytes
64 bytes from 96.47.72.84: icmp_seq=0 ttl=47 time=46.745 ms
64 bytes from 96.47.72.84: icmp_seq=1 ttl=47 time=43.930 ms

--- freebsd.org ping statistics ---
2 packets transmitted, 2 packets received, 0.0% packet loss
round-trip min/avg/max/stddev = 43.930/45.337/46.745/1.407 ms
v0jail1 /root >exit
logout


# close the vnet jail and return to starting condition.
/root >service jail stop v0jail1
Stopping jails: v0jail1.


/root >jls
   JID  IP Address      Hostname                      Path

/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
    options=8209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
    ether 50:3e:aa:06:11:22
    inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
    media: Ethernet autoselect (1000baseT <full-duplex>)
    status: active
    nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
    ether 02:3e:ba:a7:58:00
    id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
    maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
    root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
    member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
            ifmaxaddr 0 port 2 priority 128 path cost 20000
    groups: bridge
    nd6 options=1<PERFORMNUD>



# Start both the non-vnet jail and the vnet jail together.
/root >service jail start
Starting jails: zdir20 v0jail1.

# login to the non-vnet jail and it has no public access.
/root >jexec zdir20 login -f root [K
Last login: Sun Aug  2 11:36:34 on pts/0
FreeBSD 12.1-RELEASE-p6 GENERIC

Welcome to your FreeBSD jail.
zdir20 /root >ping -c 2 freebsd.org
ping: cannot resolve freebsd.org: Host name lookup failure
zdir20 /root >exit
logout


# login to the vnet jail and it has no public access.
/root >jexec v0jail1 login -f root
Last login: Sun Aug  2 11:38:56 on pts/0
FreeBSD 12.1-RELEASE-p6 GENERIC

Welcome to your FreeBSD jail.
v0jail1 /root >ping -c 2 freebsd.org
ping: cannot resolve freebsd.org: Host name lookup failure
v0jail1 /root >exit
logout
/root >jls
   JID  IP Address      Hostname                      Path
    20  10.0.22.5       zdir20                        /usr/jails/zdir20
    21                  v0jail1                       /usr/jails/v0jail1

# Here is the bug. See that the non-vnet jail ip address comes before the
# public address causing the host to lose access to the public internet.
/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
    options=82099<RXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
    ether 50:3e:aa:06:11:22
    inet 10.0.22.5 netmask 0xffffffff broadcast 10.0.22.5
    inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
    media: Ethernet autoselect (1000baseT <full-duplex>)
    status: active
    nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
    ether 02:3e:ba:a7:58:00
    inet 10.0.48.2 netmask 0xffffff00 broadcast 10.0.48.255
    id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
    maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
    root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
    member: epair55a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
            ifmaxaddr 0 port 5 priority 128 path cost 2000
    member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
            ifmaxaddr 0 port 2 priority 128 path cost 20000
    groups: bridge
    nd6 options=1<PERFORMNUD>
epair55a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
    description: vnet-v0jail1
    options=8<VLAN_MTU>
    ether 02:77:b8:5f:e4:0a
    inet6 fe80::77:b8ff:fe5f:e40a%epair55a prefixlen 64 scopeid 0x5
    groups: epair
    media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
    status: active
    nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>

# stop both jails and return to starting condition.
/root >service jail stop
Stopping jails: zdir20 v0jail1.

/root >jls
   JID  IP Address      Hostname                      Path

/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
    options=8209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
    ether 50:3e:aa:06:11:22
    inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
    media: Ethernet autoselect (1000baseT <full-duplex,master>)
    status: active
    nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
    ether 02:3e:ba:a7:58:00
    id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
    maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
    root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
    member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
            ifmaxaddr 0 port 2 priority 128 path cost 20000
    groups: bridge
    nd6 options=1<PERFORMNUD>