From aeda05bed7844563af38bdd4f2b7df0f53ccc6f6 Mon Sep 17 00:00:00 2001 From: bergware Date: Thu, 10 Aug 2023 22:08:04 +0200 Subject: [PATCH] Make bridge / macvtap selection per physical interface --- .../DockerSettings.page | 3 +- .../dynamix.vm.manager/include/libvirt.php | 9 +- .../include/libvirt_helpers.php | 6 +- etc/rc.d/rc.docker | 124 +++++++++++------- etc/rc.d/rc.libvirt | 44 +++++-- 5 files changed, 112 insertions(+), 74 deletions(-) diff --git a/emhttp/plugins/dynamix.docker.manager/DockerSettings.page b/emhttp/plugins/dynamix.docker.manager/DockerSettings.page index 79f593ab3..190852c4f 100644 --- a/emhttp/plugins/dynamix.docker.manager/DockerSettings.page +++ b/emhttp/plugins/dynamix.docker.manager/DockerSettings.page @@ -23,7 +23,8 @@ require_once "$docroot/plugins/dynamix.docker.manager/include/DockerClient.php"; $DockerClient = new DockerClient(); exec("/etc/rc.d/rc.docker status >/dev/null",$dummy,$DockerStopped); -$bridge = file_exists('/sys/class/net/br0'); +exec("ls --indicator-style=none /sys/class/net|awk '/^br[0-9]+$/'",$nics); +$bridge = count($nics)>0; function strposX($s, $c, $n=1) { $p = 0; diff --git a/emhttp/plugins/dynamix.vm.manager/include/libvirt.php b/emhttp/plugins/dynamix.vm.manager/include/libvirt.php index 367eca626..89f171297 100644 --- a/emhttp/plugins/dynamix.vm.manager/include/libvirt.php +++ b/emhttp/plugins/dynamix.vm.manager/include/libvirt.php @@ -661,12 +661,7 @@ $netmodel = $nic['model'] ?: 'virtio-net'; $net_res =$this->libvirt_get_net_res($this->conn, $nic['network']); - $bridge = file_exists('/sys/class/net/br0'); - if ($bridge) { - exec("brctl show|grep -Po '^(vir)?br[0-9]+(\.[0-9]+)?'", $br); - } else { - exec("ip -br a|grep -Po '^(virbr|vhost)[0-9][^@ ]*'",$br); - } + exec("ip -br a|grep -Po '^((vir)?br|vhost)[0-9]+(\.[0-9]+)?'",$br); if ($nic["boot"] != NULL) $nicboot = "" ; else $nicboot = "" ; if ($net_res) { $netstr .= " @@ -676,7 +671,7 @@ $nicboot "; } elseif (in_array($nic['network'], $br)) { - if ($bridge || $nic['network']=='virbr0') { + if (preg_match('/^(vir)?br/',$nic['network'])) { $netstr .= " diff --git a/emhttp/plugins/dynamix.vm.manager/include/libvirt_helpers.php b/emhttp/plugins/dynamix.vm.manager/include/libvirt_helpers.php index c7bc6e208..3b4e49664 100644 --- a/emhttp/plugins/dynamix.vm.manager/include/libvirt_helpers.php +++ b/emhttp/plugins/dynamix.vm.manager/include/libvirt_helpers.php @@ -1066,11 +1066,7 @@ private static $encoding = 'UTF-8'; function getValidNetworks() { global $lv; $arrValidNetworks = []; - if (file_exists('/sys/class/net/br0')) { - exec("brctl show|grep -Po '^(vir)?br[0-9]+(\.[0-9]+)?'", $arrBridges); - } else { - exec("ip -br a|grep -Po '^(virbr|vhost)[0-9][^@ ]*'",$arrBridges); - } + exec("ip -br a|grep -Po '^((vir)?br|vhost)[0-9]+(\.[0-9]+)?'",$arrBridges); if (!is_array($arrBridges)) { $arrBridges = []; } diff --git a/etc/rc.d/rc.docker b/etc/rc.d/rc.docker index 97cf93611..9b73fc5f9 100755 --- a/etc/rc.d/rc.docker +++ b/etc/rc.d/rc.docker @@ -14,6 +14,7 @@ UNSHARE=/usr/bin/unshare SYSTEM=/sys/class/net CONF6=/proc/sys/net/ipv6/conf ACTIVE=$(ls --indicator-style=none $SYSTEM|awk '/^(bond|br|eth)[0-9]/' ORS=' ') +NICS=$(ls --indicator-style=none $SYSTEM|awk '/^eth[0-9]+$/') DOCKERD=dockerd DOCKER=/usr/bin/$DOCKERD @@ -33,11 +34,19 @@ TMP=/var/tmp/network.tmp # Set defaults used by the docker daemon if [[ -f $DOCKER_CFG ]]; then - if ! grep -qPm1 "_${PORT^^}(_[0-9]+)?=" $DOCKER_CFG; then - # interface has changed, update configuration - sed -ri "s/_(BR0|BOND0|ETH0)(_[0-9]+)?=/_${PORT^^}\2=/" $DOCKER_CFG - sed -ri "s/(br0|bond0|eth0)(\.[0-9]+ )/$PORT\2/g" $DOCKER_CFG - fi + for NIC in $NICS; do + if [[ -e $SYSTEM/${NIC/eth/br} ]]; then + NIC=${NIC/eth/br} + elif [[ -e $SYSTEM/${NIC/eth/bond} ]]; then + NIC=${NIC/eth/bond} + fi + if ! grep -qPm1 "_${NIC^^}(_[0-9]+)?=" $DOCKER_CFG; then + # interface has changed, update configuration + X=${NIC//[^0-9]/} + sed -ri "s/_(BR|BOND|ETH)$X(_[0-9]+)?=/_${NIC^^}\2=/" $DOCKER_CFG + sed -ri "s/(br|bond|eth)$X(\.[0-9]+)? /$NIC\2 /g" $DOCKER_CFG + fi + done # Read (updated) unRAID docker configuration file . $DOCKER_CFG fi @@ -77,24 +86,6 @@ else [[ -e $SYSTEM/docker0 ]] && echo 1 > $CONF6/docker0/disable_ipv6 fi -# user selection when bridge is enabled -if [[ -z $DOCKER_NETWORK_TYPE ]]; then - DETACH='ipvlan' - ATTACH='macvlan' - MODE='bridge' -else - DETACH='macvlan' - ATTACH='ipvlan' - MODE='l2 bridge' -fi - -# fixed selection when bridge is disabled -if [[ $PORT != br0 ]]; then - DETACH='ipvlan' - ATTACH='macvlan' - MODE='bridge' -fi - export DOCKER_RAMDISK=true # Get docker daemon PID (if existing) @@ -167,9 +158,29 @@ wipe() { echo ${wet[@]/$wet} } +# Network driver +driver() { + # user selection when bridge is enabled + if [[ -z $DOCKER_NETWORK_TYPE ]]; then + DETACH='ipvlan' + ATTACH='macvlan' + MODE='bridge' + else + DETACH='macvlan' + ATTACH='ipvlan' + MODE='l2 bridge' + fi + # fixed selection when bridge is disabled + if [[ $1 != br ]]; then + DETACH='ipvlan' + ATTACH='macvlan' + MODE='bridge' + fi +} + # Custom networks network(){ - docker network ls --filter driver="$1" --format='{{.Name}}' 2>/dev/null|tr '\n' ' ' + docker network ls --filter driver="$1" --format='{{.Name}}' 2>/dev/null|grep -P "^[a-z]+$2(\$|\.)"|tr '\n' ' ' } # Is container running? @@ -223,7 +234,7 @@ add_route(){ # Add custom networks start_network(){ # create list of possible custom networks - EXCLUDE=; INCLUDE=$(ls --indicator-style=none $SYSTEM|awk '/^br[0-9]/' ORS=' ') + EXCLUDE=; INCLUDE=$(ls --indicator-style=none $SYSTEM|awk '/^br[0-9]+/' ORS=' ') while IFS=$'\n' read -r NETWORK; do if [[ ${NETWORK:0:4} == bond ]]; then if [[ $INCLUDE =~ "${NETWORK/bond/br} " ]]; then @@ -239,7 +250,7 @@ start_network(){ INCLUDE="${INCLUDE}${NETWORK} " fi fi - done <<< $(ls --indicator-style=none $SYSTEM|grep -P '^(bond|eth)[0-9]') + done <<< $(ls --indicator-style=none $SYSTEM|grep -P '^(bond|eth)[0-9]+') wait_daemon if ! is_docker_running; then return 1; fi # get container settings for custom networks to reconnect later @@ -286,18 +297,27 @@ start_network(){ done done # detach custom networks - for NETWORK in $(network $DETACH); do - [[ $STOCK =~ ${NETWORK%%[0-9]*} || $DOCKER_USER_NETWORKS != preserve ]] && docker network rm $NETWORK >/dev/null - done - # get existing custom networks - for NETWORK in $(network $ATTACH); do - if [[ $STOCK =~ ${NETWORK%%[0-9]*} ]]; then - [[ $EXCLUDE =~ "$NETWORK " || ! $ACTIVE =~ "$NETWORK " ]] && docker network rm $NETWORK >/dev/null - else - [[ $DOCKER_USER_NETWORKS != preserve ]] && docker network rm $NETWORK >/dev/null + for NIC in $NICS; do + if [[ -e $SYSTEM/${NIC/eth/br} ]]; then + NIC=${NIC/eth/br} + elif [[ -e $SYSTEM/${NIC/eth/bond} ]]; then + NIC=${NIC/eth/bond} fi + X=${NIC//[^0-9]/} + driver ${NIC//[0-9]/} + for NETWORK in $(network $DETACH $X); do + [[ $STOCK =~ ${NETWORK%%[0-9]*} || $DOCKER_USER_NETWORKS != preserve ]] && docker network rm $NETWORK >/dev/null + done + # get existing custom networks + for NETWORK in $(network $ATTACH $X); do + if [[ $STOCK =~ ${NETWORK%%[0-9]*} ]]; then + [[ $EXCLUDE =~ "$NETWORK " || ! $ACTIVE =~ "$NETWORK " ]] && docker network rm $NETWORK >/dev/null + else + [[ $DOCKER_USER_NETWORKS != preserve ]] && docker network rm $NETWORK >/dev/null + fi + done + NETWORKS=$(network $ATTACH $X) done - NETWORKS=$(network $ATTACH) # add or remove custom network for NETWORK in $INCLUDE; do if [[ ! $DOCKER_CUSTOM_NETWORKS =~ "$NETWORK " ]]; then @@ -428,7 +448,9 @@ start_network(){ [[ -n $RANGE ]] && SERVER="--aux-address=server=${R4%/*}" || SERVER="--aux-address=server=${SHIM_HIGH%/*}" fi fi - [[ ${NETWORK:0:2} == br ]] && VHOST=$NETWORK || VHOST=vhost${NETWORK//[^0-9.]/} + TYPE=${NETWORK//[0-9.]/} + driver $TYPE + [[ $TYPE == br ]] && VHOST=$NETWORK || VHOST=vhost${NETWORK//[^0-9.]/} docker network create -d $ATTACH $SUBNET $GATEWAY $SERVER $RANGE $SUBNET6 $GATEWAY6 $SERVER6 $RANGE6 -o parent=$VHOST $NETWORK | xargs docker network inspect -f "created network $ATTACH {{.Name}} with subnets: {{range .IPAM.Config}}{{.Subnet}}; {{end}}" 2>/dev/null | logger -t $(basename $0) # connect containers to this new network for CONNECT in ${NETRESTORE[$NETWORK]}; do @@ -490,16 +512,24 @@ shim_network(){ # Remove custom networks stop_network(){ - for NETWORK in $(network $ATTACH); do - [[ $STOCK =~ ${NETWORK%%[0-9]*} || $DOCKER_USER_NETWORKS != preserve ]] && docker network rm $NETWORK >/dev/null - done - for LINK in $(ls --indicator-style=none $SYSTEM|grep '^shim-'); do - ip -4 addr flush dev $LINK - ip -4 route flush dev $LINK - ip -6 addr flush dev $LINK - ip -6 route flush dev $LINK - ip link set $LINK down - ip link del $LINK + for NIC in $NICS; do + if [[ -e $SYSTEM/${NIC/eth/br} ]]; then + NIC=${NIC/eth/br} + elif [[ -e $SYSTEM/${NIC/eth/bond} ]]; then + NIC=${NIC/eth/bond} + fi + X=${NIC//[^0-9]/} + driver ${NIC//[0-9]/} + for NETWORK in $(network $ATTACH $X); do + [[ $STOCK =~ ${NETWORK%%[0-9]*} || $DOCKER_USER_NETWORKS != preserve ]] && docker network rm $NETWORK >/dev/null + LINK=shim-$NETWORK + if [[ -e $SYSTEM/$LINK ]]; then + ip -4 addr flush dev $LINK + ip -4 route flush dev $LINK + ip link set $LINK down + ip link del $LINK + fi + done done } diff --git a/etc/rc.d/rc.libvirt b/etc/rc.d/rc.libvirt index f27c919df..4cdc1f584 100755 --- a/etc/rc.d/rc.libvirt +++ b/etc/rc.d/rc.libvirt @@ -20,6 +20,7 @@ if [ -f /boot/config/domain.cfg ]; then . /boot/config/domain.cfg fi +SYSTEM=/sys/class/net MODULES=${MODULES:-"vhost_net"} TIMEOUT=${TIMEOUT:-60} HOSTSHUTDOWN=${HOSTSHUTDOWN:-"shutdown"} @@ -159,7 +160,7 @@ waitstop() { } start_libvirtd() { - if [ -f $LIBVIRTD_PIDFILE ];then + if [[ -f $LIBVIRTD_PIDFILE ]]; then echo "libvirt is already running..." exit 1 fi @@ -171,14 +172,29 @@ start_libvirtd() { sed -ri "s///g" /etc/libvirt/qemu/*.xml &> /dev/null # remove from xml because libvirt + virlogd + virlockd has an issue with locked sed -ri "s///g" /etc/libvirt/qemu/*.xml &> /dev/null - # update interface section of VM configuration files - if [[ -e /sys/class/net/br0 ]]; then - # link VM to bridge interface - sed -ri "s///;s///" /etc/libvirt/qemu/*.xml &> /dev/null - else - # link VM to macvtap interface - sed -ri "s///;s///" /etc/libvirt/qemu/*.xml &> /dev/null - fi + # update interface section((s) of VM configuration files + for XML in /etc/libvirt/qemu/*.xml; do + # get all interface sections + ROW=($(grep -nhP '//" $XML &> /dev/null + sed -ri "$SOURCE s///" $XML &> /dev/null + else + # change to bridge + sed -ri "${ROW[$i]} s///" $XML &> /dev/null + sed -ri "$SOURCE s///" $XML &> /dev/null + fi + fi + done + done # copy any new conf files we dont currently have cp -n /etc/libvirt-/*.conf /etc/libvirt &> /dev/null # ensure tpm-states path exists @@ -191,7 +207,7 @@ start_libvirtd() { } stop_libvirtd() { - if [ ! -f $LIBVIRTD_PIDFILE ];then + if [[ ! -f $LIBVIRTD_PIDFILE ]]; then echo "libvirt is not running..." exit 2 fi @@ -207,7 +223,7 @@ stop_libvirtd() { } start_virtlogd() { - if [ -f $VIRTLOGD_PIDFILE ];then + if [[ -f $VIRTLOGD_PIDFILE ]]; then echo "virtlogd is already running..." exit 1 fi @@ -217,7 +233,7 @@ start_virtlogd() { } stop_virtlogd() { - if [ ! -f $VIRTLOGD_PIDFILE ];then + if [[ ! -f $VIRTLOGD_PIDFILE ]]; then echo "virtlogd is not running..." exit 2 fi @@ -227,7 +243,7 @@ stop_virtlogd() { } start_virtlockd() { - if [ -f $VIRTLOCKD_PIDFILE ];then + if [[ -f $VIRTLOCKD_PIDFILE ]]; then echo "virtlockd is already running..." exit 1 fi @@ -237,7 +253,7 @@ start_virtlockd() { } stop_virtlockd() { - if [ ! -f $VIRTLOCKD_PIDFILE ];then + if [[ ! -f $VIRTLOCKD_PIDFILE ]]; then echo "virtlockd is not running..." exit 2 fi