mirror of
https://github.com/vdsm/virtual-dsm.git
synced 2025-06-08 01:18:32 +08:00
New networking stack
This commit is contained in:
Kroese
parent
c591f4ecfe
commit
9647d17a4e
16
Dockerfile
16
Dockerfile
@ -15,24 +15,25 @@ RUN apt-get update && apt-get -y upgrade && \
|
|||||||
wget \
|
wget \
|
||||||
unzip \
|
unzip \
|
||||||
procps \
|
procps \
|
||||||
udhcpd \
|
ethtool \
|
||||||
python3 \
|
dnsmasq \
|
||||||
|
iptables \
|
||||||
iproute2 \
|
iproute2 \
|
||||||
xz-utils \
|
xz-utils \
|
||||||
qemu-utils \
|
qemu-utils \
|
||||||
btrfs-progs \
|
btrfs-progs \
|
||||||
|
bridge-utils \
|
||||||
netcat-openbsd \
|
netcat-openbsd \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
qemu-system-x86 \
|
qemu-system-x86 \
|
||||||
&& apt-get clean
|
&& apt-get clean
|
||||||
|
|
||||||
COPY run.sh /run/
|
COPY run.sh /run/
|
||||||
|
COPY power.sh /run/
|
||||||
COPY server.sh /run/
|
COPY server.sh /run/
|
||||||
COPY install.sh /run/
|
COPY install.sh /run/
|
||||||
COPY network.sh /run/
|
COPY network.sh /run/
|
||||||
COPY qemu-ifup /run/
|
COPY disks/disk.sh /run/
|
||||||
COPY qemu-ifdown /run/
|
|
||||||
COPY generate-dhcpd-conf /run/
|
|
||||||
COPY serial/serial.sh /run/
|
COPY serial/serial.sh /run/
|
||||||
COPY agent/agent.sh /agent/
|
COPY agent/agent.sh /agent/
|
||||||
COPY agent/service.sh /agent/
|
COPY agent/service.sh /agent/
|
||||||
@ -40,14 +41,13 @@ COPY agent/service.sh /agent/
|
|||||||
COPY --from=builder /src/serial/main /run/serial.bin
|
COPY --from=builder /src/serial/main /run/serial.bin
|
||||||
|
|
||||||
RUN ["chmod", "+x", "/run/run.sh"]
|
RUN ["chmod", "+x", "/run/run.sh"]
|
||||||
|
RUN ["chmod", "+x", "/run/disk.sh"]
|
||||||
|
RUN ["chmod", "+x", "/run/power.sh"]
|
||||||
RUN ["chmod", "+x", "/run/serial.sh"]
|
RUN ["chmod", "+x", "/run/serial.sh"]
|
||||||
RUN ["chmod", "+x", "/run/server.sh"]
|
RUN ["chmod", "+x", "/run/server.sh"]
|
||||||
RUN ["chmod", "+x", "/run/install.sh"]
|
RUN ["chmod", "+x", "/run/install.sh"]
|
||||||
RUN ["chmod", "+x", "/run/network.sh"]
|
RUN ["chmod", "+x", "/run/network.sh"]
|
||||||
RUN ["chmod", "+x", "/run/serial.bin"]
|
RUN ["chmod", "+x", "/run/serial.bin"]
|
||||||
RUN ["chmod", "+x", "/run/qemu-ifup"]
|
|
||||||
RUN ["chmod", "+x", "/run/qemu-ifdown"]
|
|
||||||
RUN ["chmod", "+x", "/run/generate-dhcpd-conf"]
|
|
||||||
|
|
||||||
COPY disks/template.img.xz /data/
|
COPY disks/template.img.xz /data/
|
||||||
|
|
||||||
|
|||||||
@ -1,80 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import ipaddress
|
|
||||||
import json
|
|
||||||
import re
|
|
||||||
import socket
|
|
||||||
import subprocess
|
|
||||||
|
|
||||||
from typing import List, Iterable
|
|
||||||
|
|
||||||
DEFAULT_ROUTE = 'default'
|
|
||||||
DEFAULT_DNS_IPS = ('8.8.8.8', '8.8.4.4')
|
|
||||||
|
|
||||||
DHCP_CONF_TEMPLATE = """
|
|
||||||
start {host_addr}
|
|
||||||
end {host_addr}
|
|
||||||
|
|
||||||
# avoid dhcpd complaining that we have
|
|
||||||
# too many addresses
|
|
||||||
max_leases 1
|
|
||||||
|
|
||||||
interface {dhcp_intf}
|
|
||||||
|
|
||||||
option dns {dns}
|
|
||||||
option router {gateway}
|
|
||||||
option subnet {subnet}
|
|
||||||
option hostname {hostname}
|
|
||||||
"""
|
|
||||||
|
|
||||||
def default_route(routes):
|
|
||||||
"""Returns the host's default route"""
|
|
||||||
for route in routes:
|
|
||||||
if route['dst'] == DEFAULT_ROUTE:
|
|
||||||
return route
|
|
||||||
raise ValueError('no default route')
|
|
||||||
|
|
||||||
def addr_of(addrs, dev : str) -> ipaddress.IPv4Interface:
|
|
||||||
"""Finds and returns the IP address of `dev`"""
|
|
||||||
for addr in addrs:
|
|
||||||
if addr['ifname'] != dev:
|
|
||||||
continue
|
|
||||||
#if len(addr['addr_info']) != 1:
|
|
||||||
# raise ValueError('only exactly one address on dev is supported')
|
|
||||||
info = addr['addr_info'][0]
|
|
||||||
return ipaddress.IPv4Interface((info['local'], info['prefixlen']))
|
|
||||||
raise ValueError('dev {0} not found'.format(dev))
|
|
||||||
|
|
||||||
def generate_conf(intf_name : str, dns : Iterable[str]) -> str:
|
|
||||||
"""Generates a dhcpd config. `intf_name` is the interface to listen on."""
|
|
||||||
with subprocess.Popen(['ip', '-json', 'route'],
|
|
||||||
stdout=subprocess.PIPE) as proc:
|
|
||||||
routes = json.load(proc.stdout)
|
|
||||||
with subprocess.Popen(['ip', '-json', 'addr'],
|
|
||||||
stdout=subprocess.PIPE) as proc:
|
|
||||||
addrs = json.load(proc.stdout)
|
|
||||||
|
|
||||||
droute = default_route(routes)
|
|
||||||
host_addr = addr_of(addrs, droute['dev'])
|
|
||||||
|
|
||||||
return DHCP_CONF_TEMPLATE.format(
|
|
||||||
dhcp_intf = intf_name,
|
|
||||||
dns = ' '.join(dns),
|
|
||||||
gateway = droute['gateway'],
|
|
||||||
host_addr = host_addr.ip,
|
|
||||||
hostname = socket.gethostname(),
|
|
||||||
subnet = host_addr.network.netmask,
|
|
||||||
)
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument('intf_name')
|
|
||||||
parser.add_argument('dns_ips', nargs='*')
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
dns_ips = args.dns_ips
|
|
||||||
if not dns_ips:
|
|
||||||
dns_ips = DEFAULT_DNS_IPS
|
|
||||||
|
|
||||||
print(generate_conf(args.intf_name, dns_ips))
|
|
||||||
206
network.sh
206
network.sh
@ -1,49 +1,187 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
[ ! -e /dev/net/tun ] && echo "Error: TUN interface not available..." && exit 85
|
[ ! -e /dev/net/tun ] && echo "Error: TUN network interface not available..." && exit 85
|
||||||
|
|
||||||
# A bridge of this name will be created to host the TAP interface created for
|
: ${INFO:='N'}
|
||||||
# the VM
|
: ${DEBUG:='N'}
|
||||||
QEMU_BRIDGE='qemubr0'
|
|
||||||
|
|
||||||
# DHCPD must have an IP address to run, but that address doesn't have to
|
: ${DNSMASQ:='/usr/sbin/dnsmasq'}
|
||||||
# be valid. This is the dummy address dhcpd is configured to use.
|
: ${DNSMASQ_OPTS:=''}
|
||||||
DUMMY_DHCPD_IP='10.0.0.1'
|
: ${DNSMASQ_CONF_DIR:='/etc/dnsmasq.d'}
|
||||||
|
: ${DNS_SERVERS:=''}
|
||||||
|
|
||||||
# The name of the dhcpd config file we make
|
# # (VM_NET_IP: Dont need to change coz all is port forwarded)
|
||||||
DHCPD_CONF_FILE='dhcpd.conf'
|
# # (VM_NET_DHCP: It use MACVTAP which is not compatible with all configuration)
|
||||||
|
|
||||||
function default_intf() {
|
: ${VM_NET_TAP:=''}
|
||||||
ip -json route show |
|
: ${VM_NET_IP:='20.20.20.21'}
|
||||||
jq -r '.[] | select(.dst == "default") | .dev'
|
: ${VM_NET_MAC:='00:11:32:2C:A7:85'}
|
||||||
|
: ${VM_NET_DHCP:='N'}
|
||||||
|
: ${VM_ENABLE_VIRTIO:='Y'}
|
||||||
|
|
||||||
|
# ######################################
|
||||||
|
# Functions
|
||||||
|
# ######################################
|
||||||
|
|
||||||
|
log () {
|
||||||
|
case "$1" in
|
||||||
|
WARNING | ERROR )
|
||||||
|
echo "$1: ${@:2}"
|
||||||
|
;;
|
||||||
|
INFO)
|
||||||
|
if [[ "$INFO" == [Yy1]* ]]; then
|
||||||
|
echo "$1: ${@:2}"
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
DEBUG)
|
||||||
|
if [[ "$DEBUG" == [Yy1]* ]]; then
|
||||||
|
echo "$1: ${@:2}"
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "-- $@"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
}
|
}
|
||||||
|
|
||||||
# First step, we run the things that need to happen before we start mucking
|
setupLocalDhcp () {
|
||||||
# with the interfaces. We start by generating the DHCPD config file based
|
CIDR="24"
|
||||||
# on our current address/routes. We "steal" the container's IP, and lease
|
MAC="$1"
|
||||||
# it to the VM once it starts up.
|
IP="$2"
|
||||||
/run/generate-dhcpd-conf $QEMU_BRIDGE > $DHCPD_CONF_FILE
|
#HOSTNAME=$(hostname -s)
|
||||||
default_dev=$(default_intf)
|
HOSTNAME="VirtualDSM"
|
||||||
|
# dnsmasq configuration:
|
||||||
|
log "INFO" "DHCP configured to serve IP $IP/$CIDR via dockerbridge"
|
||||||
|
DNSMASQ_OPTS="$DNSMASQ_OPTS --dhcp-range=$IP,$IP --dhcp-host=$MAC,,$IP,$HOSTNAME,infinite --dhcp-option=option:netmask,255.255.255.0"
|
||||||
|
# Create lease File FOr faster resolve
|
||||||
|
echo "0 $MAC $IP $HOSTNAME 01:${MAC}" > /var/lib/misc/dnsmasq.leases
|
||||||
|
chmod 644 /var/lib/misc/dnsmasq.leases
|
||||||
|
}
|
||||||
|
|
||||||
# Now we start modifying the networking configuration. First we clear out
|
# Setup macvtap device to connect later the VM and setup a new macvlan devide
|
||||||
# the IP address of the default device (will also have the side-effect of
|
# to connect the host machine to the network
|
||||||
# removing the default route)
|
configureNatNetworks () {
|
||||||
ip addr flush dev "$default_dev"
|
|
||||||
|
|
||||||
# Next, we create our bridge, and add our container interface to it.
|
#For now we define static MAC because DHCP is very slow if MAC change every VM Boot
|
||||||
ip link add "$QEMU_BRIDGE" type bridge
|
#Create bridge with static IP for the VM Guest(COnnection VM-Docker)
|
||||||
ip link set dev "$default_dev" master "$QEMU_BRIDGE"
|
brctl addbr dockerbridge
|
||||||
|
ip addr add ${VM_NET_IP%.*}.1/24 broadcast ${VM_NET_IP%.*}.255 dev dockerbridge
|
||||||
|
ip link set dockerbridge up
|
||||||
|
#QEMU Works with taps, set tap to the bridge created
|
||||||
|
ip tuntap add dev ${VM_NET_TAP} mode tap
|
||||||
|
ip link set ${VM_NET_TAP} up promisc on
|
||||||
|
brctl addif dockerbridge ${VM_NET_TAP}
|
||||||
|
|
||||||
# Then, we toggle the interface and the bridge to make sure everything is up
|
#Add internet connection to the VM
|
||||||
# and running.
|
iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
|
||||||
ip link set dev "$default_dev" up
|
iptables -t nat -A PREROUTING -i eth0 -p tcp -j DNAT --to $VM_NET_IP
|
||||||
ip link set dev "$QEMU_BRIDGE" up
|
iptables -t nat -A PREROUTING -i eth0 -p udp -j DNAT --to $VM_NET_IP
|
||||||
|
|
||||||
# Prevent error about missing file
|
#Enable port forwarding flag
|
||||||
touch /var/lib/misc/udhcpd.leases
|
[[ $(< /proc/sys/net/ipv4/ip_forward) -eq 0 ]] && sysctl -w net.ipv4.ip_forward=1
|
||||||
|
|
||||||
# Finally, start our DHCPD server
|
#For now we define static MAC because DHCP is very slow if DHCP change every VM Boot
|
||||||
udhcpd -I $DUMMY_DHCPD_IP -f $DHCPD_CONF_FILE 2>&1 &
|
setupLocalDhcp $VM_NET_MAC $VM_NET_IP
|
||||||
|
}
|
||||||
|
|
||||||
exit 0
|
# ######################################
|
||||||
|
# Configure Network
|
||||||
|
# ######################################
|
||||||
|
|
||||||
|
MAJOR=""
|
||||||
|
_DhcpIP=""
|
||||||
|
|
||||||
|
#log "INFO" "Little dirty trick ..."
|
||||||
|
update-alternatives --set iptables /usr/sbin/iptables-legacy > /dev/null
|
||||||
|
update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy > /dev/null
|
||||||
|
|
||||||
|
log "INFO" "Configuring network ..."
|
||||||
|
#DEFAULT_ROUTE=$(ip route | grep default | awk '{print $3}')
|
||||||
|
|
||||||
|
if [[ "x${VM_NET_TAP}" == "x" ]]; then
|
||||||
|
if [[ "${VM_NET_DHCP}" == [Yy1]* ]]; then
|
||||||
|
VM_NET_TAP="_VmMacvtap"
|
||||||
|
log "INFO" "... to retrieve IP via DHCP through Macvtap (${VM_NET_TAP}) and MAC: ${VM_NET_MAC}"
|
||||||
|
|
||||||
|
ip l add link eth0 name ${VM_NET_TAP} address ${VM_NET_MAC} type macvtap mode bridge || true
|
||||||
|
ip l set ${VM_NET_TAP} up
|
||||||
|
|
||||||
|
ip a flush eth0
|
||||||
|
ip a flush ${VM_NET_TAP}
|
||||||
|
|
||||||
|
_DhcpIP=$( dhclient -v ${VM_NET_TAP} 2>&1 | grep ^bound | cut -d' ' -f3 )
|
||||||
|
[[ "${_DhcpIP}" == [0-9.]* ]] \
|
||||||
|
&& log "INFO" "... Retrieve IP: ${_DhcpIP} from DHCP with MAC: ${VM_NET_MAC}" \
|
||||||
|
|| ( log "ERROR" "... Cannot retrieve IP from DHCP with MAC: ${VM_NET_MAC}" && exit 16 )
|
||||||
|
|
||||||
|
ip a flush ${VM_NET_TAP}
|
||||||
|
|
||||||
|
_tmpTapPath="/dev/tap$(</sys/class/net/${VM_NET_TAP}/ifindex)"
|
||||||
|
# get MAJOR MINOR DEVNAME
|
||||||
|
MAJOR=""
|
||||||
|
eval "$(</sys/class/net/${VM_NET_TAP}/macvtap/${_tmpTapPath##*/}/uevent) _tmp=0"
|
||||||
|
|
||||||
|
[[ "x${MAJOR}" != "x" ]] \
|
||||||
|
&& log "INFO" "... PLEASE MAKE SURE, Docker run command line used: --device-cgroup-rule='c ${MAJOR}:* rwm'" \
|
||||||
|
|| ( log "ERROR" "... macvtap creation issue: Cannot find: /sys/class/net/${VM_NET_TAP}/" && exit 18 )
|
||||||
|
|
||||||
|
[[ ! -e ${_tmpTapPath} ]] && [[ -e /dev0/${_tmpTapPath##*/} ]] && ln -s /dev0/${_tmpTapPath##*/} ${_tmpTapPath}
|
||||||
|
|
||||||
|
if [[ ! -e ${_tmpTapPath} ]]; then
|
||||||
|
log "WARNING" "... file does not exist: ${_tmpTapPath}"
|
||||||
|
mknod ${_tmpTapPath} c $MAJOR $MINOR \
|
||||||
|
&& log "INFO" "... File created with mknod: ${_tmpTapPath}" \
|
||||||
|
|| ( log "ERROR" "... Cannot mknod: ${_tmpTapPath}" && exit 20 )
|
||||||
|
fi
|
||||||
|
KVM_NET_OPTS="-netdev tap,id=hostnet0,vhost=on,vhostfd=40,fd=30 30<>${_tmpTapPath} 40<>/dev/vhost-net"
|
||||||
|
else
|
||||||
|
VM_NET_TAP="_VmNatTap"
|
||||||
|
log "INFO" "... NAT Network (${VM_NET_TAP}) to ${VM_NET_IP}"
|
||||||
|
|
||||||
|
configureNatNetworks
|
||||||
|
KVM_NET_OPTS="-netdev tap,ifname=${VM_NET_TAP},script=no,downscript=no,id=hostnet0"
|
||||||
|
|
||||||
|
# Build DNS options from container /etc/resolv.conf
|
||||||
|
nameservers=($(grep '^nameserver' /etc/resolv.conf | sed 's/nameserver //'))
|
||||||
|
searchdomains=$(grep '^search' /etc/resolv.conf | sed 's/search //' | sed 's/ /,/g')
|
||||||
|
domainname=$(echo $searchdomains | awk -F"," '{print $1}')
|
||||||
|
|
||||||
|
for nameserver in "${nameservers[@]}"; do
|
||||||
|
if [[ $nameserver =~ .*:.* ]]; then
|
||||||
|
log "INFO" "Skipping IPv6 nameserver: $nameserver"
|
||||||
|
else
|
||||||
|
[[ -z $DNS_SERVERS ]] && DNS_SERVERS=$nameserver || DNS_SERVERS="$DNS_SERVERS,$nameserver"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
DNSMASQ_OPTS="$DNSMASQ_OPTS \
|
||||||
|
--dhcp-option=option:dns-server,$DNS_SERVERS \
|
||||||
|
--dhcp-option=option:router,${VM_NET_IP%.*}.1 \
|
||||||
|
--dhcp-option=option:domain-search,$searchdomains \
|
||||||
|
--dhcp-option=option:domain-name,$domainname \
|
||||||
|
"
|
||||||
|
[[ -z $(hostname -d) ]] || DNSMASQ_OPTS="$DNSMASQ_OPTS --dhcp-option=option:domain-name,$(hostname -d)"
|
||||||
|
|
||||||
|
log "INFO" "... Lauching dnsmasq"
|
||||||
|
log "DEBUG" "dnsmasq options: $DNSMASQ_OPTS"
|
||||||
|
$DNSMASQ $DNSMASQ_OPTS
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
log "INFO" "... No configuration, just using tuntap : ${VM_NET_TAP}"
|
||||||
|
KVM_NET_OPTS="-netdev tap,ifname=${VM_NET_TAP},script=no,downscript=no,id=hostnet0"
|
||||||
|
fi
|
||||||
|
|
||||||
|
#KVM_NET_OPTS="-netdev user,hostfwd=tcp:127.0.0.1:5000-:5000"
|
||||||
|
[[ "${VM_ENABLE_VIRTIO}" == [Yy1]* ]] \
|
||||||
|
&& KVM_NET_OPTS="${KVM_NET_OPTS} -device virtio-net-pci,netdev=hostnet0,mac=${VM_NET_MAC},id=net0" \
|
||||||
|
|| KVM_NET_OPTS="${KVM_NET_OPTS} -device e1000e,netdev=hostnet0,mac=${VM_NET_MAC},id=net0"
|
||||||
|
|
||||||
|
# Hack for guest VMs complaining about "bad udp checksums in 5 packets"
|
||||||
|
log "INFO" "Hack for guest VMs complaining about: bad udp checksums in 5 packets"
|
||||||
|
iptables -A POSTROUTING -t mangle -p udp --dport bootpc -j CHECKSUM --checksum-fill \
|
||||||
|
|| ( log "WARNING" "Iptables hack for checksum FAILED" && ethtool -K eth0 tx off || true )
|
||||||
|
|
||||||
|
[[ "x${MAJOR}" != "x" ]] && log "INFO" "PLEASE MAKE SURE, Docker is using the following option otherwise you may have permission issue on ${_tmpTapPath} file: --device-cgroup-rule='c ${MAJOR}:* rwm' "
|
||||||
|
[[ "${_DhcpIP}" == [0-9.]* ]] && log "INFO" "You should access your DSM with: http://${_DhcpIP}:5000"
|
||||||
|
|
||||||
|
log "INFO" "Done setting up network.."
|
||||||
|
|||||||
@ -1,6 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
#QEMU_BRIDGE="qemubr0"
|
|
||||||
|
|
||||||
ip link set dev "$1" nomaster
|
|
||||||
ip link set dev "$1" down
|
|
||||||
Loading…
x
Reference in New Issue
Block a user