mirror of
https://github.com/vdsm/virtual-dsm.git
synced 2025-06-07 17:07:58 +08:00
New networking stack
This commit is contained in:
Kroese
parent
c591f4ecfe
commit
9647d17a4e
16
Dockerfile
16
Dockerfile
@ -15,24 +15,25 @@ RUN apt-get update && apt-get -y upgrade && \
|
||||
wget \
|
||||
unzip \
|
||||
procps \
|
||||
udhcpd \
|
||||
python3 \
|
||||
ethtool \
|
||||
dnsmasq \
|
||||
iptables \
|
||||
iproute2 \
|
||||
xz-utils \
|
||||
qemu-utils \
|
||||
btrfs-progs \
|
||||
bridge-utils \
|
||||
netcat-openbsd \
|
||||
ca-certificates \
|
||||
qemu-system-x86 \
|
||||
&& apt-get clean
|
||||
|
||||
COPY run.sh /run/
|
||||
COPY power.sh /run/
|
||||
COPY server.sh /run/
|
||||
COPY install.sh /run/
|
||||
COPY network.sh /run/
|
||||
COPY qemu-ifup /run/
|
||||
COPY qemu-ifdown /run/
|
||||
COPY generate-dhcpd-conf /run/
|
||||
COPY disks/disk.sh /run/
|
||||
COPY serial/serial.sh /run/
|
||||
COPY agent/agent.sh /agent/
|
||||
COPY agent/service.sh /agent/
|
||||
@ -40,14 +41,13 @@ COPY agent/service.sh /agent/
|
||||
COPY --from=builder /src/serial/main /run/serial.bin
|
||||
|
||||
RUN ["chmod", "+x", "/run/run.sh"]
|
||||
RUN ["chmod", "+x", "/run/disk.sh"]
|
||||
RUN ["chmod", "+x", "/run/power.sh"]
|
||||
RUN ["chmod", "+x", "/run/serial.sh"]
|
||||
RUN ["chmod", "+x", "/run/server.sh"]
|
||||
RUN ["chmod", "+x", "/run/install.sh"]
|
||||
RUN ["chmod", "+x", "/run/network.sh"]
|
||||
RUN ["chmod", "+x", "/run/serial.bin"]
|
||||
RUN ["chmod", "+x", "/run/qemu-ifup"]
|
||||
RUN ["chmod", "+x", "/run/qemu-ifdown"]
|
||||
RUN ["chmod", "+x", "/run/generate-dhcpd-conf"]
|
||||
|
||||
COPY disks/template.img.xz /data/
|
||||
|
||||
|
||||
@ -1,80 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import ipaddress
|
||||
import json
|
||||
import re
|
||||
import socket
|
||||
import subprocess
|
||||
|
||||
from typing import List, Iterable
|
||||
|
||||
DEFAULT_ROUTE = 'default'
|
||||
DEFAULT_DNS_IPS = ('8.8.8.8', '8.8.4.4')
|
||||
|
||||
DHCP_CONF_TEMPLATE = """
|
||||
start {host_addr}
|
||||
end {host_addr}
|
||||
|
||||
# avoid dhcpd complaining that we have
|
||||
# too many addresses
|
||||
max_leases 1
|
||||
|
||||
interface {dhcp_intf}
|
||||
|
||||
option dns {dns}
|
||||
option router {gateway}
|
||||
option subnet {subnet}
|
||||
option hostname {hostname}
|
||||
"""
|
||||
|
||||
def default_route(routes):
|
||||
"""Returns the host's default route"""
|
||||
for route in routes:
|
||||
if route['dst'] == DEFAULT_ROUTE:
|
||||
return route
|
||||
raise ValueError('no default route')
|
||||
|
||||
def addr_of(addrs, dev : str) -> ipaddress.IPv4Interface:
|
||||
"""Finds and returns the IP address of `dev`"""
|
||||
for addr in addrs:
|
||||
if addr['ifname'] != dev:
|
||||
continue
|
||||
#if len(addr['addr_info']) != 1:
|
||||
# raise ValueError('only exactly one address on dev is supported')
|
||||
info = addr['addr_info'][0]
|
||||
return ipaddress.IPv4Interface((info['local'], info['prefixlen']))
|
||||
raise ValueError('dev {0} not found'.format(dev))
|
||||
|
||||
def generate_conf(intf_name : str, dns : Iterable[str]) -> str:
|
||||
"""Generates a dhcpd config. `intf_name` is the interface to listen on."""
|
||||
with subprocess.Popen(['ip', '-json', 'route'],
|
||||
stdout=subprocess.PIPE) as proc:
|
||||
routes = json.load(proc.stdout)
|
||||
with subprocess.Popen(['ip', '-json', 'addr'],
|
||||
stdout=subprocess.PIPE) as proc:
|
||||
addrs = json.load(proc.stdout)
|
||||
|
||||
droute = default_route(routes)
|
||||
host_addr = addr_of(addrs, droute['dev'])
|
||||
|
||||
return DHCP_CONF_TEMPLATE.format(
|
||||
dhcp_intf = intf_name,
|
||||
dns = ' '.join(dns),
|
||||
gateway = droute['gateway'],
|
||||
host_addr = host_addr.ip,
|
||||
hostname = socket.gethostname(),
|
||||
subnet = host_addr.network.netmask,
|
||||
)
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('intf_name')
|
||||
parser.add_argument('dns_ips', nargs='*')
|
||||
args = parser.parse_args()
|
||||
|
||||
dns_ips = args.dns_ips
|
||||
if not dns_ips:
|
||||
dns_ips = DEFAULT_DNS_IPS
|
||||
|
||||
print(generate_conf(args.intf_name, dns_ips))
|
||||
206
network.sh
206
network.sh
@ -1,49 +1,187 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
[ ! -e /dev/net/tun ] && echo "Error: TUN interface not available..." && exit 85
|
||||
[ ! -e /dev/net/tun ] && echo "Error: TUN network interface not available..." && exit 85
|
||||
|
||||
# A bridge of this name will be created to host the TAP interface created for
|
||||
# the VM
|
||||
QEMU_BRIDGE='qemubr0'
|
||||
: ${INFO:='N'}
|
||||
: ${DEBUG:='N'}
|
||||
|
||||
# DHCPD must have an IP address to run, but that address doesn't have to
|
||||
# be valid. This is the dummy address dhcpd is configured to use.
|
||||
DUMMY_DHCPD_IP='10.0.0.1'
|
||||
: ${DNSMASQ:='/usr/sbin/dnsmasq'}
|
||||
: ${DNSMASQ_OPTS:=''}
|
||||
: ${DNSMASQ_CONF_DIR:='/etc/dnsmasq.d'}
|
||||
: ${DNS_SERVERS:=''}
|
||||
|
||||
# The name of the dhcpd config file we make
|
||||
DHCPD_CONF_FILE='dhcpd.conf'
|
||||
# # (VM_NET_IP: Dont need to change coz all is port forwarded)
|
||||
# # (VM_NET_DHCP: It use MACVTAP which is not compatible with all configuration)
|
||||
|
||||
function default_intf() {
|
||||
ip -json route show |
|
||||
jq -r '.[] | select(.dst == "default") | .dev'
|
||||
: ${VM_NET_TAP:=''}
|
||||
: ${VM_NET_IP:='20.20.20.21'}
|
||||
: ${VM_NET_MAC:='00:11:32:2C:A7:85'}
|
||||
: ${VM_NET_DHCP:='N'}
|
||||
: ${VM_ENABLE_VIRTIO:='Y'}
|
||||
|
||||
# ######################################
|
||||
# Functions
|
||||
# ######################################
|
||||
|
||||
log () {
|
||||
case "$1" in
|
||||
WARNING | ERROR )
|
||||
echo "$1: ${@:2}"
|
||||
;;
|
||||
INFO)
|
||||
if [[ "$INFO" == [Yy1]* ]]; then
|
||||
echo "$1: ${@:2}"
|
||||
fi
|
||||
;;
|
||||
DEBUG)
|
||||
if [[ "$DEBUG" == [Yy1]* ]]; then
|
||||
echo "$1: ${@:2}"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "-- $@"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# First step, we run the things that need to happen before we start mucking
|
||||
# with the interfaces. We start by generating the DHCPD config file based
|
||||
# on our current address/routes. We "steal" the container's IP, and lease
|
||||
# it to the VM once it starts up.
|
||||
/run/generate-dhcpd-conf $QEMU_BRIDGE > $DHCPD_CONF_FILE
|
||||
default_dev=$(default_intf)
|
||||
setupLocalDhcp () {
|
||||
CIDR="24"
|
||||
MAC="$1"
|
||||
IP="$2"
|
||||
#HOSTNAME=$(hostname -s)
|
||||
HOSTNAME="VirtualDSM"
|
||||
# dnsmasq configuration:
|
||||
log "INFO" "DHCP configured to serve IP $IP/$CIDR via dockerbridge"
|
||||
DNSMASQ_OPTS="$DNSMASQ_OPTS --dhcp-range=$IP,$IP --dhcp-host=$MAC,,$IP,$HOSTNAME,infinite --dhcp-option=option:netmask,255.255.255.0"
|
||||
# Create lease File FOr faster resolve
|
||||
echo "0 $MAC $IP $HOSTNAME 01:${MAC}" > /var/lib/misc/dnsmasq.leases
|
||||
chmod 644 /var/lib/misc/dnsmasq.leases
|
||||
}
|
||||
|
||||
# Now we start modifying the networking configuration. First we clear out
|
||||
# the IP address of the default device (will also have the side-effect of
|
||||
# removing the default route)
|
||||
ip addr flush dev "$default_dev"
|
||||
# Setup macvtap device to connect later the VM and setup a new macvlan devide
|
||||
# to connect the host machine to the network
|
||||
configureNatNetworks () {
|
||||
|
||||
# Next, we create our bridge, and add our container interface to it.
|
||||
ip link add "$QEMU_BRIDGE" type bridge
|
||||
ip link set dev "$default_dev" master "$QEMU_BRIDGE"
|
||||
#For now we define static MAC because DHCP is very slow if MAC change every VM Boot
|
||||
#Create bridge with static IP for the VM Guest(COnnection VM-Docker)
|
||||
brctl addbr dockerbridge
|
||||
ip addr add ${VM_NET_IP%.*}.1/24 broadcast ${VM_NET_IP%.*}.255 dev dockerbridge
|
||||
ip link set dockerbridge up
|
||||
#QEMU Works with taps, set tap to the bridge created
|
||||
ip tuntap add dev ${VM_NET_TAP} mode tap
|
||||
ip link set ${VM_NET_TAP} up promisc on
|
||||
brctl addif dockerbridge ${VM_NET_TAP}
|
||||
|
||||
# Then, we toggle the interface and the bridge to make sure everything is up
|
||||
# and running.
|
||||
ip link set dev "$default_dev" up
|
||||
ip link set dev "$QEMU_BRIDGE" up
|
||||
#Add internet connection to the VM
|
||||
iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
|
||||
iptables -t nat -A PREROUTING -i eth0 -p tcp -j DNAT --to $VM_NET_IP
|
||||
iptables -t nat -A PREROUTING -i eth0 -p udp -j DNAT --to $VM_NET_IP
|
||||
|
||||
# Prevent error about missing file
|
||||
touch /var/lib/misc/udhcpd.leases
|
||||
#Enable port forwarding flag
|
||||
[[ $(< /proc/sys/net/ipv4/ip_forward) -eq 0 ]] && sysctl -w net.ipv4.ip_forward=1
|
||||
|
||||
# Finally, start our DHCPD server
|
||||
udhcpd -I $DUMMY_DHCPD_IP -f $DHCPD_CONF_FILE 2>&1 &
|
||||
#For now we define static MAC because DHCP is very slow if DHCP change every VM Boot
|
||||
setupLocalDhcp $VM_NET_MAC $VM_NET_IP
|
||||
}
|
||||
|
||||
exit 0
|
||||
# ######################################
|
||||
# Configure Network
|
||||
# ######################################
|
||||
|
||||
MAJOR=""
|
||||
_DhcpIP=""
|
||||
|
||||
#log "INFO" "Little dirty trick ..."
|
||||
update-alternatives --set iptables /usr/sbin/iptables-legacy > /dev/null
|
||||
update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy > /dev/null
|
||||
|
||||
log "INFO" "Configuring network ..."
|
||||
#DEFAULT_ROUTE=$(ip route | grep default | awk '{print $3}')
|
||||
|
||||
if [[ "x${VM_NET_TAP}" == "x" ]]; then
|
||||
if [[ "${VM_NET_DHCP}" == [Yy1]* ]]; then
|
||||
VM_NET_TAP="_VmMacvtap"
|
||||
log "INFO" "... to retrieve IP via DHCP through Macvtap (${VM_NET_TAP}) and MAC: ${VM_NET_MAC}"
|
||||
|
||||
ip l add link eth0 name ${VM_NET_TAP} address ${VM_NET_MAC} type macvtap mode bridge || true
|
||||
ip l set ${VM_NET_TAP} up
|
||||
|
||||
ip a flush eth0
|
||||
ip a flush ${VM_NET_TAP}
|
||||
|
||||
_DhcpIP=$( dhclient -v ${VM_NET_TAP} 2>&1 | grep ^bound | cut -d' ' -f3 )
|
||||
[[ "${_DhcpIP}" == [0-9.]* ]] \
|
||||
&& log "INFO" "... Retrieve IP: ${_DhcpIP} from DHCP with MAC: ${VM_NET_MAC}" \
|
||||
|| ( log "ERROR" "... Cannot retrieve IP from DHCP with MAC: ${VM_NET_MAC}" && exit 16 )
|
||||
|
||||
ip a flush ${VM_NET_TAP}
|
||||
|
||||
_tmpTapPath="/dev/tap$(</sys/class/net/${VM_NET_TAP}/ifindex)"
|
||||
# get MAJOR MINOR DEVNAME
|
||||
MAJOR=""
|
||||
eval "$(</sys/class/net/${VM_NET_TAP}/macvtap/${_tmpTapPath##*/}/uevent) _tmp=0"
|
||||
|
||||
[[ "x${MAJOR}" != "x" ]] \
|
||||
&& log "INFO" "... PLEASE MAKE SURE, Docker run command line used: --device-cgroup-rule='c ${MAJOR}:* rwm'" \
|
||||
|| ( log "ERROR" "... macvtap creation issue: Cannot find: /sys/class/net/${VM_NET_TAP}/" && exit 18 )
|
||||
|
||||
[[ ! -e ${_tmpTapPath} ]] && [[ -e /dev0/${_tmpTapPath##*/} ]] && ln -s /dev0/${_tmpTapPath##*/} ${_tmpTapPath}
|
||||
|
||||
if [[ ! -e ${_tmpTapPath} ]]; then
|
||||
log "WARNING" "... file does not exist: ${_tmpTapPath}"
|
||||
mknod ${_tmpTapPath} c $MAJOR $MINOR \
|
||||
&& log "INFO" "... File created with mknod: ${_tmpTapPath}" \
|
||||
|| ( log "ERROR" "... Cannot mknod: ${_tmpTapPath}" && exit 20 )
|
||||
fi
|
||||
KVM_NET_OPTS="-netdev tap,id=hostnet0,vhost=on,vhostfd=40,fd=30 30<>${_tmpTapPath} 40<>/dev/vhost-net"
|
||||
else
|
||||
VM_NET_TAP="_VmNatTap"
|
||||
log "INFO" "... NAT Network (${VM_NET_TAP}) to ${VM_NET_IP}"
|
||||
|
||||
configureNatNetworks
|
||||
KVM_NET_OPTS="-netdev tap,ifname=${VM_NET_TAP},script=no,downscript=no,id=hostnet0"
|
||||
|
||||
# Build DNS options from container /etc/resolv.conf
|
||||
nameservers=($(grep '^nameserver' /etc/resolv.conf | sed 's/nameserver //'))
|
||||
searchdomains=$(grep '^search' /etc/resolv.conf | sed 's/search //' | sed 's/ /,/g')
|
||||
domainname=$(echo $searchdomains | awk -F"," '{print $1}')
|
||||
|
||||
for nameserver in "${nameservers[@]}"; do
|
||||
if [[ $nameserver =~ .*:.* ]]; then
|
||||
log "INFO" "Skipping IPv6 nameserver: $nameserver"
|
||||
else
|
||||
[[ -z $DNS_SERVERS ]] && DNS_SERVERS=$nameserver || DNS_SERVERS="$DNS_SERVERS,$nameserver"
|
||||
fi
|
||||
done
|
||||
DNSMASQ_OPTS="$DNSMASQ_OPTS \
|
||||
--dhcp-option=option:dns-server,$DNS_SERVERS \
|
||||
--dhcp-option=option:router,${VM_NET_IP%.*}.1 \
|
||||
--dhcp-option=option:domain-search,$searchdomains \
|
||||
--dhcp-option=option:domain-name,$domainname \
|
||||
"
|
||||
[[ -z $(hostname -d) ]] || DNSMASQ_OPTS="$DNSMASQ_OPTS --dhcp-option=option:domain-name,$(hostname -d)"
|
||||
|
||||
log "INFO" "... Lauching dnsmasq"
|
||||
log "DEBUG" "dnsmasq options: $DNSMASQ_OPTS"
|
||||
$DNSMASQ $DNSMASQ_OPTS
|
||||
fi
|
||||
else
|
||||
log "INFO" "... No configuration, just using tuntap : ${VM_NET_TAP}"
|
||||
KVM_NET_OPTS="-netdev tap,ifname=${VM_NET_TAP},script=no,downscript=no,id=hostnet0"
|
||||
fi
|
||||
|
||||
#KVM_NET_OPTS="-netdev user,hostfwd=tcp:127.0.0.1:5000-:5000"
|
||||
[[ "${VM_ENABLE_VIRTIO}" == [Yy1]* ]] \
|
||||
&& KVM_NET_OPTS="${KVM_NET_OPTS} -device virtio-net-pci,netdev=hostnet0,mac=${VM_NET_MAC},id=net0" \
|
||||
|| KVM_NET_OPTS="${KVM_NET_OPTS} -device e1000e,netdev=hostnet0,mac=${VM_NET_MAC},id=net0"
|
||||
|
||||
# Hack for guest VMs complaining about "bad udp checksums in 5 packets"
|
||||
log "INFO" "Hack for guest VMs complaining about: bad udp checksums in 5 packets"
|
||||
iptables -A POSTROUTING -t mangle -p udp --dport bootpc -j CHECKSUM --checksum-fill \
|
||||
|| ( log "WARNING" "Iptables hack for checksum FAILED" && ethtool -K eth0 tx off || true )
|
||||
|
||||
[[ "x${MAJOR}" != "x" ]] && log "INFO" "PLEASE MAKE SURE, Docker is using the following option otherwise you may have permission issue on ${_tmpTapPath} file: --device-cgroup-rule='c ${MAJOR}:* rwm' "
|
||||
[[ "${_DhcpIP}" == [0-9.]* ]] && log "INFO" "You should access your DSM with: http://${_DhcpIP}:5000"
|
||||
|
||||
log "INFO" "Done setting up network.."
|
||||
|
||||
@ -1,6 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
#QEMU_BRIDGE="qemubr0"
|
||||
|
||||
ip link set dev "$1" nomaster
|
||||
ip link set dev "$1" down
|
||||
Loading…
x
Reference in New Issue
Block a user