Commit ac391fbc authored by Kroese's avatar Kroese
Browse files

New networking stack

parent c0c77fa2

Dockerfile

+8 −8
Original line number Diff line number Diff line
@@ -15,24 +15,25 @@ RUN apt-get update && apt-get -y upgrade && \ 
	wget \

	unzip \

	procps \

	udhcpd \

	python3 \

	ethtool \

	dnsmasq \

	iptables \

	iproute2 \

	xz-utils \

	qemu-utils \

	btrfs-progs \

	bridge-utils \

	netcat-openbsd \

	ca-certificates \

	qemu-system-x86 \

    && apt-get clean



COPY run.sh /run/

COPY power.sh /run/

COPY server.sh /run/

COPY install.sh /run/

COPY network.sh /run/

COPY qemu-ifup /run/

COPY qemu-ifdown /run/

COPY generate-dhcpd-conf /run/

COPY disks/disk.sh /run/

COPY serial/serial.sh /run/

COPY agent/agent.sh /agent/

COPY agent/service.sh /agent/
@@ -40,14 +41,13 @@ COPY agent/service.sh /agent/ 
COPY --from=builder /src/serial/main /run/serial.bin



RUN ["chmod", "+x", "/run/run.sh"]

RUN ["chmod", "+x", "/run/disk.sh"]

RUN ["chmod", "+x", "/run/power.sh"]

RUN ["chmod", "+x", "/run/serial.sh"]

RUN ["chmod", "+x", "/run/server.sh"]

RUN ["chmod", "+x", "/run/install.sh"]

RUN ["chmod", "+x", "/run/network.sh"]

RUN ["chmod", "+x", "/run/serial.bin"]

RUN ["chmod", "+x", "/run/qemu-ifup"]

RUN ["chmod", "+x", "/run/qemu-ifdown"]

RUN ["chmod", "+x", "/run/generate-dhcpd-conf"]



COPY disks/template.img.xz /data/

generate-dhcpd-conf

deleted100755 → 0
+0 −80
Original line number Diff line number Diff line 
#!/usr/bin/env python3



import argparse

import ipaddress

import json

import re

import socket

import subprocess



from typing import List, Iterable



DEFAULT_ROUTE = 'default'

DEFAULT_DNS_IPS = ('8.8.8.8', '8.8.4.4')



DHCP_CONF_TEMPLATE = """

start {host_addr}

end   {host_addr}



# avoid dhcpd complaining that we have

# too many addresses

max_leases 1



interface {dhcp_intf}



option dns      {dns}

option router   {gateway}

option subnet   {subnet}

option hostname {hostname}

"""



def default_route(routes):

    """Returns the host's default route"""

    for route in routes:

        if route['dst'] == DEFAULT_ROUTE:

            return route

    raise ValueError('no default route')



def addr_of(addrs, dev : str) -> ipaddress.IPv4Interface:

    """Finds and returns the IP address of `dev`"""

    for addr in addrs:

        if addr['ifname'] != dev:

            continue

        #if len(addr['addr_info']) != 1:

        #    raise ValueError('only exactly one address on dev is supported')

        info = addr['addr_info'][0]

        return ipaddress.IPv4Interface((info['local'], info['prefixlen']))

    raise ValueError('dev {0} not found'.format(dev))



def generate_conf(intf_name : str, dns : Iterable[str]) -> str:

    """Generates a dhcpd config. `intf_name` is the interface to listen on."""

    with subprocess.Popen(['ip', '-json', 'route'],

                          stdout=subprocess.PIPE) as proc:

        routes = json.load(proc.stdout)

    with subprocess.Popen(['ip', '-json', 'addr'],

                          stdout=subprocess.PIPE) as proc:

        addrs = json.load(proc.stdout)



    droute = default_route(routes)

    host_addr = addr_of(addrs, droute['dev'])



    return DHCP_CONF_TEMPLATE.format(

        dhcp_intf = intf_name,

        dns = ' '.join(dns),

        gateway = droute['gateway'],

        host_addr = host_addr.ip,

        hostname = socket.gethostname(),

        subnet = host_addr.network.netmask,

    )



if __name__ == '__main__':

    parser = argparse.ArgumentParser()

    parser.add_argument('intf_name')

    parser.add_argument('dns_ips', nargs='*')

    args = parser.parse_args()



    dns_ips = args.dns_ips

    if not dns_ips:

        dns_ips = DEFAULT_DNS_IPS



    print(generate_conf(args.intf_name, dns_ips))

network.sh

+172 −34
Original line number Diff line number Diff line 
#!/usr/bin/env bash

set -eu



[ ! -e /dev/net/tun ] && echo "Error: TUN interface not available..." && exit 85

[ ! -e /dev/net/tun ] && echo "Error: TUN network interface not available..." && exit 85



# A bridge of this name will be created to host the TAP interface created for

# the VM

QEMU_BRIDGE='qemubr0'

: ${INFO:='N'}

: ${DEBUG:='N'}



# DHCPD must have an IP address to run, but that address doesn't have to

# be valid. This is the dummy address dhcpd is configured to use.

DUMMY_DHCPD_IP='10.0.0.1'

: ${DNSMASQ:='/usr/sbin/dnsmasq'}

: ${DNSMASQ_OPTS:=''}

: ${DNSMASQ_CONF_DIR:='/etc/dnsmasq.d'}

: ${DNS_SERVERS:=''}



# The name of the dhcpd config file we make

DHCPD_CONF_FILE='dhcpd.conf'

# # (VM_NET_IP: Dont need to change coz all is port forwarded)

# # (VM_NET_DHCP: It use MACVTAP which is not compatible with all configuration)



function default_intf() {

    ip -json route show |

        jq -r '.[] | select(.dst == "default") | .dev'

: ${VM_NET_TAP:=''}

: ${VM_NET_IP:='20.20.20.21'}

: ${VM_NET_MAC:='00:11:32:2C:A7:85'}

: ${VM_NET_DHCP:='N'}

: ${VM_ENABLE_VIRTIO:='Y'}



# ######################################

#  Functions

# ######################################



log () {

  case "$1" in

    WARNING | ERROR )

      echo "$1: ${@:2}"

      ;;

    INFO)

      if [[ "$INFO" == [Yy1]* ]]; then

          echo "$1: ${@:2}"

      fi

      ;;

    DEBUG)

      if [[ "$DEBUG" == [Yy1]* ]]; then

          echo "$1: ${@:2}"

      fi

      ;;

    *)

      echo "-- $@"

      ;;

  esac

}



setupLocalDhcp () {

  CIDR="24"

  MAC="$1"

  IP="$2"

  #HOSTNAME=$(hostname -s)

  HOSTNAME="VirtualDSM"

  # dnsmasq configuration:

  log "INFO" "DHCP configured to serve IP $IP/$CIDR via dockerbridge"

  DNSMASQ_OPTS="$DNSMASQ_OPTS --dhcp-range=$IP,$IP --dhcp-host=$MAC,,$IP,$HOSTNAME,infinite --dhcp-option=option:netmask,255.255.255.0"

  # Create lease File FOr faster resolve

  echo "0 $MAC $IP $HOSTNAME 01:${MAC}" > /var/lib/misc/dnsmasq.leases

  chmod 644 /var/lib/misc/dnsmasq.leases

}



# Setup macvtap device to connect later the VM and setup a new macvlan devide

# to connect the host machine to the network

configureNatNetworks () {



  #For now we define static MAC because DHCP is very slow if MAC change every VM Boot

  #Create bridge with static IP for the VM Guest(COnnection VM-Docker)

  brctl addbr dockerbridge

  ip addr add ${VM_NET_IP%.*}.1/24 broadcast ${VM_NET_IP%.*}.255 dev dockerbridge

  ip link set dockerbridge up

  #QEMU Works with taps, set tap to the bridge created

  ip tuntap add dev ${VM_NET_TAP} mode tap

  ip link set ${VM_NET_TAP} up promisc on

  brctl addif dockerbridge ${VM_NET_TAP}



  #Add internet connection to the VM

  iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE

  iptables -t nat -A PREROUTING -i eth0 -p tcp  -j DNAT --to $VM_NET_IP

  iptables -t nat -A PREROUTING -i eth0 -p udp  -j DNAT --to $VM_NET_IP



  #Enable port forwarding flag

  [[ $(< /proc/sys/net/ipv4/ip_forward) -eq 0 ]] && sysctl -w net.ipv4.ip_forward=1



  #For now we define static MAC because DHCP is very slow if DHCP change every VM Boot

  setupLocalDhcp $VM_NET_MAC $VM_NET_IP

}



# First step, we run the things that need to happen before we start mucking

# with the interfaces. We start by generating the DHCPD config file based

# on our current address/routes. We "steal" the container's IP, and lease

# it to the VM once it starts up.

/run/generate-dhcpd-conf $QEMU_BRIDGE > $DHCPD_CONF_FILE

default_dev=$(default_intf)

# ######################################

#  Configure Network

# ######################################



MAJOR=""

_DhcpIP=""



#log "INFO" "Little dirty trick ..."

update-alternatives --set iptables /usr/sbin/iptables-legacy > /dev/null

update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy > /dev/null



log "INFO" "Configuring network ..."

#DEFAULT_ROUTE=$(ip route | grep default | awk '{print $3}')



if [[ "x${VM_NET_TAP}" == "x" ]]; then

	if [[ "${VM_NET_DHCP}" == [Yy1]* ]]; then

		VM_NET_TAP="_VmMacvtap"

		log "INFO" "... to retrieve IP via DHCP through Macvtap (${VM_NET_TAP}) and MAC: ${VM_NET_MAC}"



		ip l add link eth0 name ${VM_NET_TAP} address ${VM_NET_MAC} type macvtap mode bridge || true

		ip l set ${VM_NET_TAP} up



		ip a flush eth0

		ip a flush ${VM_NET_TAP}



		_DhcpIP=$( dhclient -v ${VM_NET_TAP} 2>&1 | grep ^bound | cut -d' ' -f3 )

		[[ "${_DhcpIP}" == [0-9.]* ]] \

		&& log "INFO" "... Retrieve IP: ${_DhcpIP} from DHCP with MAC: ${VM_NET_MAC}" \

		|| ( log "ERROR" "... Cannot retrieve IP from DHCP with MAC: ${VM_NET_MAC}" && exit 16 )



		ip a flush ${VM_NET_TAP}



		_tmpTapPath="/dev/tap$(</sys/class/net/${VM_NET_TAP}/ifindex)"

		# get MAJOR MINOR DEVNAME

		MAJOR=""

		eval "$(</sys/class/net/${VM_NET_TAP}/macvtap/${_tmpTapPath##*/}/uevent) _tmp=0"



		[[ "x${MAJOR}" != "x" ]] \

			&& log "INFO" "... PLEASE MAKE SURE, Docker run command line used: --device-cgroup-rule='c ${MAJOR}:* rwm'" \

			|| ( log "ERROR" "... macvtap creation issue: Cannot find: /sys/class/net/${VM_NET_TAP}/" && exit 18 )



		[[ ! -e ${_tmpTapPath} ]] && [[ -e /dev0/${_tmpTapPath##*/} ]] && ln -s /dev0/${_tmpTapPath##*/} ${_tmpTapPath}



		if [[ ! -e ${_tmpTapPath} ]]; then

			log "WARNING" "... file does not exist: ${_tmpTapPath}"

			mknod ${_tmpTapPath} c $MAJOR $MINOR \

				&& log "INFO" "... File created with mknod: ${_tmpTapPath}" \

				|| ( log "ERROR" "... Cannot mknod: ${_tmpTapPath}" && exit 20 )

		fi

		KVM_NET_OPTS="-netdev tap,id=hostnet0,vhost=on,vhostfd=40,fd=30 30<>${_tmpTapPath} 40<>/dev/vhost-net"

	else

		VM_NET_TAP="_VmNatTap"

		log "INFO" "... NAT Network (${VM_NET_TAP}) to ${VM_NET_IP}"



		configureNatNetworks

		KVM_NET_OPTS="-netdev tap,ifname=${VM_NET_TAP},script=no,downscript=no,id=hostnet0"



		# Build DNS options from container /etc/resolv.conf

		nameservers=($(grep '^nameserver' /etc/resolv.conf | sed 's/nameserver //'))

		searchdomains=$(grep '^search' /etc/resolv.conf | sed 's/search //' | sed 's/ /,/g')

		domainname=$(echo $searchdomains | awk -F"," '{print $1}')



# Now we start modifying the networking configuration. First we clear out

# the IP address of the default device (will also have the side-effect of

# removing the default route)

ip addr flush dev "$default_dev"

		for nameserver in "${nameservers[@]}"; do

		  if [[ $nameserver =~ .*:.* ]]; then

		    log "INFO" "Skipping IPv6 nameserver: $nameserver"

		  else

		    [[ -z $DNS_SERVERS ]] && DNS_SERVERS=$nameserver || DNS_SERVERS="$DNS_SERVERS,$nameserver"

		  fi

		done

		DNSMASQ_OPTS="$DNSMASQ_OPTS                         \

		  --dhcp-option=option:dns-server,$DNS_SERVERS      \

		  --dhcp-option=option:router,${VM_NET_IP%.*}.1         \

		  --dhcp-option=option:domain-search,$searchdomains \

		  --dhcp-option=option:domain-name,$domainname      \

		  "

		[[ -z $(hostname -d) ]] || DNSMASQ_OPTS="$DNSMASQ_OPTS --dhcp-option=option:domain-name,$(hostname -d)"



# Next, we create our bridge, and add our container interface to it.

ip link add "$QEMU_BRIDGE" type bridge

ip link set dev "$default_dev" master "$QEMU_BRIDGE"

		log "INFO" "... Lauching dnsmasq"

		log "DEBUG" "dnsmasq options: $DNSMASQ_OPTS"

		$DNSMASQ $DNSMASQ_OPTS

	fi

else

	log "INFO" "... No configuration, just using tuntap : ${VM_NET_TAP}"

	KVM_NET_OPTS="-netdev tap,ifname=${VM_NET_TAP},script=no,downscript=no,id=hostnet0"

fi



# Then, we toggle the interface and the bridge to make sure everything is up

# and running.

ip link set dev "$default_dev" up

ip link set dev "$QEMU_BRIDGE" up

#KVM_NET_OPTS="-netdev user,hostfwd=tcp:127.0.0.1:5000-:5000"

[[ "${VM_ENABLE_VIRTIO}" == [Yy1]* ]] \

	&& KVM_NET_OPTS="${KVM_NET_OPTS} -device virtio-net-pci,netdev=hostnet0,mac=${VM_NET_MAC},id=net0" \

	|| KVM_NET_OPTS="${KVM_NET_OPTS} -device e1000e,netdev=hostnet0,mac=${VM_NET_MAC},id=net0"



# Prevent error about missing file

touch /var/lib/misc/udhcpd.leases

# Hack for guest VMs complaining about "bad udp checksums in 5 packets"

log "INFO" "Hack for guest VMs complaining about: bad udp checksums in 5 packets"

iptables -A POSTROUTING -t mangle -p udp --dport bootpc -j CHECKSUM --checksum-fill \

        || ( log "WARNING" "Iptables hack for checksum FAILED" && ethtool -K eth0 tx off || true )



# Finally, start our DHCPD server

udhcpd -I $DUMMY_DHCPD_IP -f $DHCPD_CONF_FILE 2>&1 &

[[ "x${MAJOR}" != "x" ]] && log "INFO" "PLEASE MAKE SURE, Docker is using the following option otherwise you may have permission issue on ${_tmpTapPath} file: --device-cgroup-rule='c ${MAJOR}:* rwm' "

[[ "${_DhcpIP}" == [0-9.]* ]] && log "INFO" "You should access your DSM with: http://${_DhcpIP}:5000"



exit 0
 
log "INFO" "Done setting up network.."

qemu-ifdown

deleted100755 → 0
+0 −6
Original line number Diff line number Diff line 
#!/usr/bin/env bash



#QEMU_BRIDGE="qemubr0"



ip link set dev "$1" nomaster

ip link set dev "$1" down

qemu-ifup

deleted100755 → 0
+0 −6
Original line number Diff line number Diff line 
#!/usr/bin/env bash



QEMU_BRIDGE="qemubr0"



ip link set dev "$1" up

ip link set dev "$1" master "$QEMU_BRIDGE"