Unverified Commit 8bc17b5a authored by Kroese's avatar Kroese Committed by GitHub
Browse files

feat: Support download of disk images (#560)

parent 5dfcfcc3

Dockerfile

+3 −0
Original line number Diff line number Diff line
@@ -12,7 +12,9 @@ RUN set -eu && \ 
    apt-get --no-install-recommends -y install \

        tini \

        wget \

        7zip \

        ovmf \

        fdisk \

        nginx \

        swtpm \

        procps \
@@ -20,6 +22,7 @@ RUN set -eu && \ 
        iproute2 \

        apt-utils \

        dnsmasq \

        xz-utils \

        net-tools \

        qemu-utils \

        genisoimage \

readme.md

+16 −8
Original line number Diff line number Diff line
@@ -17,13 +17,15 @@ Docker container for running virtual machines using QEMU. 


  - Create VM's which behave just like normal containers



  - Manage them using all your existing tools (like Portainer) and configure them in a language (YAML) you are already familiar with

  - Manage them using all your existing tools (like Portainer)



  - Reduces the learning curve and eliminates the need for a dedicated Proxmox or ESXi server

  - Configure them in a language (YAML) you are already familiar with



  - Web-based viewer to control the machine directly from your browser



  - High-performance QEMU options (like KVM acceleration, kernel-mode networking, IO threading, etc.) to achieve near-native speed

  - Supports `.iso`, `.img`, `.qcow2`, `.vhd`, `.vhdx`, `.vdi`, `.vmdk` and `.raw` disk formats



  - High-performance options (like KVM acceleration, kernel-mode networking, IO threading, etc.) to achieve near-native speed



## Usage  🐳
@@ -63,7 +65,7 @@ kubectl apply -f kubernetes.yml 


  Very simple! These are the steps:



  - Set the `BOOT` environment variable to the URL of an ISO image you want to install.

  - Set the `BOOT` environment variable to the URL of any [disk image](https://github.com/qemus/qemu-docker#what-image-formats-are-supported) you want to install.



  - Start the container and connect to [port 8006](http://localhost:8006) using your web browser.
@@ -93,16 +95,16 @@ kubectl apply -f kubernetes.yml 
  

  This can also be used to resize the existing disk to a larger capacity without any data loss.



* ### How do I boot a local ISO?

* ### How do I boot a local image?



  You can use a local file directly, and skip the download altogether, by binding it in your compose file in this way:

  You can use a local image file directly, and skip the download altogether, by binding it in your compose file:

  

  ```yaml

  volumes:

    - /home/user/example.iso:/boot.iso

  ```



  Replace the example path `/home/user/example.iso` with the filename of the desired ISO file, the value of `BOOT` will be ignored in this case.

  This way you can supply a `boot.iso`, `boot.img` or `boot.qcow2` file. The URL of the `BOOT` variable will be ignored in this case.



* ### How do I boot ARM images?
@@ -260,6 +262,12 @@ kubectl apply -f kubernetes.yml 
    ARGUMENTS: "-device usb-tablet"

  ```



* ### What image formats are supported?



  You can set the `BOOT` URL to any `.iso`, `.img`, `.raw`, `.qcow2`, `.vhd`, `.vhdx`, `.vdi` or `.vmdk` file.



  It will even automaticly extract compressed images, like `.img.gz`, `.qcow2.xz`, `.iso.zip` and many more!



## Stars 🌟

[![Stars](https://starchart.cc/qemus/qemu-docker.svg?variant=adaptive)](https://starchart.cc/qemus/qemu-docker)

src/config.sh

+2 −1
Original line number Diff line number Diff line
@@ -4,12 +4,13 @@ set -Eeuo pipefail 
: "${SERIAL:="mon:stdio"}"

: "${USB:="qemu-xhci,id=xhci"}"

: "${MONITOR:="telnet:localhost:7100,server,nowait,nodelay"}"

: "${SMP:="$CPU_CORES,sockets=1,dies=1,cores=$CPU_CORES,threads=1"}"



DEF_OPTS="-nodefaults"

SERIAL_OPTS="-serial $SERIAL"

CPU_OPTS="-cpu $CPU_FLAGS -smp $SMP"

USB_OPTS="-device $USB -device usb-tablet"

RAM_OPTS=$(echo "-m ${RAM_SIZE^^}" | sed 's/MB/M/g;s/GB/G/g;s/TB/T/g')

CPU_OPTS="-cpu $CPU_FLAGS -smp $CPU_CORES,sockets=1,dies=1,cores=$CPU_CORES,threads=1"

MON_OPTS="-monitor $MONITOR -name $PROCESS,process=$PROCESS,debug-threads=on"

MAC_OPTS="-machine type=${MACHINE},smm=${SECURE},graphics=off,vmport=off,dump-guest-core=off,hpet=off${KVM_OPTS}"

src/disk.sh

+21 −14
Original line number Diff line number Diff line
@@ -524,6 +524,7 @@ addDevice () { 
html "Initializing disks..."



[ -z "${DISK_OPTS:-}" ] && DISK_OPTS=""

[ -z "${DISK_NAME:-}" ] && DISK_NAME="data"

[ -z "${DISK_TYPE:-}" ] && DISK_TYPE="scsi"



case "${DISK_TYPE,,}" in
@@ -564,10 +565,16 @@ if [ -f "$DRIVERS" ] && [ -s "$DRIVERS" ]; then 
  DISK_OPTS+=$(addMedia "$DRIVERS" "$FALLBACK" "1" "" "0x6")

fi



DISK1_FILE="$STORAGE/data"

DISK2_FILE="/storage2/data2"

DISK3_FILE="/storage3/data3"

DISK4_FILE="/storage4/data4"

DISK1_FILE="/boot"

if [ ! -f "$DISK1_FILE.img" ] || [ ! -s "$DISK1_FILE.img" ]; then

  if [ ! -f "$DISK1_FILE.qcow2" ] || [ ! -s "$DISK1_FILE.qcow2" ]; then

    DISK1_FILE="$STORAGE/${DISK_NAME}"

  fi

fi



DISK2_FILE="/storage2/${DISK_NAME}2"

DISK3_FILE="/storage3/${DISK_NAME}3"

DISK4_FILE="/storage4/${DISK_NAME}4"



if [ -z "$DISK_FMT" ]; then

  if [ -f "$DISK1_FILE.qcow2" ]; then

src/install.sh

+284 −41
Original line number Diff line number Diff line 
#!/usr/bin/env bash

set -Eeuo pipefail



detect () {

detectType() {



  local dir=""

  local file="$1"



  [ ! -f "$file" ] && return 1

  [ ! -s "$file" ] && return 1



  if [ -z "${BOOT_MODE:-}" ]; then

  case "${file,,}" in

    *".iso" )



      BOOT="$file"

      [ -n "${BOOT_MODE:-}" ] && return 0



      # Automaticly detect UEFI-compatible ISO's

      dir=$(isoinfo -f -i "$file")

      [ -z "$dir" ] && error "Failed to read ISO file, invalid format!" && BOOT="" && return 1



      dir=$(echo "${dir^^}" | grep "^/EFI")

      [ -n "$dir" ] && BOOT_MODE="uefi"

      ;;



    *".img" )



      DISK_NAME=$(basename "$file")

      DISK_NAME="${DISK_NAME%.*}"

      [ -n "${BOOT_MODE:-}" ] && return 0



      # Automaticly detect UEFI-compatible images

      dir=$(sfdisk -l "$file")

      [ -z "$dir" ] && error "Failed to read IMG file, invalid format!" && DISK_NAME="" && return 1



      dir=$(echo "${dir^^}" | grep "EFI SYSTEM")

      [ -n "$dir" ] && BOOT_MODE="uefi"

      ;;



    *".qcow2" )



      DISK_NAME=$(basename "$file")

      DISK_NAME="${DISK_NAME%.*}"

      [ -n "${BOOT_MODE:-}" ] && return 0



      # TODO: Detect boot mode from partition table in image

      BOOT_MODE="uefi"

      ;;



    * )

      return 1 ;;

  esac



  return 0

}



downloadFile() {



  local url="$1"

  local base="$2"

  local msg rc total progress



  local dest="$STORAGE/$base.tmp"

  rm -f "$dest"



  # Check if running with interactive TTY or redirected to docker log

  if [ -t 1 ]; then

    progress="--progress=bar:noscroll"

  else

    progress="--progress=dot:giga"

  fi



  BOOT="$file"

  msg="Downloading image"

  info "Downloading $base..."

  html "$msg..."



  /run/progress.sh "$dest" "0" "$msg ([P])..." &



  { wget "$url" -O "$dest" -q --timeout=30 --show-progress "$progress"; rc=$?; } || :



  fKill "progress.sh"



  if (( rc == 0 )) && [ -f "$dest" ]; then

    total=$(stat -c%s "$dest")

    if [ "$total" -lt 100000 ]; then

      error "Invalid image file: is only $total bytes?" && return 1

    fi

    html "Download finished successfully..."

    mv -f "$dest" "$STORAGE/$base"

    return 0

  fi



  msg="Failed to download $url"

  (( rc == 3 )) && error "$msg , cannot write file (disk full?)" && return 1

  (( rc == 4 )) && error "$msg , network failure!" && return 1

  (( rc == 8 )) && error "$msg , server issued an error response!" && return 1



  error "$msg , reason: $rc"

  return 1

}



convertImage() {



  local source_file=$1

  local source_fmt=$2

  local dst_file=$3

  local dst_fmt=$4

  local dir base fs fa cur_size src_size space disk_param



  [ -f "$dst_file" ] && error "Conversion failed, destination file $dst_file already exists?" && return 1

  [ ! -f "$source_file" ] && error "Conversion failed, source file $source_file does not exists?" && return 1



  if [[ "$source_fmt" == "raw" ]] && [[ "$dst_fmt" == "raw" ]]; then

    mv -f "$source_file" "$dst_file"

    return 0

  fi



  local tmp_file="$dst_file.tmp"

  dir=$(dirname "$tmp_file")



  rm -f "$tmp_file"



  if [ -n "$ALLOCATE" ] && [[ "$ALLOCATE" != [Nn]* ]]; then



    # Check free diskspace

    src_size=$(qemu-img info "$source_file" -f "$source_fmt" | grep '^virtual size: ' | sed 's/.*(\(.*\) bytes)/\1/')

    space=$(df --output=avail -B 1 "$dir" | tail -n 1)



    if (( src_size > space )); then

      local space_gb=$(( (space + 1073741823)/1073741824 ))

      error "Not enough free space to convert image in $dir, it has only $space_gb GB available..." && return 1

    fi

  fi



  base=$(basename "$source_file")

  info "Converting $base..."

  html "Converting image..."



  local conv_flags="-p"



  if [ -z "$ALLOCATE" ] || [[ "$ALLOCATE" == [Nn]* ]]; then

    disk_param="preallocation=off"

  else

    disk_param="preallocation=falloc"

  fi



  fs=$(stat -f -c %T "$dir")

  [[ "${fs,,}" == "btrfs" ]] && disk_param+=",nocow=on"



  if [[ "$dst_fmt" != "raw" ]]; then

    if [ -z "$ALLOCATE" ] || [[ "$ALLOCATE" == [Nn]* ]]; then

      conv_flags+=" -c"

    fi

    [ -n "${DISK_FLAGS:-}" ] && disk_param+=",$DISK_FLAGS"

  fi



  # shellcheck disable=SC2086

  if ! qemu-img convert -f "$source_fmt" $conv_flags -o "$disk_param" -O "$dst_fmt" -- "$source_file" "$tmp_file"; then

    rm -f "$tmp_file"

    error "Failed to convert image in $dir, is there enough space available?" && return 1

  fi



  if [[ "$dst_fmt" == "raw" ]]; then

    if [ -n "$ALLOCATE" ] && [[ "$ALLOCATE" != [Nn]* ]]; then

      # Work around qemu-img bug

      cur_size=$(stat -c%s "$tmp_file")

      if ! fallocate -l "$cur_size" "$tmp_file"; then

        error "Failed to allocate $cur_size bytes for image!"

      fi

    fi

  fi



  rm -f "$source_file"

  mv "$tmp_file" "$dst_file"



  if [[ "${fs,,}" == "btrfs" ]]; then

    fa=$(lsattr "$dst_file")

    if [[ "$fa" != *"C"* ]]; then

      error "Failed to disable COW for image on ${fs^^} filesystem!"

    fi

  fi



  html "Conversion completed..."



  return 0

}



file=$(find / -maxdepth 1 -type f -iname boot.iso | head -n 1)

[ ! -s "$file" ] && file=$(find "$STORAGE" -maxdepth 1 -type f -iname boot.iso | head -n 1)

detect "$file" && return 0

findFile() {



  local ext="$1"

  local file



  file=$(find / -maxdepth 1 -type f -iname "boot.$ext" | head -n 1)

  [ ! -s "$file" ] && file=$(find "$STORAGE" -maxdepth 1 -type f -iname "boot.$ext" | head -n 1)

  detectType "$file" && return 0



  return 1

}



findFile "iso" && return 0

findFile "img" && return 0

findFile "qcow2" && return 0



if [ -z "$BOOT" ] || [[ "$BOOT" == *"example.com/image.iso" ]]; then

  hasDisk && return 0

  error "No boot disk specified, set BOOT= to the URL of an ISO file." && exit 64

  error "No boot disk specified, set BOOT= to the URL of a disk image file." && exit 64

fi



base=$(basename "$BOOT")

detect "$STORAGE/$base" && return 0



base=$(basename "${BOOT%%\?*}")

: "${base//+/ }"; printf -v base '%b' "${_//%/\\x}"

base=$(echo "$base" | sed -e 's/[^A-Za-z0-9._-]/_/g')

detect "$STORAGE/$base" && return 0



TMP="$STORAGE/${base%.*}.tmp"

rm -f "$TMP"

case "${base,,}" in



# Check if running with interactive TTY or redirected to docker log

if [ -t 1 ]; then

  progress="--progress=bar:noscroll"

else

  progress="--progress=dot:giga"

  *".iso" | *".img" | *".qcow2" )



    detectType "$STORAGE/$base" && return 0 ;;



  *".raw" | *".vdi" | *".vmdk" | *".vhd" | *".vhdx" )



    detectType "$STORAGE/${base%.*}.img" && return 0

    detectType "$STORAGE/${base%.*}.qcow2" && return 0 ;;



  *".gz" | *".gzip" | *".xz" | *".7z" | *".zip" | *".rar" | *".lzma" | *".bz" | *".bz2" )



    case "${base%.*}" in

      *".iso" | *".img" | *".qcow2" )



        detectType "$STORAGE/${base%.*}" && return 0 ;;



      *".raw" | *".vdi" | *".vmdk" | *".vhd" | *".vhdx" )



        find="${base%.*}"



        detectType "$STORAGE/${find%.*}.img" && return 0

        detectType "$STORAGE/${find%.*}.qcow2" && return 0 ;;



    esac ;;



  * )

    error "Unknown file format, extension \".${base/*./}\" is not recognized!" && exit 33 ;;

esac



if ! downloadFile "$BOOT" "$base"; then

  rm -f "$STORAGE/$base.tmp" && exit 60

fi



msg="Downloading $base"

info "$msg..." && html "$msg..."

case "${base,,}" in

  *".gz" | *".gzip" | *".xz" | *".7z" | *".zip" | *".rar" | *".lzma" | *".bz" | *".bz2" )

    info "Extracting $base..."

    html "Extracting image..." ;;

esac



/run/progress.sh "$TMP" "" "$msg ([P])..." &

{ wget "$BOOT" -O "$TMP" -q --timeout=30 --show-progress "$progress"; rc=$?; } || :

case "${base,,}" in

  *".gz" | *".gzip" )



fKill "progress.sh"

    gzip -dc "$STORAGE/$base" > "$STORAGE/${base%.*}"

    rm -f "$STORAGE/$base"

    base="${base%.*}"



msg="Failed to download $BOOT"

(( rc == 3 )) && error "$msg , cannot write file (disk full?)" && exit 60

(( rc == 4 )) && error "$msg , network failure!" && exit 60

(( rc == 8 )) && error "$msg , server issued an error response!" && exit 60

(( rc != 0 )) && error "$msg , reason: $rc" && exit 60

[ ! -s "$TMP" ] && error "$msg" && exit 61

    ;;

  *".xz" )



html "Download finished successfully..."

    xz -dc "$STORAGE/$base" > "$STORAGE/${base%.*}"

    rm -f "$STORAGE/$base"

    base="${base%.*}"



size=$(stat -c%s "$TMP")

    ;;

  *".7z" | *".zip" | *".rar" | *".lzma" | *".bz" | *".bz2" )



if ((size<100000)); then

  error "Invalid ISO file: Size is smaller than 100 KB" && exit 62

    tmp="$STORAGE/extract"

    rm -rf "$tmp"

    mkdir -p "$tmp"

    7z x "$STORAGE/$base" -o"$tmp" > /dev/null



    rm -f "$STORAGE/$base"

    base="${base%.*}"



    if [ ! -s "$tmp/$base" ]; then

      rm -rf "$tmp"

      error "Cannot find file \"${base}\" in .${BOOT/*./} archive!" && exit 32

    fi



mv -f "$TMP" "$STORAGE/$base"

! detect "$STORAGE/$base" && exit 63

    mv "$tmp/$base" "$STORAGE/$base"

    rm -rf "$tmp"



return 0
 
    ;;

esac



case "${base,,}" in

  *".iso" | *".img" | *".qcow2" )

    detectType "$STORAGE/$base" && return 0

    error "Cannot read file \"${base}\"" && exit 63 ;;

esac



target_ext="img"

target_fmt="${DISK_FMT:-}"

[ -z "$target_fmt" ] && target_fmt="raw"

[[ "$target_fmt" != "raw" ]] && target_ext="qcow2"



case "${base,,}" in

  *".raw" ) source_fmt="raw" ;;

  *".vdi" ) source_fmt="vdi" ;;

  *".vhd" ) source_fmt="vhd" ;;

  *".vmdk" ) source_fmt="vmdk" ;;

  *".vhdx" ) source_fmt="vhdx" ;;

  * )

    error "Unknown file format, extension \".${base/*./}\" is not recognized!" && exit 33 ;;

esac



dst="$STORAGE/${base%.*}.$target_ext"



! convertImage "$STORAGE/$base" "$source_fmt" "$dst" "$target_fmt" && exit 35



base=$(basename "$dst")

detectType "$STORAGE/$base" && return 0

error "Cannot read file \"${base}\"" && exit 36