diff --git a/README.md b/README.md index 0fb5665..2d7d1d7 100644 --- a/README.md +++ b/README.md @@ -48,5 +48,13 @@ ISO (for example `Win11_25H2_English_x64_v2.iso` → existing `*Win*11*x64*.qcow2` in `base-qcow2/` is used instead. Download URLs can be overridden with `DEBIAN_IMAGE_URL`, `ALMA_IMAGE_URL`, -`FEDORA_IMAGE_URL`, and `VIRTIO_WIN_ISO_URL`. The Fedora checksum can be +`FEDORA_IMAGE_URL`, and `VIRTIO_WIN_ISO_URL`. Each may list several mirrors of +the same file, separated by whitespace; the Fedora image defaults to the TUNA, +BFSU and NJU mirrors. + +When [aria2](https://aria2.github.io/) is installed (`aria2c`), an image is +downloaded from all of its mirrors at once, with a single connection to each +mirror. Without it, `curl` (or `wget`) tries the mirrors in order. A partial +download is only resumed with the same downloader and mirror list; otherwise +it starts over. The Fedora checksum can be overridden with `FEDORA_IMAGE_SHA256` when using a custom image. diff --git a/download-images.sh b/download-images.sh index 4a1f84e..048f995 100755 --- a/download-images.sh +++ b/download-images.sh @@ -21,7 +21,9 @@ if [[ "$image_arch" == arm64 ]]; then fedora_image="${base_qcow2_dir}/Fedora-Cloud-Base-Generic-44-1.7.aarch64.qcow2" debian_url_default="https://cloud.debian.org/images/cloud/trixie/20260810-2566/debian-13-generic-arm64-20260810-2566.qcow2" alma_url_default="https://repo.almalinux.org/almalinux/10/cloud/aarch64/images/AlmaLinux-10-GenericCloud-10.2-20260526.0.aarch64.qcow2" - fedora_url_default="https://mirrors.tuna.tsinghua.edu.cn/fedora/releases/44/Cloud/aarch64/images/Fedora-Cloud-Base-Generic-44-1.7.aarch64.qcow2" + fedora_url_default="https://mirrors.tuna.tsinghua.edu.cn/fedora/releases/44/Cloud/aarch64/images/Fedora-Cloud-Base-Generic-44-1.7.aarch64.qcow2 + https://mirrors.bfsu.edu.cn/fedora/releases/44/Cloud/aarch64/images/Fedora-Cloud-Base-Generic-44-1.7.aarch64.qcow2 + https://mirrors.nju.edu.cn/fedora/releases/44/Cloud/aarch64/images/Fedora-Cloud-Base-Generic-44-1.7.aarch64.qcow2" # TODO: these were pinned for the amd64/x86_64 builds; verification is # skipped (empty default) for the arm64/aarch64 builds until their # checksums are fetched and pinned via DEBIAN_IMAGE_SHA512 / @@ -35,7 +37,9 @@ else fedora_image="${base_qcow2_dir}/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2" debian_url_default="https://cloud.debian.org/images/cloud/trixie/20260810-2566/debian-13-generic-amd64-20260810-2566.qcow2" alma_url_default="https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/AlmaLinux-10-GenericCloud-10.2-20260526.0.x86_64.qcow2" - fedora_url_default="https://mirrors.tuna.tsinghua.edu.cn/fedora/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2" + fedora_url_default="https://mirrors.tuna.tsinghua.edu.cn/fedora/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2 + https://mirrors.bfsu.edu.cn/fedora/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2 + https://mirrors.nju.edu.cn/fedora/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2" debian_sha512_default="f6978100d8031c266d55d7815ceea7fcdeacf28e1e5834fdb9c94ac96880a054a6e6f8681c2d3b0584e0057eaf3ef7353856b85212d04134744faa9b3bb1f24f" alma_sha256_default="47f2218668dd4776be140dd92fa3bea700be1766e2c7d88bdfd6a4b50f477b4d" fedora_sha256_default="28680fe5b371a5a82ebf43a31926e086a168e59949d03969c5093e7071f90b7f" @@ -79,6 +83,9 @@ windows_qcow2_path() { fi } +# Each *_url may list several mirrors of the same file, separated by +# whitespace. aria2c downloads from all of them at once; curl and wget try +# them in order. debian_url="${DEBIAN_IMAGE_URL:-$debian_url_default}" alma_url="${ALMA_IMAGE_URL:-$alma_url_default}" fedora_url="${FEDORA_IMAGE_URL:-$fedora_url_default}" @@ -97,6 +104,10 @@ Download one or more shared VM images. With no arguments, all targets are processed. The Debian/AlmaLinux/Fedora image architecture (arm64 or amd64) is auto-detected from the host; override it with VM_IMAGE_ARCH. +When aria2c is installed and an image has several mirrors, the image is +downloaded from all of them at once, with one connection per mirror. +Otherwise curl or wget downloads from the mirrors in order. + --force removes existing files and partial downloads for the selected targets, then downloads them again from the beginning. --print-path prints only the absolute image path without downloading it. @@ -201,19 +212,72 @@ verify_file() { printf 'Verified %s checksum for %s.\n' "${algorithm^^}" "$label" } +# Download to $partial with curl or wget from one URL, resuming an existing +# partial file and restarting from the beginning if resuming fails. +download_single_source() { + local downloader="$1" + local url="$2" + local partial="$3" + + if [[ "$downloader" == curl ]]; then + curl --fail --location --retry 3 --continue-at - --output "$partial" "$url" && return + [[ -s "$partial" ]] || return 1 + printf 'Resume failed; restarting download from the beginning.\n' + rm -f -- "$partial" + curl --fail --location --retry 3 --output "$partial" "$url" + else + wget --tries=3 --continue --output-document="$partial" "$url" && return + [[ -s "$partial" ]] || return 1 + printf 'Resume failed; restarting download from the beginning.\n' + rm -f -- "$partial" + wget --tries=3 --output-document="$partial" "$url" + fi +} + +# Download to $partial with aria2c, splitting the file across all URLs with a +# single connection to each server. +download_multi_source() { + local partial="$1" + shift + + aria2c \ + --continue=true \ + --split="$#" \ + --max-connection-per-server=1 \ + --file-allocation=none \ + --auto-file-renaming=false \ + --max-tries=5 \ + --retry-wait=3 \ + --summary-interval=0 \ + --console-log-level=warn \ + --dir="$(dirname -- "$partial")" \ + --out="$(basename -- "$partial")" \ + "$@" +} + download_file() { local label="$1" - local url="$2" + local urls + read -r -d '' -a urls <<<"$2" || true local destination="$3" local checksum_algorithm="${4:-}" local expected_checksum="${5:-}" local partial="${destination}.part" + # Records how the partial file was started, so that a partial download is + # only resumed with the same downloader and URLs. local partial_url="${partial}.url" local downloader + local download_state + local url + + if ((${#urls[@]} == 0)); then + printf 'Error: no download URL for %s.\n' "$label" >&2 + exit 1 + fi if [[ "$force_download" == true ]]; then printf 'Forcing a fresh download of %s.\n' "$label" - rm -f -- "$destination" "$partial" "$partial_url" + rm -f -- "$destination" "$partial" "${partial}.aria2" "$partial_url" fi if [[ -s "$destination" ]]; then @@ -224,48 +288,51 @@ download_file() { return fi - printf 'Downloading %s from:\n%s\n' "$label" "$url" - printf 'Download destination:\n%s\n' "$destination" - - if [[ -e "$partial" ]]; then - if [[ ! -f "$partial_url" ]] || [[ "$(<"$partial_url")" != "$url" ]]; then - printf 'Download URL changed; restarting partial download.\n' - rm -f -- "$partial" "$partial_url" - else - printf 'Resuming partial download:\n%s\n' "$partial" - fi - fi - printf '%s' "$url" >"$partial_url" - - if command -v curl >/dev/null 2>&1; then + if command -v aria2c >/dev/null 2>&1; then + downloader=aria2c + elif command -v curl >/dev/null 2>&1; then downloader=curl elif command -v wget >/dev/null 2>&1; then downloader=wget else - printf 'Error: curl or wget is required.\n' >&2 + printf 'Error: aria2c, curl or wget is required.\n' >&2 exit 1 fi - if [[ "$downloader" == curl ]]; then - if ! curl --fail --location --retry 3 --continue-at - --output "$partial" "$url"; then - if [[ -s "$partial" ]]; then - printf 'Resume failed; restarting download from the beginning.\n' - rm -f -- "$partial" - curl --fail --location --retry 3 --output "$partial" "$url" - else - exit 1 - fi + printf 'Downloading %s with %s from:\n' "$label" "$downloader" + printf '%s\n' "${urls[@]}" + printf 'Download destination:\n%s\n' "$destination" + + # aria2c fills a partial file out of order, so it must not be resumed by + # curl or wget, and vice versa. + download_state="$(printf '%s\n' "$downloader" "${urls[@]}")" + if [[ -e "$partial" ]]; then + if [[ ! -f "$partial_url" ]] || [[ "$(<"$partial_url")" != "$download_state" ]]; then + printf 'Download source changed; restarting partial download.\n' + rm -f -- "$partial" "${partial}.aria2" "$partial_url" + else + printf 'Resuming partial download:\n%s\n' "$partial" + fi + fi + printf '%s\n' "$download_state" >"$partial_url" + + if [[ "$downloader" == aria2c ]]; then + if ! download_multi_source "$partial" "${urls[@]}"; then + printf 'Error: failed to download %s.\n' "$label" >&2 + exit 1 fi else - if ! wget --tries=3 --continue --output-document="$partial" "$url"; then - if [[ -s "$partial" ]]; then - printf 'Resume failed; restarting download from the beginning.\n' - rm -f -- "$partial" - wget --tries=3 --output-document="$partial" "$url" - else + for url in "${urls[@]}"; do + # Some mirrors reject clients with an empty "200 OK" response. + if download_single_source "$downloader" "$url" "$partial" && [[ -s "$partial" ]]; then + break + fi + if [[ "$url" == "${urls[-1]}" ]]; then + printf 'Error: failed to download %s.\n' "$label" >&2 exit 1 fi - fi + printf 'Download from %s failed; trying the next mirror.\n' "$url" >&2 + done fi if [[ ! -s "$partial" ]]; then @@ -275,7 +342,7 @@ download_file() { if [[ -n "$expected_checksum" ]] && \ ! verify_file "$label" "$partial" "$checksum_algorithm" "$expected_checksum"; then - rm -f -- "$partial" "$partial_url" + rm -f -- "$partial" "${partial}.aria2" "$partial_url" return 1 fi