Download images from several mirrors at once with aria2c
Each *_IMAGE_URL may now list several mirrors of the same file. When aria2c is installed, the file is split across all mirrors with a single connection to each; otherwise curl or wget tries the mirrors in order, skipping mirrors that fail or return an empty response. A partial download is only resumed with the same downloader and mirror list. Fedora now defaults to the TUNA, BFSU and NJU mirrors. The Aliyun mirror is not used because it rejects aria2c. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YKxXLyiL7XJ3a9e8HmR2my
This commit is contained in:
GuofangandClaude Opus 5.5 committed
1 parent
eab2cc5283
commit
5ba114f5ad
2 files changed
+112
-37
No files matched your search
+103
-36
@@ -21,7 +21,9 @@ if [[ "$image_arch" == arm64 ]]; then
|
||||
fedora_image="${base_qcow2_dir}/Fedora-Cloud-Base-Generic-44-1.7.aarch64.qcow2"
|
||||
debian_url_default="https://cloud.debian.org/images/cloud/trixie/20260810-2566/debian-13-generic-arm64-20260810-2566.qcow2"
|
||||
alma_url_default="https://repo.almalinux.org/almalinux/10/cloud/aarch64/images/AlmaLinux-10-GenericCloud-10.2-20260526.0.aarch64.qcow2"
|
||||
fedora_url_default="https://mirrors.tuna.tsinghua.edu.cn/fedora/releases/44/Cloud/aarch64/images/Fedora-Cloud-Base-Generic-44-1.7.aarch64.qcow2"
|
||||
fedora_url_default="https://mirrors.tuna.tsinghua.edu.cn/fedora/releases/44/Cloud/aarch64/images/Fedora-Cloud-Base-Generic-44-1.7.aarch64.qcow2
|
||||
https://mirrors.bfsu.edu.cn/fedora/releases/44/Cloud/aarch64/images/Fedora-Cloud-Base-Generic-44-1.7.aarch64.qcow2
|
||||
https://mirrors.nju.edu.cn/fedora/releases/44/Cloud/aarch64/images/Fedora-Cloud-Base-Generic-44-1.7.aarch64.qcow2"
|
||||
# TODO: these were pinned for the amd64/x86_64 builds; verification is
|
||||
# skipped (empty default) for the arm64/aarch64 builds until their
|
||||
# checksums are fetched and pinned via DEBIAN_IMAGE_SHA512 /
|
||||
@@ -35,7 +37,9 @@ else
|
||||
fedora_image="${base_qcow2_dir}/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2"
|
||||
debian_url_default="https://cloud.debian.org/images/cloud/trixie/20260810-2566/debian-13-generic-amd64-20260810-2566.qcow2"
|
||||
alma_url_default="https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/AlmaLinux-10-GenericCloud-10.2-20260526.0.x86_64.qcow2"
|
||||
fedora_url_default="https://mirrors.tuna.tsinghua.edu.cn/fedora/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2"
|
||||
fedora_url_default="https://mirrors.tuna.tsinghua.edu.cn/fedora/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2
|
||||
https://mirrors.bfsu.edu.cn/fedora/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2
|
||||
https://mirrors.nju.edu.cn/fedora/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2"
|
||||
debian_sha512_default="f6978100d8031c266d55d7815ceea7fcdeacf28e1e5834fdb9c94ac96880a054a6e6f8681c2d3b0584e0057eaf3ef7353856b85212d04134744faa9b3bb1f24f"
|
||||
alma_sha256_default="47f2218668dd4776be140dd92fa3bea700be1766e2c7d88bdfd6a4b50f477b4d"
|
||||
fedora_sha256_default="28680fe5b371a5a82ebf43a31926e086a168e59949d03969c5093e7071f90b7f"
|
||||
@@ -79,6 +83,9 @@ windows_qcow2_path() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Each *_url may list several mirrors of the same file, separated by
|
||||
# whitespace. aria2c downloads from all of them at once; curl and wget try
|
||||
# them in order.
|
||||
debian_url="${DEBIAN_IMAGE_URL:-$debian_url_default}"
|
||||
alma_url="${ALMA_IMAGE_URL:-$alma_url_default}"
|
||||
fedora_url="${FEDORA_IMAGE_URL:-$fedora_url_default}"
|
||||
@@ -97,6 +104,10 @@ Download one or more shared VM images. With no arguments, all targets are
|
||||
processed. The Debian/AlmaLinux/Fedora image architecture (arm64 or amd64)
|
||||
is auto-detected from the host; override it with VM_IMAGE_ARCH.
|
||||
|
||||
When aria2c is installed and an image has several mirrors, the image is
|
||||
downloaded from all of them at once, with one connection per mirror.
|
||||
Otherwise curl or wget downloads from the mirrors in order.
|
||||
|
||||
--force removes existing files and partial downloads for the selected targets,
|
||||
then downloads them again from the beginning.
|
||||
--print-path prints only the absolute image path without downloading it.
|
||||
@@ -201,19 +212,72 @@ verify_file() {
|
||||
printf 'Verified %s checksum for %s.\n' "${algorithm^^}" "$label"
|
||||
}
|
||||
|
||||
# Download to $partial with curl or wget from one URL, resuming an existing
|
||||
# partial file and restarting from the beginning if resuming fails.
|
||||
download_single_source() {
|
||||
local downloader="$1"
|
||||
local url="$2"
|
||||
local partial="$3"
|
||||
|
||||
if [[ "$downloader" == curl ]]; then
|
||||
curl --fail --location --retry 3 --continue-at - --output "$partial" "$url" && return
|
||||
[[ -s "$partial" ]] || return 1
|
||||
printf 'Resume failed; restarting download from the beginning.\n'
|
||||
rm -f -- "$partial"
|
||||
curl --fail --location --retry 3 --output "$partial" "$url"
|
||||
else
|
||||
wget --tries=3 --continue --output-document="$partial" "$url" && return
|
||||
[[ -s "$partial" ]] || return 1
|
||||
printf 'Resume failed; restarting download from the beginning.\n'
|
||||
rm -f -- "$partial"
|
||||
wget --tries=3 --output-document="$partial" "$url"
|
||||
fi
|
||||
}
|
||||
|
||||
# Download to $partial with aria2c, splitting the file across all URLs with a
|
||||
# single connection to each server.
|
||||
download_multi_source() {
|
||||
local partial="$1"
|
||||
shift
|
||||
|
||||
aria2c \
|
||||
--continue=true \
|
||||
--split="$#" \
|
||||
--max-connection-per-server=1 \
|
||||
--file-allocation=none \
|
||||
--auto-file-renaming=false \
|
||||
--max-tries=5 \
|
||||
--retry-wait=3 \
|
||||
--summary-interval=0 \
|
||||
--console-log-level=warn \
|
||||
--dir="$(dirname -- "$partial")" \
|
||||
--out="$(basename -- "$partial")" \
|
||||
"$@"
|
||||
}
|
||||
|
||||
download_file() {
|
||||
local label="$1"
|
||||
local url="$2"
|
||||
local urls
|
||||
read -r -d '' -a urls <<<"$2" || true
|
||||
local destination="$3"
|
||||
local checksum_algorithm="${4:-}"
|
||||
local expected_checksum="${5:-}"
|
||||
local partial="${destination}.part"
|
||||
# Records how the partial file was started, so that a partial download is
|
||||
# only resumed with the same downloader and URLs.
|
||||
local partial_url="${partial}.url"
|
||||
local downloader
|
||||
local download_state
|
||||
local url
|
||||
|
||||
if ((${#urls[@]} == 0)); then
|
||||
printf 'Error: no download URL for %s.\n' "$label" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$force_download" == true ]]; then
|
||||
printf 'Forcing a fresh download of %s.\n' "$label"
|
||||
rm -f -- "$destination" "$partial" "$partial_url"
|
||||
rm -f -- "$destination" "$partial" "${partial}.aria2" "$partial_url"
|
||||
fi
|
||||
|
||||
if [[ -s "$destination" ]]; then
|
||||
@@ -224,48 +288,51 @@ download_file() {
|
||||
return
|
||||
fi
|
||||
|
||||
printf 'Downloading %s from:\n%s\n' "$label" "$url"
|
||||
printf 'Download destination:\n%s\n' "$destination"
|
||||
|
||||
if [[ -e "$partial" ]]; then
|
||||
if [[ ! -f "$partial_url" ]] || [[ "$(<"$partial_url")" != "$url" ]]; then
|
||||
printf 'Download URL changed; restarting partial download.\n'
|
||||
rm -f -- "$partial" "$partial_url"
|
||||
else
|
||||
printf 'Resuming partial download:\n%s\n' "$partial"
|
||||
fi
|
||||
fi
|
||||
printf '%s' "$url" >"$partial_url"
|
||||
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
if command -v aria2c >/dev/null 2>&1; then
|
||||
downloader=aria2c
|
||||
elif command -v curl >/dev/null 2>&1; then
|
||||
downloader=curl
|
||||
elif command -v wget >/dev/null 2>&1; then
|
||||
downloader=wget
|
||||
else
|
||||
printf 'Error: curl or wget is required.\n' >&2
|
||||
printf 'Error: aria2c, curl or wget is required.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$downloader" == curl ]]; then
|
||||
if ! curl --fail --location --retry 3 --continue-at - --output "$partial" "$url"; then
|
||||
if [[ -s "$partial" ]]; then
|
||||
printf 'Resume failed; restarting download from the beginning.\n'
|
||||
rm -f -- "$partial"
|
||||
curl --fail --location --retry 3 --output "$partial" "$url"
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
printf 'Downloading %s with %s from:\n' "$label" "$downloader"
|
||||
printf '%s\n' "${urls[@]}"
|
||||
printf 'Download destination:\n%s\n' "$destination"
|
||||
|
||||
# aria2c fills a partial file out of order, so it must not be resumed by
|
||||
# curl or wget, and vice versa.
|
||||
download_state="$(printf '%s\n' "$downloader" "${urls[@]}")"
|
||||
if [[ -e "$partial" ]]; then
|
||||
if [[ ! -f "$partial_url" ]] || [[ "$(<"$partial_url")" != "$download_state" ]]; then
|
||||
printf 'Download source changed; restarting partial download.\n'
|
||||
rm -f -- "$partial" "${partial}.aria2" "$partial_url"
|
||||
else
|
||||
printf 'Resuming partial download:\n%s\n' "$partial"
|
||||
fi
|
||||
fi
|
||||
printf '%s\n' "$download_state" >"$partial_url"
|
||||
|
||||
if [[ "$downloader" == aria2c ]]; then
|
||||
if ! download_multi_source "$partial" "${urls[@]}"; then
|
||||
printf 'Error: failed to download %s.\n' "$label" >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
if ! wget --tries=3 --continue --output-document="$partial" "$url"; then
|
||||
if [[ -s "$partial" ]]; then
|
||||
printf 'Resume failed; restarting download from the beginning.\n'
|
||||
rm -f -- "$partial"
|
||||
wget --tries=3 --output-document="$partial" "$url"
|
||||
else
|
||||
for url in "${urls[@]}"; do
|
||||
# Some mirrors reject clients with an empty "200 OK" response.
|
||||
if download_single_source "$downloader" "$url" "$partial" && [[ -s "$partial" ]]; then
|
||||
break
|
||||
fi
|
||||
if [[ "$url" == "${urls[-1]}" ]]; then
|
||||
printf 'Error: failed to download %s.\n' "$label" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
printf 'Download from %s failed; trying the next mirror.\n' "$url" >&2
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ ! -s "$partial" ]]; then
|
||||
@@ -275,7 +342,7 @@ download_file() {
|
||||
|
||||
if [[ -n "$expected_checksum" ]] && \
|
||||
! verify_file "$label" "$partial" "$checksum_algorithm" "$expected_checksum"; then
|
||||
rm -f -- "$partial" "$partial_url"
|
||||
rm -f -- "$partial" "${partial}.aria2" "$partial_url"
|
||||
return 1
|
||||
fi
|
||||
|
||||
|
||||
Reference in new issue
Block a user