New subject: [pacman-dev] [PATCH] Added mirror support to makepkg's source array.

1 Nov 2013

From: Ido Rosen code@idorosen.com
To specify multiple mirrors, simply add multiple files to the source array
that have the same downloaded filename, e.g.:
source=("file.tar.gz::http://mirror1.example.com/file.tar.gz"
        "file.tar.gz::http://mirror2.example.com/file.tar.gz")
...makepkg will try them all, and if all fail, will abort.  This also applies
to VCS repositories and local files references in the source array, so for
example:
source=("git+https://github.com/torvalds/linux.git
        "git+https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git")
...will allow cloning the git repository from GitHub if kernel.org is down or
fails.
Signed-off-by: Florian Pritz bluewind@xinu.at
---
I've swichted from get_filepath for the array index to get_filename as get_filepath only returns something useful if the local file already exists. I guess this was just an oversight, but noted for reference.
I've also tested this a little for http urls and it seems to work as expected.
One thing that we might want to fix is that the downloads get validated multiple times right now (once for each entry).
scripts/makepkg.sh.in | 73 +++++++++++++++++++++++++++++++--------------------
 1 file changed, 45 insertions(+), 28 deletions(-)

diff --git a/scripts/makepkg.sh.in b/scripts/makepkg.sh.in
index 4cb8173..452df09 100644
--- a/scripts/makepkg.sh.in
+++ b/scripts/makepkg.sh.in
@@ -326,7 +326,7 @@ download_local() {
    else
    	local filename=$(get_filename "$netfile")
    	error "$(gettext "%s was not found in the build directory and is not a URL.")" "$filename"
-		exit 1 # $E_MISSING_FILE
+		return 1 # $E_MISSING_FILE
    fi
 }
@@ -375,8 +375,7 @@ download_file() {
    if (( ret )); then
    	[[ ! -s $dlfile ]] && rm -f -- "$dlfile"
    	error "$(gettext "Failure while downloading %s")" "$filename"
-		plain "$(gettext "Aborting...")"
-		exit 1
+		return 1
    fi
# rename the temporary download file to the final destination
@@ -460,8 +459,7 @@ download_bzr() {
    	msg2 "$(gettext "Branching %s ...")" "${displaylocation}"
    	if ! bzr branch "$url" "$dir" --no-tree --use-existing-dir; then
    		error "$(gettext "Failure while branching %s")" "${displaylocation}"
-			plain "$(gettext "Aborting...")"
-			exit 1
+			return 1
    	fi
    elif (( ! HOLDVER )); then
    	# Make sure we are fetching the right repo
@@ -470,15 +468,13 @@ download_bzr() {
    	if [[ -n $distant_url ]]; then
    		if [[ $distant_url != "$local_url" ]]; then
    			error "$(gettext "%s is not a branch of %s")" "$dir" "$url"
-				plain "$(gettext "Aborting...")"
-				exit 1
+				return 1
    		fi
    	else
    		if [[ $url != "$local_url" ]] ; then
    			error "$(gettext "%s is not a branch of %s")" "$dir" "$url"
    			error "$(gettext "The local URL is %s")" "$local_url"
-				plain "$(gettext "Aborting...")"
-				exit 1
+				return 1
    		fi
    	fi
    	msg2 "$(gettext "Pulling %s ...")" "${displaylocation}"
@@ -545,16 +541,14 @@ download_git() {
    	msg2 "$(gettext "Cloning %s %s repo...")" "${repo}" "git"
    	if ! git clone --mirror "$url" "$dir"; then
    		error "$(gettext "Failure while downloading %s %s repo")" "${repo}" "git"
-			plain "$(gettext "Aborting...")"
-			exit 1
+			return 1
    	fi
    elif (( ! HOLDVER )); then
    	cd_safe "$dir"
    	# Make sure we are fetching the right repo
    	if [[ "$url" != "$(git config --get remote.origin.url)" ]] ; then
    		error "$(gettext "%s is not a clone of %s")" "$dir" "$url"
-			plain "$(gettext "Aborting...")"
-			exit 1
+			return 1
    	fi
    	msg2 "$(gettext "Updating %s %s repo...")" "${repo}" "git"
    	if ! git fetch --all -p; then
@@ -634,8 +628,7 @@ download_hg() {
    	msg2 "$(gettext "Cloning %s %s repo...")" "${repo}" "hg"
    	if ! hg clone -U "$url" "$dir"; then
    		error "$(gettext "Failure while downloading %s %s repo")" "${repo}" "hg"
-			plain "$(gettext "Aborting...")"
-			exit 1
+			return 1
    	fi
    elif (( ! HOLDVER )); then
    	msg2 "$(gettext "Updating %s %s repo...")" "${repo}" "hg"
@@ -673,15 +666,13 @@ extract_hg() {
    			;;
    		*)
    			error "$(gettext "Unrecognized reference: %s")" "${fragment}"
-				plain "$(gettext "Aborting...")"
-				exit 1
+				return 1
    	esac
    fi
if ! hg clone "${ref[@]}" "$dir" "${dir##*/}"; then
    	error "$(gettext "Failure while creating working copy of %s %s repo")" "${repo}" "hg"
-		plain "$(gettext "Aborting...")"
-		exit 1
+		return 1
    fi
popd &>/dev/null
@@ -711,8 +702,7 @@ download_svn() {
    	mkdir -p "$dir/.makepkg"
    	if ! svn checkout --config-dir "$dir/.makepkg" "$url" "$dir"; then
    		error "$(gettext "Failure while downloading %s %s repo")" "${repo}" "svn"
-			plain "$(gettext "Aborting...")"
-			exit 1
+			return 1
    	fi
    elif (( ! HOLDVER )); then
    	msg2 "$(gettext "Updating %s %s repo...")" "${repo}" "svn"
@@ -776,34 +766,61 @@ download_sources() {
    	GET_VCS=0
    fi
+	declare -A downloaded # local since no -g supplied to declare.
+	declare -A expected # local since no -g supplied to declare.
    local netfile
    for netfile in "${source[@]}"; do
    	pushd "$SRCDEST" &>/dev/null
+		local filename=$(get_filename "${netfile}")
+		expected["${filename}"]=1
+		if [[ ${downloaded["${filename}"]} ]]; then
+			msg "$(gettext "Skipping mirror: %s")" "$netfile"
+			continue # file already downloaded, current mirror is redundant.
+		fi
local proto=$(get_protocol "$netfile")
    	case "$proto" in
    		local)
-				download_local "$netfile"
+				download_local "$netfile" && \
+					downloaded["${filename}"]=1
    			;;
    		bzr*)
-				(( GET_VCS )) && download_bzr "$netfile"
+				(( GET_VCS )) && download_bzr "$netfile" && \
+					downloaded["${filename}"]=1
    			;;
    		git*)
-				(( GET_VCS )) && download_git "$netfile"
+				(( GET_VCS )) && download_git "$netfile" && \
+					downloaded["${filename}"]=1
    			;;
    		hg*)
-				(( GET_VCS )) && download_hg "$netfile"
+				(( GET_VCS )) && download_hg "$netfile" && \
+					downloaded["${filename}"]=1
    			;;
    		svn*)
-				(( GET_VCS )) && download_svn "$netfile"
+				(( GET_VCS )) && download_svn "$netfile" && \
+					downloaded["${filename}"]=1
    			;;
    		*)
-				download_file "$netfile"
+				download_file "$netfile" && \
+					downloaded["${filename}"]=1
    			;;
    	esac
-
    	popd &>/dev/null
    done
+
+	local failed=()
+	for expectedfilename in "${!expected[@]}"; do
+		if [[ ${downloaded["${expectedfilename}"]} ]]; then
+			continue
+		else
+			failed=(${failed[@]} "${expectedfilename}")
+			error "$(gettext "Fatal failure while downloading file %s")" "${expectedfilename}"
+		fi
+	done
+	if [[ ${#failed[@]} -gt 0 ]]; then
+		plain "$(gettext "Aborting...")"
+		exit 1
+	fi
 }
# Automatically update pkgver variable if a pkgver() function is provided
-- 
1.8.4.2

    