[aur-general] Support for source mirror lists in PKGBUILD

Ido Rosen ido at kernel.org
Fri Nov 1 00:08:44 EDT 2013


Oops, looks like the attachment didn't make it.  Here's the patch pasted:

>From bf4b5a6b531e2e9ec629907426ddf854735c649e Mon Sep 17 00:00:00 2001
From: Ido Rosen <code at idorosen.com>
Date: Thu, 31 Oct 2013 19:50:46 -0400
Subject: [PATCH] Added mirror support to makepkg's source array.

To specify multiple mirrors, simply add multiple files to the source array
that have the same downloaded filename, e.g.:

source=("file.tar.gz::http://mirror1.example.com/file.tar.gz"
        "file.tar.gz::http://mirror2.example.com/file.tar.gz")

...makepkg will try them all, and if all fail, will abort.  This also
applies
to VCS repositories and local files references in the source array, so for
example:

source=("git+
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git"
        "git+
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git")

...will allow cloning the git repository from GitHub if kernel.org is down
or
fails.
---
 scripts/makepkg.sh.in | 73
+++++++++++++++++++++++++++++++--------------------
 1 file changed, 45 insertions(+), 28 deletions(-)

diff --git a/scripts/makepkg.sh.in b/scripts/makepkg.sh.in
index 4cb8173..2fe43cf 100644
--- a/scripts/makepkg.sh.in
+++ b/scripts/makepkg.sh.in
@@ -326,7 +326,7 @@ download_local() {
  else
  local filename=$(get_filename "$netfile")
  error "$(gettext "%s was not found in the build directory and is not a
URL.")" "$filename"
- exit 1 # $E_MISSING_FILE
+ return 1 # $E_MISSING_FILE
  fi
 }

@@ -375,8 +375,7 @@ download_file() {
  if (( ret )); then
  [[ ! -s $dlfile ]] && rm -f -- "$dlfile"
  error "$(gettext "Failure while downloading %s")" "$filename"
- plain "$(gettext "Aborting...")"
- exit 1
+ return 1
  fi

  # rename the temporary download file to the final destination
@@ -460,8 +459,7 @@ download_bzr() {
  msg2 "$(gettext "Branching %s ...")" "${displaylocation}"
  if ! bzr branch "$url" "$dir" --no-tree --use-existing-dir; then
  error "$(gettext "Failure while branching %s")" "${displaylocation}"
- plain "$(gettext "Aborting...")"
- exit 1
+ return 1
  fi
  elif (( ! HOLDVER )); then
  # Make sure we are fetching the right repo
@@ -470,15 +468,13 @@ download_bzr() {
  if [[ -n $distant_url ]]; then
  if [[ $distant_url != "$local_url" ]]; then
  error "$(gettext "%s is not a branch of %s")" "$dir" "$url"
- plain "$(gettext "Aborting...")"
- exit 1
+ return 1
  fi
  else
  if [[ $url != "$local_url" ]] ; then
  error "$(gettext "%s is not a branch of %s")" "$dir" "$url"
  error "$(gettext "The local URL is %s")" "$local_url"
- plain "$(gettext "Aborting...")"
- exit 1
+ return 1
  fi
  fi
  msg2 "$(gettext "Pulling %s ...")" "${displaylocation}"
@@ -545,16 +541,14 @@ download_git() {
  msg2 "$(gettext "Cloning %s %s repo...")" "${repo}" "git"
  if ! git clone --mirror "$url" "$dir"; then
  error "$(gettext "Failure while downloading %s %s repo")" "${repo}" "git"
- plain "$(gettext "Aborting...")"
- exit 1
+ return 1
  fi
  elif (( ! HOLDVER )); then
  cd_safe "$dir"
  # Make sure we are fetching the right repo
  if [[ "$url" != "$(git config --get remote.origin.url)" ]] ; then
  error "$(gettext "%s is not a clone of %s")" "$dir" "$url"
- plain "$(gettext "Aborting...")"
- exit 1
+ return 1
  fi
  msg2 "$(gettext "Updating %s %s repo...")" "${repo}" "git"
  if ! git fetch --all -p; then
@@ -634,8 +628,7 @@ download_hg() {
  msg2 "$(gettext "Cloning %s %s repo...")" "${repo}" "hg"
  if ! hg clone -U "$url" "$dir"; then
  error "$(gettext "Failure while downloading %s %s repo")" "${repo}" "hg"
- plain "$(gettext "Aborting...")"
- exit 1
+ return 1
  fi
  elif (( ! HOLDVER )); then
  msg2 "$(gettext "Updating %s %s repo...")" "${repo}" "hg"
@@ -673,15 +666,13 @@ extract_hg() {
  ;;
  *)
  error "$(gettext "Unrecognized reference: %s")" "${fragment}"
- plain "$(gettext "Aborting...")"
- exit 1
+ return 1
  esac
  fi

  if ! hg clone "${ref[@]}" "$dir" "${dir##*/}"; then
  error "$(gettext "Failure while creating working copy of %s %s repo")"
"${repo}" "hg"
- plain "$(gettext "Aborting...")"
- exit 1
+ return 1
  fi

  popd &>/dev/null
@@ -711,8 +702,7 @@ download_svn() {
  mkdir -p "$dir/.makepkg"
  if ! svn checkout --config-dir "$dir/.makepkg" "$url" "$dir"; then
  error "$(gettext "Failure while downloading %s %s repo")" "${repo}" "svn"
- plain "$(gettext "Aborting...")"
- exit 1
+ return 1
  fi
  elif (( ! HOLDVER )); then
  msg2 "$(gettext "Updating %s %s repo...")" "${repo}" "svn"
@@ -776,34 +766,61 @@ download_sources() {
  GET_VCS=0
  fi

+ declare -A downloaded # local since no -g supplied to declare.
+ declare -A expected # local since no -g supplied to declare.
  local netfile
  for netfile in "${source[@]}"; do
  pushd "$SRCDEST" &>/dev/null
+ local filepath=$(get_filepath "${netfile}")
+ expected["${filepath}"]=1
+ if [[ ${downloaded["${filepath}"]} ]]; then
+ msg "$(gettext "Skipping mirror: %s")" "$netfile"
+ continue # file already downloaded, current mirror is redundant.
+ fi

  local proto=$(get_protocol "$netfile")
  case "$proto" in
  local)
- download_local "$netfile"
+ download_local "$netfile" && \
+ downloaded["${filepath}"]=1
  ;;
  bzr*)
- (( GET_VCS )) && download_bzr "$netfile"
+ (( GET_VCS )) && download_bzr "$netfile" && \
+ downloaded["${filepath}"]=1
  ;;
  git*)
- (( GET_VCS )) && download_git "$netfile"
+ (( GET_VCS )) && download_git "$netfile" && \
+ downloaded["${filepath}"]=1
  ;;
  hg*)
- (( GET_VCS )) && download_hg "$netfile"
+ (( GET_VCS )) && download_hg "$netfile" && \
+ downloaded["${filepath}"]=1
  ;;
  svn*)
- (( GET_VCS )) && download_svn "$netfile"
+ (( GET_VCS )) && download_svn "$netfile" && \
+ downloaded["${filepath}"]=1
  ;;
  *)
- download_file "$netfile"
+ download_file "$netfile" && \
+ downloaded["${filepath}"]=1
  ;;
  esac
-
  popd &>/dev/null
  done
+
+ local failed=()
+ for expectedfilepath in "${expected[@]}"; do
+ if [[ ${downloaded["${expectedfilepath}"]} ]]; then
+ continue
+ else
+ failed=(${failed[@]} "${expectedfilepath}")
+ error "$(gettext "Fatal failure while downloading file %s")"
"${expectedfilepath}"
+ fi
+ done
+ if [[ ${#failed[@]} -gt 0 ]]; then
+ plain "$(gettext "Aborting...")"
+ exit 1
+ fi
 }

 # Automatically update pkgver variable if a pkgver() function is provided
-- 
1.8.4.2



On Thu, Oct 31, 2013 at 7:57 PM, Ido Rosen <ido at kernel.org> wrote:

> Here is my POC.  It should work for remote files (URLs), local files, VCS
> repos, and across different types (e.g. if you want to offer git and hg
> mirrors of a repo).
>
> Totally untested.  Feel free to change it if it's broken and take all the
> credit and submit to pacman-dev. :)
>
> Ido
>
>
> On Thu, Oct 31, 2013 at 7:20 PM, Florian Pritz <bluewind at xinu.at> wrote:
>
>> On 31.10.2013 23:30, Ido Rosen wrote:
>> > If my first pacman patch is going to be this big of a change to
>> > makepkg.sh.in I'd feel a lot more comfortable if an existing
>> pacman/makepkg
>> > dev were available off-list via email or IM for some brain-picking.  Any
>> > volunteers?
>>
>> Got a poc for a rather simple way of implementing it, not sure if it's
>> going to be pretty enough.
>>
>> I'll take a closer look tomorrow.
>>
>> Feel free to ping me via irc, mail or jabber [1].
>>
>> [1]: https://www.archlinux.org/developers/#bluewind
>>
>>
>


More information about the aur-general mailing list