[pacman-dev] [PATCH] Make makepkg handle SCMs better.

22 Nov 2010

* Treat SCMs like download agents when attempting to download code from a
  source repository.
* Split the operations related to SCMs into three groups:
  1) pkgver_update(): update the version of a package build based on what SCM
     source repository is being pulled from (what if there are more than one?)
  2) devel_update(): If the source code repository already exists locally
     use these commands to update it.
  3) makepkg.conf additions: Use these commands to download a source code
     repository that does not exist locally (determined by existence of diretory)
* Use new syntax "scm@@filename::url". Filename is still optional. @@ is a
  neumonic hg 'at' url and @@ are special symbols in urls so we know a url
  will never have @@ in it.
* To avoid errors because the checksums may be wrong or missing, fill in the
  checksum with "SCM".
---

This may be a little rough at the moment, but I want some feedback on my ideas
to see what other people think. I tested this with git and hg and they both
seem to work. Currently, devel_update just hard wires the lines used to update
a repository; although, I was thinking maybe updating should be handled
similar to the way downloading is handled as well and UDAGENTS should also be
added to makepkg.conf. See [1] for current bugs and discussion.
[1]: https://wiki.archlinux.org/index.php/User:Allan/Makepkg_VCS_overhauln about SCM handling.

 scripts/makepkg.sh.in |  203 +++++++++++++++++++++++++-----------------------
 1 files changed, 106 insertions(+), 97 deletions(-)

diff --git a/scripts/makepkg.sh.in b/scripts/makepkg.sh.in
index de24338..844390e 100644
--- a/scripts/makepkg.sh.in
+++ b/scripts/makepkg.sh.in
@@ -177,9 +177,11 @@ trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT
 trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT
 trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR
 
-# a source entry can have two forms :
+# a source entry can have four forms:
 # 1) "filename::http://path/to/file"
 # 2) "http://path/to/file"
+# 3) "scm@@filename::http://path/to/repository"
+# 4) "scm@@http://path/to/repository"
 
 # Return the absolute filename of a source entry
 #
@@ -188,9 +190,9 @@ trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR
 get_filepath() {
 	local file="$(get_filename "$1")"
 
-	if [[ -f "$startdir/$file" ]]; then
+	if [[ -e "$startdir/$file" ]]; then
 		file="$startdir/$file"
-	elif [[ -f "$SRCDEST/$file" ]]; then
+	elif [[ -e "$SRCDEST/$file" ]]; then
 		file="$SRCDEST/$file"
 	else
 		return 1
@@ -210,14 +212,25 @@ missing_source_file() {
 get_filename() {
 	# if a filename is specified, use it
 	local filename="${1%%::*}"
+	filename="${filename#*@@}"
 	# if it is just an URL, we only keep the last component
 	echo "${filename##*/}"
 }
 
 # extract the URL from a source entry
 get_url() {
-	# strip an eventual filename
-	echo "${1#*::}"
+	# strip scm and filename
+	local url="${1#*@@}"
+	echo "${url#*::}"
+}
+
+get_scm() {
+	# strip off everything except scm.
+	if [[ "$1" == "${1%%@@*}" ]]; then
+		echo ""
+	else
+		echo "${1%%@@*}"
+	fi
 }
 
 ##
@@ -302,8 +315,15 @@ in_array() {
 
 get_downloadclient() {
 	# $1 = URL with valid protocol prefix
-	local url=$1
-	local proto="${url%%://*}"
+	local url="$1"
+	local scm="$2"
+	local proto
+
+	if [[ -n scm ]]; then
+		proto="$scm"
+	else
+		proto="${url%%://*}"
+	fi
 
 	# loop through DOWNLOAD_AGENTS variable looking for protocol
 	local i
@@ -391,8 +411,8 @@ check_deps() {
 	local ret=0
 	local pmout
 	pmout=$(run_pacman -T "$@") || ret=$?
-	set -E 
-	
+	set -E
+
 	if (( ret == 127 )); then #unresolved deps
 		echo "$pmout"
 	elif (( ret )); then
@@ -484,21 +504,26 @@ remove_deps() {
 
 download_sources() {
 	msg "$(gettext "Retrieving Sources...")"
-
 	pushd "$SRCDEST" &>/dev/null
 
 	local netfile
 	for netfile in "${source[@]}"; do
-		local file
-		if file=$(get_filepath "$netfile"); then
-			msg2 "$(gettext "Found %s")" "${file##*/}"
-			ln -sf "$file" "$srcdir/"
+		local file=$(get_filename "$netfile")
+		local url=$(get_url "$netfile")
+		local scm=$(get_scm "$netfile")
+		local filepath
+
+		if filepath=$(get_filepath "$file"); then
+			msg2 "$(gettext "Found %s")" "${file}"
+			ln -sf "$filepath" "$srcdir/"
+		  if [[ -n "$scm" ]] && [[ -z $FORCE_VER ]] && ! (( HOLDVER )); then
+				msg2 "$(gettext "Updating %s to latest revision...")" "$file"
+				devel_update "$scm" "$file" "$url"
+				pkgver_update
+			fi
 			continue
 		fi
 
-		file=$(get_filename "$netfile")
-		local url=$(get_url "$netfile")
-
 		# if we get here, check to make sure it was a URL, else fail
 		if [[ $file = $url ]]; then
 			error "$(gettext "%s was not found in the build directory and is not a URL.")" "$file"
@@ -506,7 +531,7 @@ download_sources() {
 		fi
 
 		# find the client we should use for this URL
-		local dlclient=$(get_downloadclient "$url") || exit $?
+		local dlclient=$(get_downloadclient "$url" "$scm") || exit $?
 
 		msg2 "$(gettext "Downloading %s...")" "$file"
 		# fix flyspray bug #3289
@@ -581,14 +606,17 @@ generate_checksums() {
 		local netfile
 		for netfile in "${source[@]}"; do
 			local file="$(get_filepath "$netfile")" || missing_source_file "$netfile"
-			local sum="$(openssl dgst -${integ} "$file")"
-			sum=${sum##* }
-			(( ct )) && echo -n "$indent"
-			echo -n "'$sum'"
-			ct=$(($ct+1))
-			(( $ct < $numsrc )) && echo
+			if [[ -d "$file" ]]; then
+				sum="scm"
+			else
+				local sum="$(openssl dgst -${integ} "$file")"
+				sum=${sum##* }
+			fi
+			  (( ct )) && echo -n "$indent"
+				echo -n "'$sum'"
+				ct=$(($ct+1))
+				(( $ct < $numsrc )) && echo
 		done
-
 		echo ")"
 	done
 }
@@ -623,11 +651,14 @@ check_checksums() {
 				fi
 
 				if (( $found )) ; then
+					local realsum
 					local expectedsum=$(tr '[:upper:]' '[:lower:]' <<< "${integrity_sums[$idx]}")
-					local realsum="$(openssl dgst -${integ} "$file")"
+					[[ -d $file ]] || realsum="$(openssl dgst -${integ} "$file")"
 					realsum="${realsum##* }"
 					if [[ $expectedsum = $realsum ]]; then
 						echo "$(gettext "Passed")" >&2
+					elif [[ $expectedsum = "scm" ]]; then
+						echo "$(gettext "SCM Directory")" >&2
 					else
 						echo "$(gettext "FAILED")" >&2
 						errors=1
@@ -1288,70 +1319,26 @@ check_sanity() {
 	return 0
 }
 
-devel_check() {
-	newpkgver=""
-
-	# Do not update pkgver if --holdver is set, when building a source package, repackaging,
-	# reading PKGBUILD from pipe (-f), or if we cannot write to the file (-w)
-	if (( HOLDVER || SOURCEONLY || REPKG )) \
-		            || [[ ! -f $BUILDFILE || ! -w $BUILDFILE ]]; then
-		return
-	fi
-
-	if [[ -z $FORCE_VER ]]; then
-		# Check if this is a svn/cvs/etc PKGBUILD; set $newpkgver if so.
-		# This will only be used on the first call to makepkg; subsequent
-		# calls to makepkg via fakeroot will explicitly pass the version
-		# number to avoid having to determine the version number twice.
-		# Also do a brief check to make sure we have the VCS tool available.
-		oldpkgver=$pkgver
-		if [[ -n ${_darcstrunk} && -n ${_darcsmod} ]] ; then
-			type -p darcs >/dev/null || return 0
-			msg "$(gettext "Determining latest darcs revision...")"
-			newpkgver=$(date +%Y%m%d)
-		elif [[ -n ${_cvsroot} && -n ${_cvsmod} ]] ; then
-			type -p cvs >/dev/null || return 0
-			msg "$(gettext "Determining latest cvs revision...")"
-			newpkgver=$(date +%Y%m%d)
-		elif [[ -n ${_gitroot} && -n ${_gitname} ]] ; then
-			type -p git >/dev/null || return 0
-			msg "$(gettext "Determining latest git revision...")"
-			newpkgver=$(date +%Y%m%d)
-		elif [[ -n ${_svntrunk} && -n ${_svnmod} ]] ; then
-			type -p svn >/dev/null || return 0
-			msg "$(gettext "Determining latest svn revision...")"
-			newpkgver=$(LC_ALL=C svn info $_svntrunk | sed -n 's/^Last Changed Rev: \([0-9]*\)$/\1/p')
-		elif [[ -n ${_bzrtrunk} && -n ${_bzrmod} ]] ; then
-			type -p bzr >/dev/null || return 0
-			msg "$(gettext "Determining latest bzr revision...")"
-			newpkgver=$(bzr revno ${_bzrtrunk})
-		elif [[ -n ${_hgroot} && -n ${_hgrepo} ]] ; then
-			type -p hg >/dev/null || return 0
-			msg "$(gettext "Determining latest hg revision...")"
-			if [[ -d ./src/$_hgrepo ]] ; then
-				cd ./src/$_hgrepo
-				hg pull
-				hg update
-			else
-				[[ ! -d ./src/ ]] && mkdir ./src/
-				hg clone $_hgroot/$_hgrepo ./src/$_hgrepo
-				cd ./src/$_hgrepo
-			fi
-			newpkgver=$(hg tip --template "{rev}")
-			cd ../../
-		fi
-
-		if [[ -n $newpkgver ]]; then
-			msg2 "$(gettext "Version found: %s")" "$newpkgver"
-		fi
-
-	else
-		# Version number retrieved from fakeroot->makepkg argument
-		newpkgver=$FORCE_VER
-	fi
+##
+# 1 = scm
+# 2 = filename
+# 3 = url
+##
+devel_update() {
+	pushd "$srcdir/$2" &>/dev/null
+	case $1 in
+		darcs) darcs pull -a $3;;
+		cvs)   cvs -z3 update -d;;
+		git)   git pull origin;;
+		svn)   svn up -r $pkgver;;
+		bzr)   ;;
+		hg) hg pull; hg update ;;
+		*) error "$(gettext "Uknown SCM %s listed in source array.")" "$1"
+	esac
+	popd &>/dev/null
 }
 
-devel_update() {
+pkgver_update() {
 	# This is lame, but if we're wanting to use an updated pkgver for
 	# retrieving svn/cvs/etc sources, we'll update the PKGBUILD with
 	# the new pkgver and then re-source it. This is the most robust
@@ -1361,12 +1348,43 @@ devel_update() {
 	#  ...
 	#  _foo=pkgver
 	#
+	local file
+	for file in "${source[@]}"; do
+		local scm=$(get_scm "$file")
+		if [[ -n "$scm" ]]; then
+			msg2 "$(gettext "Updating PKGBUILD pkgver...")"
+			newpkgver=""
+			oldpkgver=$pkgver
+
+			pushd "$srcdir/$(get_filename "$file")" &>/dev/null
+			case "$scm" in
+				darcs) newpkgver=$(date +%Y%m%d) ;;
+				cvs)   newpkgver=$(date +%Y%m%d) ;;
+				git)   newpkgver=$(date +%Y%m%d) ;;
+				svn)   newpkgver=$(LC_ALL=C svn info "$srcdir/$2" | sed -n 's/^Last Changed Rev: \([0-9]*\)$/\1/p') ;;
+				bzr)   newpkgver=$(bzr revno "$srcdir/$2") ;;
+				hg)    newpkgver=$(hg tip --template "{rev}") ;;
+				*)     error "$(gettext "Uknown SCM %s listed in source array.")" "$1"; continue ;;
+			esac
+			popd &>/dev/null
+		fi
+	done
+
+	if [[ -n $newpkgver ]]; then
+		msg2 "$(gettext "Version found: %s")" "$newpkgver"
+	else
+		# Version number retrieved from fakeroot->makepkg argument
+		newpkgver=$FORCE_VER
+	fi
+
 	if [[ -n $newpkgver ]]; then
 		if [[ $newpkgver != $pkgver ]]; then
 			if [[ -f $BUILDFILE && -w $BUILDFILE ]]; then
 				@SEDINPLACE@ "s/^pkgver=[^ ]*/pkgver=$newpkgver/" "$BUILDFILE"
 				@SEDINPLACE@ "s/^pkgrel=[^ ]*/pkgrel=1/" "$BUILDFILE"
 				source "$BUILDFILE"
+			else
+			 	error "$(gettext "Could not update pkgver in PKGBUILD to %i")" "$newpkgver"
 			fi
 		fi
 	fi
@@ -1793,13 +1811,6 @@ fi
 # check the PKGBUILD for some basic requirements
 check_sanity || exit 1
 
-# We need to run devel_update regardless of whether we are in the fakeroot
-# build process so that if the user runs makepkg --forcever manually, we
-# 1) output the correct pkgver, and 2) use the correct filename when
-# checking if the package file already exists - fixes FS #9194
-devel_check
-devel_update
-
 if (( ${#pkgname[@]} > 1 )); then
 	SPLITPKG=1
 fi
@@ -1984,7 +1995,6 @@ else
 	# if we are root or if fakeroot is not enabled, then we don't use it
 	if [[ $(check_buildenv fakeroot) != "y" ]] || (( EUID == 0 )); then
 		if (( ! REPKG )); then
-			devel_update
 			(( BUILDFUNC )) && run_build
 		fi
 		if (( ! SPLITPKG )); then
@@ -2005,7 +2015,6 @@ else
 		fi
 	else
 		if (( ! REPKG && ( PKGFUNC || SPLITPKG ) )); then
-			devel_update
 			(( BUILDFUNC )) && run_build
 			cd "$startdir"
 		fi
-- 
1.7.3.1

    

[pacman-dev] [PATCH] Make makepkg handle SCMs better.

David Campbell