[pacman-dev] [PATCH 1/3] bacman: allow for parallel packaging
Gordian Edenhofer
gordian.edenhofer at gmail.com
Sun Aug 14 19:15:17 UTC 2016
On Sun, 2016-08-14 at 14:23 -0400, Dave Reisner wrote:
> On Sun, Aug 14, 2016 at 08:10:52PM +0200, Gordian Edenhofer wrote:
> >
> > * move the actual assembly process into its own function
> > * allow for packaging multiple packages with one command
> > * handle SIGHUP SIGINT SIGTERM and remove working dirs accordingly
> > * add some comments
>
> Can't this be done already by running multiple instances of bacman?
> In
> other words, what's gained by adding this complexity to bacman
> itself?
Yes, sure this basically achieves the same purpose. Though I would
argue the added complexity is negligible since it merely puts the
assembly process in its own function, nothing more. The git-diff may
suggest something different but this is mainly due to the changed
indentation.
> Bonus, if you're on a system that has bash 4.3, you can do better
> than
> the parallelize function you wrote and avoiding a tight loop and the
> CPU
> it'll eat up:
>
> pbacman() {
> local max_jobs=$1; shift
>
> for arg; do
> while $(( $(jobs -p | wc -l) > max_jobs )); do
> wait -n
> done
>
> bacman "$arg" &
> done
> wait
> }
>
> We can't use this in bacman without bumping the bash version
> requirement
> which probably isn't something the pacman maintainers are willing to
> do.
I am aware this probably isn't an elegant way to do things but it is
simple and gets the job done. The CPU usage can be drastically
decreased by adding a short sleep in between the checks. But I guess
there is still room for improvement.
> >
> >
> > Signed-off-by: Gordian Edenhofer <gordian.edenhofer at gmail.com>
> > ---
> > contrib/bacman.sh.in | 460 ++++++++++++++++++++++++++-------------
> > ------------
> > 1 file changed, 238 insertions(+), 222 deletions(-)
> > mode change 100644 => 100755 contrib/bacman.sh.in
> >
> > diff --git a/contrib/bacman.sh.in b/contrib/bacman.sh.in
> > old mode 100644
> > new mode 100755
> > index a611c1a..4cd78e4
> > --- a/contrib/bacman.sh.in
> > +++ b/contrib/bacman.sh.in
> > @@ -33,9 +33,18 @@ ARGS=("$@")
> >
> > m4_include(../scripts/library/output_format.sh)
> >
> > -#
> > -# User Friendliness
> > -#
> > +# Lazy recursive clean up of temporary dirs
> > +work_dir_root="${TMPDIR:-/tmp}/bacman"
> > +clean_up() {
> > + rm -r "${work_dir_root}".*
> > + echo
> > + exit
> > +}
> > +
> > +# Trap termination signals
> > +trap clean_up SIGHUP SIGINT SIGTERM
> > +
> > +# Print usage information
> > usage() {
> > echo "${myname} (pacman) v${myver}"
> > echo
> > @@ -46,12 +55,14 @@ usage() {
> > echo "Example: ${myname} linux-headers"
> > }
> >
> > +# Print version information
> > version() {
> > printf "%s %s\n" "$myname" "$myver"
> > echo 'Copyright (C) 2008 locci
> > <carlocci_at_gmail_dot_com>'
> > echo 'Copyright (C) 2008-2016 Pacman Development Team <pac
> > man-dev at archlinux.org>'
> > }
> >
> > +# Check for specified arguments
> > while [[ ! -z $1 ]]; do
> > if [[ $1 == "--nocolor" ]]; then
> > USE_COLOR='n'
> > @@ -64,13 +75,16 @@ while [[ ! -z $1 ]]; do
> > fi
> > done
> >
> > +# Configure colored output
> > m4_include(../scripts/library/term_colors.sh)
> >
> > -if (( $# != 1 )); then
> > +# Break if no argument was given
> > +if (( $# < 1 )); then
> > usage
> > exit 1
> > fi
> >
> > +# Print usage or version if requested
> > if [[ $1 = -@(h|-help) ]]; then
> > usage
> > exit 0
> > @@ -79,9 +93,7 @@ elif [[ $1 = -@(V|-version) ]]; then
> > exit 0
> > fi
> >
> > -#
> > -# Fakeroot support
> > -#
> > +# Run in fakeroot if EUID is not root
> > if (( EUID )); then
> > if [[ -f /usr/bin/fakeroot ]]; then
> > msg "Entering fakeroot environment"
> > @@ -94,264 +106,268 @@ if (( EUID )); then
> > fi
> > fi
> >
> > -#
> > -# Setting environmental variables
> > -#
> > +# Source environmental variables and specify fallbacks
> > if [[ ! -r @sysconfdir@/pacman.conf ]]; then
> > error "unable to read @sysconfdir@/pacman.conf"
> > exit 1
> > fi
> > -
> > eval $(awk '/DBPath/ {print $1$2$3}' @sysconfdir@/pacman.conf)
> > pac_db="${DBPath:- at localstatedir@/lib/pacman/}/local"
> > -
> > if [[ ! -r @sysconfdir@/makepkg.conf ]]; then
> > error "unable to read @sysconfdir@/makepkg.conf"
> > exit 1
> > fi
> > -
> > source "@sysconfdir@/makepkg.conf"
> > if [[ -r ~/.makepkg.conf ]]; then
> > source ~/.makepkg.conf
> > fi
> > -
> > pkg_dest="${PKGDEST:-$PWD}"
> > pkg_pkger=${PACKAGER:-'Unknown Packager'}
> >
> > -pkg_name="$1"
> > -pkg_dir=("$pac_db/$pkg_name"-+([^-])-+([^-]))
> > -pkg_namver=("${pkg_dir[@]##*/}")
> > -
> > -#
> > -# Checks everything is in place
> > -#
> > +# Check for an existing database
> > if [[ ! -d $pac_db ]]; then
> > error "pacman database directory ${pac_db} not found"
> > exit 1
> > fi
> >
> > -if (( ${#pkg_dir[@]} != 1 )); then
> > - error "%d entries for package %s found in pacman database"
> > \
> > - ${#pkg_dir[@]} "${pkg_name}"
> > - msg2 "%s" "${pkg_dir[@]}"
> > - exit 1
> > -fi
> > -
> > -if [[ ! -d $pkg_dir ]]; then
> > - error "package %s is found in pacman database,"
> > "${pkg_name}"
> > - plain " but '%s' is not a directory" "${pkg_dir}"
> > - exit 1
> > -fi
> > -
> > -#
> > -# Begin
> > -#
> > -msg "Package: ${pkg_namver}"
> > -work_dir=$(mktemp -d "${TMPDIR:-/tmp}/bacman.XXXXXXXXXX")
> > -cd "$work_dir" || exit 1
> > -
> > -#
> > -# File copying
> > -#
> > -msg2 "Copying package files..."
> > -
> > -while read i; do
> > - if [[ -z $i ]]; then
> > - continue
> > +# Assemble a single package: $1 = pkgname
> > +fakebuild() {
> > + pkg_name="$1"
> > + pkg_dir=("$pac_db/$pkg_name"-+([^-])-+([^-]))
> > + pkg_namver=("${pkg_dir[@]##*/}")
> > +
> > + # Checks database for specified package
> > + if (( ${#pkg_dir[@]} != 1 )); then
> > + error "%d entries for package %s found in pacman
> > database" \
> > + ${#pkg_dir[@]} "${pkg_name}"
> > + msg2 "%s" "${pkg_dir[@]}"
> > + exit 1
> > fi
> > -
> > - if [[ $i == %+([A-Z])% ]]; then
> > - current=$i
> > - continue
> > + if [[ ! -d $pkg_dir ]]; then
> > + error "package %s is found in pacman database,"
> > "${pkg_name}"
> > + plain " but '%s' is not a directory"
> > "${pkg_dir}"
> > + exit 1
> > fi
> >
> > - case "$current" in
> > - %FILES%)
> > - local_file="/$i"
> > - package_file="$work_dir/$i"
> > + # Create working directory
> > + msg "Package: ${pkg_namver}"
> > + work_dir=$(mktemp -d "${work_dir_root}.XXXXXXXXXX")
> > + cd "$work_dir" || exit 1
> >
> > - if [[ ! -e $local_file ]]; then
> > - warning "package file $local_file
> > is missing"
> > - continue
> > - fi
> > - ;;
> > -
> > - %BACKUP%)
> > - # Get the MD5 checksum.
> > - original_md5="${i##*$'\t'}"
> > - # Strip the md5sum after the tab.
> > - i="${i%$'\t'*}"
> > - local_file="/$i.pacnew"
> > - package_file="$work_dir/$i"
> > -
> > - # Include unmodified .pacnew files.
> > - local_md5="$(md5sum "$local_file" | cut
> > -d' ' -f1)"
> > - if [[ $INCLUDE_PACNEW == 'n' ]] \
> > - || [[ ! -e $local_file ]] \
> > - || [[ $local_md5 != $original_md5 ]]; then
> > - # Warn about modified files.
> > - local_md5="$(md5sum "/$i" | cut
> > -d' ' -f1)"
> > - if [[ $local_md5 != $original_md5
> > ]]; then
> > - warning "package file /$i
> > has been modified"
> > - fi
> > - # Let the normal file be included
> > in the %FILES% list.
> > - continue
> > - fi
> > - ;;
> > + # Assemble list of files which belong to the package and
> > tar them
> > + msg2 "Copying package files..."
> > + while read i; do
> > + if [[ -z $i ]]; then
> > + continue
> > + fi
> >
> > - *)
> > + if [[ $i == %+([A-Z])% ]]; then
> > + current=$i
> > continue
> > - ;;
> > - esac
> > + fi
> >
> > - ret=0
> > - bsdtar -cnf - -s'/.pacnew$//' "$local_file" 2> /dev/null |
> > bsdtar -xpf - 2> /dev/null
> > + case "$current" in
> > + %FILES%)
> > + local_file="/$i"
> > + package_file="$work_dir/$i"
> > +
> > + if [[ ! -e $local_file ]]; then
> > + warning "package file
> > $local_file is missing"
> > + continue
> > + fi
> > + ;;
> > +
> > + %BACKUP%)
> > + # Get the MD5 checksum.
> > + original_md5="${i##*$'\t'}"
> > + # Strip the md5sum after the tab.
> > + i="${i%$'\t'*}"
> > + local_file="/$i.pacnew"
> > + package_file="$work_dir/$i"
> > +
> > + # Include unmodified .pacnew
> > files.
> > + local_md5="$(md5sum "$local_file"
> > | cut -d' ' -f1)"
> > + if [[ $INCLUDE_PACNEW == 'n' ]] \
> > + || [[ ! -e $local_file ]] \
> > + || [[ $local_md5 != $original_md5
> > ]]; then
> > + # Warn about modified
> > files.
> > + local_md5="$(md5sum "/$i"
> > | cut -d' ' -f1)"
> > + if [[ $local_md5 !=
> > $original_md5 ]]; then
> > + warning "package
> > file /$i has been modified"
> > + fi
> > + # Let the normal file be
> > included in the %FILES% list.
> > + continue
> > + fi
> > + ;;
> >
> > - # Workaround to bsdtar not reporting a missing file as an
> > error
> > - if ! [[ -e $package_file || -L $package_file ]]; then
> > - error "unable to add $local_file to the package"
> > - plain " If your user does not have
> > permission to read this file, then"
> > - plain " you will need to run $myname as
> > root."
> > + *)
> > + continue
> > + ;;
> > + esac
> > +
> > + # Tar files
> > + ret=0
> > + bsdtar -cnf - -s'/.pacnew$//' "$local_file" 2>
> > /dev/null | bsdtar -xpf - 2> /dev/null
> > + # Workaround to bsdtar not reporting a missing
> > file as an error
> > + if ! [[ -e $package_file || -L $package_file ]];
> > then
> > + error "unable to add $local_file to the
> > package"
> > + plain " If your user does not have
> > permission to read this file, then"
> > + plain " you will need to run $myname
> > as root."
> > + rm -rf "$work_dir"
> > + exit 1
> > + fi
> > + done < "$pkg_dir"/files
> > +
> > + ret=$?
> > + if (( ret )); then
> > rm -rf "$work_dir"
> > exit 1
> > fi
> > -done < "$pkg_dir"/files
> >
> > -ret=$?
> > -if (( ret )); then
> > - rm -rf "$work_dir"
> > - exit 1
> > -fi
> > + # Calculate package size
> > + pkg_size=$(du -sk | awk '{print $1 * 1024}')
> >
> > -pkg_size=$(du -sk | awk '{print $1 * 1024}')
> > -
> > -#
> > -# .PKGINFO stuff
> > -# TODO adopt makepkg's write_pkginfo() into this or
> > scripts/library
> > -#
> > -msg2 "Generating .PKGINFO metadata..."
> > -echo "# Generated by $myname $myver" > .PKGINFO
> > -if [[ $INFAKEROOT == "1" ]]; then
> > - echo "# Using $(fakeroot -v)" >> .PKGINFO
> > -fi
> > -echo "# $(LC_ALL=C date)" >> .PKGINFO
> > -echo "#" >> .PKGINFO
> > -
> > -while read i; do
> > - if [[ -z $i ]]; then
> > - continue;
> > + # Reconstruct .PKGINFO from database
> > + # TODO adopt makepkg's write_pkginfo() into this or
> > scripts/library
> > + msg2 "Generating .PKGINFO metadata..."
> > + echo "# Generated by $myname $myver" > .PKGINFO
> > + if [[ $INFAKEROOT == "1" ]]; then
> > + echo "# Using $(fakeroot -v)" >> .PKGINFO
> > fi
> > -
> > - if [[ $i == %+([A-Z])% ]]; then
> > - current=$i
> > - continue
> > + echo "# $(LC_ALL=C date)" >> .PKGINFO
> > + echo "#" >> .PKGINFO
> > + while read i; do
> > + if [[ -z $i ]]; then
> > + continue;
> > + fi
> > + if [[ $i == %+([A-Z])% ]]; then
> > + current=$i
> > + continue
> > + fi
> > +
> > + case "$current" in
> > + # desc
> > + %NAME%)
> > + echo "pkgname = $i" >> .PKGINFO
> > + ;;
> > + %VERSION%)
> > + echo "pkgver = $i" >> .PKGINFO
> > + ;;
> > + %DESC%)
> > + echo "pkgdesc = $i" >> .PKGINFO
> > + ;;
> > + %URL%)
> > + echo "url = $i" >> .PKGINFO
> > + ;;
> > + %LICENSE%)
> > + echo "license = $i" >> .PKGINFO
> > + ;;
> > + %ARCH%)
> > + echo "arch = $i" >> .PKGINFO
> > + pkg_arch="$i"
> > + ;;
> > + %BUILDDATE%)
> > + echo "builddate = $(date -u
> > "+%s")" >> .PKGINFO
> > + ;;
> > + %PACKAGER%)
> > + echo "packager =
> > $pkg_pkger" >> .PKGINFO
> > + ;;
> > + %SIZE%)
> > + echo "size = $pkg_size" >>
> > .PKGINFO
> > + ;;
> > + %GROUPS%)
> > + echo "group = $i" >> .PKGINFO
> > + ;;
> > + %REPLACES%)
> > + echo "replaces = $i" >>
> > .PKGINFO
> > + ;;
> > + %DEPENDS%)
> > + echo "depend = $i" >> .PKGINFO
> > + ;;
> > + %OPTDEPENDS%)
> > + echo "optdepend = $i" >> .PKGINFO
> > + ;;
> > + %CONFLICTS%)
> > + echo "conflict = $i" >> .PKGINFO
> > + ;;
> > + %PROVIDES%)
> > + echo "provides = $i" >> .PKGINFO
> > + ;;
> > + %BACKUP%)
> > + # Strip the md5sum after the tab
> > + echo "backup = ${i%%$'\t'*}" >>
> > .PKGINFO
> > + ;;
> > + esac
> > + done < <(cat "$pkg_dir"/{desc,files})
> > +
> > + comp_files=".PKGINFO"
> > +
> > + # Add instal file if present
> > + if [[ -f $pkg_dir/install ]]; then
> > + cp "$pkg_dir/install" "$work_dir/.INSTALL"
> > + comp_files+=" .INSTALL"
> > + fi
> > + if [[ -f $pkg_dir/changelog ]]; then
> > + cp "$pkg_dir/changelog" "$work_dir/.CHANGELOG"
> > + comp_files+=" .CHANGELOG"
> > fi
> >
> > - case "$current" in
> > - # desc
> > - %NAME%)
> > - echo "pkgname = $i" >> .PKGINFO
> > - ;;
> > - %VERSION%)
> > - echo "pkgver = $i" >> .PKGINFO
> > - ;;
> > - %DESC%)
> > - echo "pkgdesc = $i" >> .PKGINFO
> > - ;;
> > - %URL%)
> > - echo "url = $i" >> .PKGINFO
> > - ;;
> > - %LICENSE%)
> > - echo "license = $i" >> .PKGINFO
> > - ;;
> > - %ARCH%)
> > - echo "arch = $i" >> .PKGINFO
> > - pkg_arch="$i"
> > - ;;
> > - %BUILDDATE%)
> > - echo "builddate = $(date -u "+%s")" >>
> > .PKGINFO
> > - ;;
> > - %PACKAGER%)
> > - echo "packager = $pkg_pkger" >>
> > .PKGINFO
> > - ;;
> > - %SIZE%)
> > - echo "size = $pkg_size" >> .PKGINFO
> > - ;;
> > - %GROUPS%)
> > - echo "group = $i" >> .PKGINFO
> > - ;;
> > - %REPLACES%)
> > - echo "replaces = $i" >> .PKGINFO
> > - ;;
> > - %DEPENDS%)
> > - echo "depend = $i" >> .PKGINFO
> > - ;;
> > - %OPTDEPENDS%)
> > - echo "optdepend = $i" >> .PKGINFO
> > - ;;
> > - %CONFLICTS%)
> > - echo "conflict = $i" >> .PKGINFO
> > - ;;
> > - %PROVIDES%)
> > - echo "provides = $i" >> .PKGINFO
> > - ;;
> > -
> > - # files
> > - %BACKUP%)
> > - # Strip the md5sum after the tab
> > - echo "backup = ${i%%$'\t'*}" >> .PKGINFO
> > - ;;
> > - esac
> > -done < <(cat "$pkg_dir"/{desc,files})
> > -
> > -comp_files=".PKGINFO"
> > -
> > -if [[ -f $pkg_dir/install ]]; then
> > - cp "$pkg_dir/install" "$work_dir/.INSTALL"
> > - comp_files+=" .INSTALL"
> > -fi
> > -if [[ -f $pkg_dir/changelog ]]; then
> > - cp "$pkg_dir/changelog" "$work_dir/.CHANGELOG"
> > - comp_files+=" .CHANGELOG"
> > -fi
> > + # Fixes owner:group and permissions for .PKGINFO,
> > .CHANGELOG, .INSTALL
> > + chown root:root "$work_dir"/{.PKGINFO,.CHANGELOG,.INSTALL}
> > 2> /dev/null
> > + chmod 644 "$work_dir"/{.PKGINFO,.CHANGELOG,.INSTALL} 2>
> > /dev/null
> >
> > -#
> > -# Fixes owner:group and permissions for .PKGINFO, .CHANGELOG,
> > .INSTALL
> > -#
> > -chown root:root "$work_dir"/{.PKGINFO,.CHANGELOG,.INSTALL} 2>
> > /dev/null
> > -chmod 644 "$work_dir"/{.PKGINFO,.CHANGELOG,.INSTALL} 2> /dev/null
> > + # Generate the package
> > + msg2 "Generating the package..."
> >
> > -#
> > -# Generate the package
> > -#
> > -msg2 "Generating the package..."
> > -
> > -pkg_file="$pkg_dest/$pkg_namver-$pkg_arch${PKGEXT}"
> > -ret=0
> > -
> > -# TODO: Maybe this can be set globally for robustness
> > -shopt -s -o pipefail
> > -bsdtar -cf - $comp_files * |
> > -case "$PKGEXT" in
> > - *tar.gz) gzip -c -f -n ;;
> > - *tar.bz2) bzip2 -c -f ;;
> > - *tar.xz) xz -c -z - ;;
> > - *tar.Z) compress -c -f ;;
> > - *tar) cat ;;
> > - *) warning "'%s' is not a valid archive extension." \
> > - "$PKGEXT"; cat ;;
> > -esac > "${pkg_file}"; ret=$?
> > -
> > -if (( ret )); then
> > - error "Unable to write package to $pkg_dest"
> > - plain " Maybe the disk is full or you do not have
> > write access"
> > + pkg_file="$pkg_dest/$pkg_namver-$pkg_arch${PKGEXT}"
> > + ret=0
> > +
> > + # TODO: Maybe this can be set globally for robustness
> > + shopt -s -o pipefail
> > + bsdtar -cf - $comp_files * |
> > + case "$PKGEXT" in
> > + *tar.gz) gzip -c -f -n ;;
> > + *tar.bz2) bzip2 -c -f ;;
> > + *tar.xz) xz -c -z - ;;
> > + *tar.Z) compress -c -f ;;
> > + *tar) cat ;;
> > + *) warning "'%s' is not a valid archive
> > extension." \
> > + "$PKGEXT"; cat ;;
> > + esac > "${pkg_file}"; ret=$?
> > +
> > + # Move compressed package to destination
> > + if (( ret )); then
> > + error "Unable to write package to $pkg_dest"
> > + plain " Maybe the disk is full or you do not
> > have write access"
> > + rm -rf "$work_dir"
> > + exit 1
> > + fi
> > +
> > + # Clean up working directory
> > rm -rf "$work_dir"
> > - exit 1
> > -fi
> > + msg "Done."
> > +}
> >
> > -rm -rf "$work_dir"
> > +# Run fakebuild in parralel with at maximum $MAX_JOBS jobs
> > +# By default only run one job
> > +MAX_JOBS=${MAX_JOBS:-1}
> > +parallelize() {
> > + while [[ $# -gt 0 ]] ; do
> > + job_count=($(jobs -p))
> > + if [[ ${#job_count[@]} -lt $MAX_JOBS ]] ; then
> > + fakebuild "$1" &
> > + shift
> > + fi
> > + done
> > + wait
> > +}
> >
> > -msg "Done."
> > +# Initiate assembly function
> > +if [[ $MAX_JOBS -gt "1" ]]; then
> > + parallelize "$@"
> > +else
> > + for PKG in $@; do fakebuild $PKG; done
> > +fi
> >
> > exit 0
> >
> > --
> > 2.9.2
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 819 bytes
Desc: This is a digitally signed message part
URL: <https://lists.archlinux.org/pipermail/pacman-dev/attachments/20160814/d4e7de25/attachment-0001.asc>
More information about the pacman-dev
mailing list