[arch-dev-public] [PATCH 1/3] sourceballs: Make cleanup more efficient
Moved all cleanup related code in sourceballs-cleanup script and moved, now common, functions in db-functions. The cleanup script is now ran after all the new sourceballs have been fetched. Signed-off-by: Eric Bélanger <snowmaniscool@gmail.com> --- cron-jobs/sourceballs | 2 + db-functions | 21 ++++++++ misc-scripts/make-sourceball | 54 --------------------- misc-scripts/sourceballs-cleanup | 95 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 54 deletions(-) create mode 100755 misc-scripts/sourceballs-cleanup diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b7a4885..ccb00ad 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -89,4 +89,6 @@ if [ -n "$FAILED_PKGS" ]; then echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "$srcbase/failed.txt" fi +$dirname/../misc-scripts/sourceballs-cleanup + cleanup diff --git a/db-functions b/db-functions index a4cffd7..ecf53d9 100644 --- a/db-functions +++ b/db-functions @@ -126,4 +126,25 @@ chk_license() { return 1 } +pkgname_from_src() { + local tmp + tmp=${1##*/} + tmp=${tmp%$SRCEXT} + for a in ${ARCHES[@]}; do + tmp=${tmp%-$a} + done + tmp=${tmp%-any} + echo ${tmp%-*-*} +} + +pkgver_from_src() { + tmp=${1##*/} + tmp=${tmp%$SRCEXT} + for a in ${ARCHES[@]}; do + tmp=${tmp%-$a} + done + tmp=${tmp%-any} + echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g' +} + # vim: set ts=4 sw=4 noet ft=sh: diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 8899fca..610d7f1 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -40,27 +40,6 @@ die() { cleanup 1 } -pkgname_from_src() { - local tmp - tmp=${1##*/} - tmp=${tmp%$SRCEXT} - for a in ${ARCHES[@]}; do - tmp=${tmp%-$a} - done - tmp=${tmp%-any} - echo ${tmp%-*-*} -} - -pkgver_from_src() { - tmp=${1##*/} - tmp=${tmp%$SRCEXT} - for a in ${ARCHES[@]}; do - tmp=${tmp%-$a} - done - tmp=${tmp%-any} - echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g' -} - create_srcpackage() { if [ -d "$1" ]; then pushd "$1" >/dev/null @@ -94,38 +73,6 @@ create_srcpackage() { fi } -remove_old() { - if [ -d "$1" ]; then - pushd "$1" >/dev/null - PKGVERS="" - for repo in *; do - cd "$repo" - . "$BUILDSCRIPT" - PKGVERS="$PKGVERS $pkgver-$pkgrel" - cd .. - done - - for srcpkg in "$srcpath/$packagename-"*; do - [ -f "$srcpkg" ] || continue - if [ "$(pkgname_from_src $srcpkg)" == "$packagename" ]; then - skip=0 - pver="$(pkgver_from_src $srcpkg)" - for v in $PKGVERS; do - if [ "$v" = "$pver" ]; then - skip=1 - break - fi - done - if [ $skip -ne 1 ]; then - mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR - fi - fi - done - - popd >/dev/null - fi -} - trap ctrl_c 2 trap cleanup 0 1 @@ -135,7 +82,6 @@ set_umask cd "$WORKDIR" if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then - remove_old "$packagename/repos/" create_srcpackage "$packagename/repos/$reponame-$_arch" else die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup new file mode 100755 index 0000000..0a1ac4d --- /dev/null +++ b/misc-scripts/sourceballs-cleanup @@ -0,0 +1,95 @@ +#!/bin/bash + +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +srcpath="$FTP_BASE/sources/" +logfile="$srcpath/cleanup.txt" + +LOCKFILE="/tmp/.sourceball-cleanup.lock" +WORKDIR="/tmp/sourceball-cleanup.$packagename.$UID" + +cleanup () { + restore_umask + rm -rf "$WORKDIR" + rm -f "$LOCKFILE" + exit 0 +} + +ctrl_c() { + echo "Interrupted" >&2 + cleanup 0 +} + +remove_old() { + if [ -d "$1" ]; then + pushd "$1" >/dev/null + PKGVERS="" + for repo in *; do + cd "$repo" + . "$BUILDSCRIPT" + PKGVERS="$PKGVERS $pkgver-$pkgrel" + cd .. + done + + for srcpkg in "$srcpath/$packagename-"*; do + [ -f "$srcpkg" ] || continue + if [ "$(pkgname_from_src $srcpkg)" == "$packagename" ]; then + skip=0 + pver="$(pkgver_from_src $srcpkg)" + for v in $PKGVERS; do + if [ "$v" = "$pver" ]; then + skip=1 + break + fi + done + if [ $skip -ne 1 ]; then + mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR + fi + fi + done + + popd >/dev/null + fi +} + +if [ -f "$LOCKFILE" ]; then + owner="$(/usr/bin/stat -c %U $LOCKFILE)" + echo "error: source tarball generation is already in progress (started by $owner)" + exit 1 +fi + +trap cleanup 0 +trap ctrl_c 2 + +/bin/touch "$LOCKFILE" + +#adjust the nice level to run at a lower priority +/usr/bin/renice +10 -p $$ > /dev/null + +set_umask +/bin/mkdir -p "$WORKDIR" +cd "$WORKDIR" + +[ -e "$logfile" ] && /bin/mv "$logfile" "$logfile.old" +echo "Orphaned sourceballs:" > "$logfile" + +for sourceball in "$srcpath"/*$SRCEXT; do + packagename=$(basename $sourceball) + packagename=${packagename%-*-*$SRCEXT} + + if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 ; then + echo "$packagename : no longer in svn. Removing sourceball." >> "$logfile" + mv $sourceball $SOURCE_CLEANUP_DESTDIR + elif [ -z "$(ls -A "$packagename/repos")" ]; then + echo "$packagename : no longer in repos but trunk is still in svn. Removing sourceball." >> "$logfile" + mv $sourceball $SOURCE_CLEANUP_DESTDIR + elif ! source "$packagename/trunk/$BUILDSCRIPT" && chk_license ${license[@]}; then + echo "$packagename : source hosting no longer required by license. Removing sourceball." >> "$logfile" + mv $sourceball $SOURCE_CLEANUP_DESTDIR + else + remove_old "$packagename/repos/" + fi +done + +cleanup 0 -- 1.6.6.1
--- cron-jobs/sourceballs-cleanup | 61 ----------------------------------------- 1 files changed, 0 insertions(+), 61 deletions(-) delete mode 100755 cron-jobs/sourceballs-cleanup diff --git a/cron-jobs/sourceballs-cleanup b/cron-jobs/sourceballs-cleanup deleted file mode 100755 index c8d7b15..0000000 --- a/cron-jobs/sourceballs-cleanup +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -srcpath="$FTP_BASE/sources/" -logfile="$srcpath/cleanup.txt" - -LOCKFILE="/tmp/.sourceball-cleanup.lock" -WORKDIR="/tmp/sourceball-cleanup.$packagename.$UID" - -cleanup () { - restore_umask - rm -rf "$WORKDIR" - rm -f "$LOCKFILE" - exit 0 -} - -ctrl_c() { - echo "Interrupted" >&2 - cleanup 0 -} - -if [ -f "$LOCKFILE" ]; then - owner="$(/usr/bin/stat -c %U $LOCKFILE)" - echo "error: source tarball generation is already in progress (started by $owner)" - exit 1 -fi - -trap cleanup 0 -trap ctrl_c 2 - -/bin/touch "$LOCKFILE" - -#adjust the nice level to run at a lower priority -/usr/bin/renice +10 -p $$ > /dev/null - -set_umask -/bin/mkdir -p "$WORKDIR" -cd "$WORKDIR" - -[ -e "$logfile" ] && /bin/mv "$logfile" "$logfile.old" -echo "Orphaned sourceballs:" > "$logfile" - -for sourceball in "$srcpath"/*$SRCEXT; do - packagename=$(basename $sourceball) - packagename=${packagename%-*-*$SRCEXT} - - if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 ; then - echo "$packagename : no longer in svn. Removing sourceball." >> "$logfile" - mv $sourceball $SOURCE_CLEANUP_DESTDIR - elif [ -z "$(ls -A "$packagename/repos")" ]; then - echo "$packagename : no longer in repos but trunk is still in svn. Removing sourceball." >> "$logfile" - mv $sourceball $SOURCE_CLEANUP_DESTDIR - elif ! source "$packagename/trunk/$BUILDSCRIPT" && chk_license ${license[@]}; then - echo "$packagename : source hosting no longer required by license. Removing sourceball." >> "$logfile" - mv $sourceball $SOURCE_CLEANUP_DESTDIR - fi -done - -cleanup 0 -- 1.6.6.1
Signed-off-by: Eric Bélanger <snowmaniscool@gmail.com> --- config | 1 + cron-jobs/sourceballs | 2 +- misc-scripts/make-sourceball | 14 +++++++++++--- misc-scripts/sourceballs-cleanup | 5 +++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/config b/config index 92def37..337210b 100644 --- a/config +++ b/config @@ -3,6 +3,7 @@ FTP_BASE="/srv/ftp" FTP_OS_SUFFIX="os" SVNREPO="file:///srv/svn-packages" +SVNREPOCOMMUNITY="file:///srv/svn-community" CLEANUP_DESTDIR="/srv/package-cleanup" SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index ccb00ad..93cb842 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -2,7 +2,7 @@ ftpbase="/srv/ftp" srcbase="/srv/ftp/sources" -repos="core extra testing" #community" +repos="core extra testing community community-testing" . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 610d7f1..37141c7 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -81,8 +81,16 @@ set_umask /bin/mkdir -p "$logpath" cd "$WORKDIR" -if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then - create_srcpackage "$packagename/repos/$reponame-$_arch" +if [[ "$reponame" = "community" || "$reponame" = "community-testing" ]]; then + if /usr/bin/svn export -q "$SVNREPOCOMMUNITY/$packagename" $packagename; then + create_srcpackage "$packagename/repos/$reponame-$_arch" + else + die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" + fi else - die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" + if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then + create_srcpackage "$packagename/repos/$reponame-$_arch" + else + die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" + fi fi diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup index 0a1ac4d..feaa345 100755 --- a/misc-scripts/sourceballs-cleanup +++ b/misc-scripts/sourceballs-cleanup @@ -43,7 +43,7 @@ remove_old() { break fi done - if [ $skip -ne 1 ]; then + if [ $skip -ne 1 ]; then mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR fi fi @@ -78,7 +78,8 @@ for sourceball in "$srcpath"/*$SRCEXT; do packagename=$(basename $sourceball) packagename=${packagename%-*-*$SRCEXT} - if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 ; then + if ! $(/usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1) \ + && ! $(/usr/bin/svn export -q --force "$SVNREPOCOMMUNITY/$packagename" "$packagename" >/dev/null 2>&1) ; then echo "$packagename : no longer in svn. Removing sourceball." >> "$logfile" mv $sourceball $SOURCE_CLEANUP_DESTDIR elif [ -z "$(ls -A "$packagename/repos")" ]; then -- 1.6.6.1
Am Montag, 8. Februar 2010 16:13:44 schrieb Eric Bélanger:
- if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 ; then + if ! $(/usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1) \ + && ! $(/usr/bin/svn export -q --force "$SVNREPOCOMMUNITY/$packagename" "$packagename" >/dev/null 2>&1) ; then
I think replacing "if command" with "if $(command)" is probably not what you want as you are no longer checking the return value but the output. Or I am just wrong here? -- Pierre Schmitz, https://users.archlinux.de/~pierre
On Sat, Mar 13, 2010 at 5:46 AM, Pierre Schmitz <pierre@archlinux.de> wrote:
Am Montag, 8. Februar 2010 16:13:44 schrieb Eric Bélanger:
- if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 ; then + if ! $(/usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1) \ + && ! $(/usr/bin/svn export -q --force "$SVNREPOCOMMUNITY/$packagename" "$packagename" >/dev/null 2>&1) ; then
I think replacing "if command" with "if $(command)" is probably not what you want as you are no longer checking the return value but the output. Or I am just wrong here?
I'm not a bash expert but it seem to be doing the same thing: $ if ! ls >/dev/null 2>&1 ;then echo fail; fi $ if ! $(ls >/dev/null 2>&1) ;then echo fail; fi $ if ! foo >/dev/null 2>&1 ;then echo fail; fi fail $ if ! $(foo >/dev/null 2>&1) ;then echo fail; fi fail
--
Pierre Schmitz, https://users.archlinux.de/~pierre
On Sat, Mar 13, 2010 at 2:58 PM, Eric Bélanger <snowmaniscool@gmail.com> wrote:
On Sat, Mar 13, 2010 at 5:46 AM, Pierre Schmitz <pierre@archlinux.de> wrote:
Am Montag, 8. Februar 2010 16:13:44 schrieb Eric Bélanger:
- if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 ; then + if ! $(/usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1) \ + && ! $(/usr/bin/svn export -q --force "$SVNREPOCOMMUNITY/$packagename" "$packagename" >/dev/null 2>&1) ; then
I think replacing "if command" with "if $(command)" is probably not what you want as you are no longer checking the return value but the output. Or I am just wrong here?
I'm not a bash expert but it seem to be doing the same thing:
$ if ! ls >/dev/null 2>&1 ;then echo fail; fi $ if ! $(ls >/dev/null 2>&1) ;then echo fail; fi $ if ! foo >/dev/null 2>&1 ;then echo fail; fi fail $ if ! $(foo >/dev/null 2>&1) ;then echo fail; fi fail
This is a very bad example. Try not suppressing all output which is very misleading (why do we do this so damn often in scripts anyway?). dmcgee@galway /tmp/test $ if ! ls; then echo "fail"; fi onefile dmcgee@galway /tmp/test $ if ! $(ls); then echo "fail"; fi bash: onefile: command not found fail dmcgee@galway /tmp/test $ if ! foo; then echo "fail"; fi bash: foo: command not found fail dmcgee@galway /tmp/test $ if ! $(foo); then echo "fail"; fi bash: foo: command not found fail -Dan
Here's a fixed version of the patch. I just removed the parenthesis and it seems to be working without them. I don't remember why I added them originally.
On Sun, Jun 27, 2010 at 11:40 PM, Eric Bélanger <snowmaniscool@gmail.com> wrote:
Here's a fixed version of the patch. I just removed the parenthesis and it seems to be working without them. I don't remember why I added them originally.
FYI, the patch: http://mailman.archlinux.org/pipermail/arch-dev-public/2010-June/017249.html was submitted in the original ML thread: http://mailman.archlinux.org/pipermail/arch-dev-public/2010-February/015356.... This message was supposed to accompany it but git send-email has messed up.
On 28/06/10 13:47, Eric Bélanger wrote:
On Sun, Jun 27, 2010 at 11:40 PM, Eric Bélanger<snowmaniscool@gmail.com> wrote:
Here's a fixed version of the patch. I just removed the parenthesis and it seems to be working without them. I don't remember why I added them originally.
FYI, the patch:
http://mailman.archlinux.org/pipermail/arch-dev-public/2010-June/017249.html
was submitted in the original ML thread: http://mailman.archlinux.org/pipermail/arch-dev-public/2010-February/015356....
This message was supposed to accompany it but git send-email has messed up.
For future reference, you can add comments directly in the patch after the "---" line and they will be ignored by git when applying. Allan
Signed-off-by: Eric Bélanger <snowmaniscool@gmail.com> --- config | 1 + cron-jobs/sourceballs | 2 +- misc-scripts/make-sourceball | 14 +++++++++++--- misc-scripts/sourceballs-cleanup | 5 +++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/config b/config index 57ae64b..1cd533f 100644 --- a/config +++ b/config @@ -2,6 +2,7 @@ FTP_BASE="/srv/ftp" SVNREPO="file:///srv/svn-packages" +SVNREPOCOMMUNITY="file:///srv/svn-community" CLEANUP_DESTDIR="/srv/package-cleanup" CLEANUP_DRYRUN=false diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index ccb00ad..93cb842 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -2,7 +2,7 @@ ftpbase="/srv/ftp" srcbase="/srv/ftp/sources" -repos="core extra testing" #community" +repos="core extra testing community community-testing" . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 610d7f1..37141c7 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -81,8 +81,16 @@ set_umask /bin/mkdir -p "$logpath" cd "$WORKDIR" -if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then - create_srcpackage "$packagename/repos/$reponame-$_arch" +if [[ "$reponame" = "community" || "$reponame" = "community-testing" ]]; then + if /usr/bin/svn export -q "$SVNREPOCOMMUNITY/$packagename" $packagename; then + create_srcpackage "$packagename/repos/$reponame-$_arch" + else + die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" + fi else - die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" + if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then + create_srcpackage "$packagename/repos/$reponame-$_arch" + else + die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" + fi fi diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup index 0a1ac4d..c7f284d 100755 --- a/misc-scripts/sourceballs-cleanup +++ b/misc-scripts/sourceballs-cleanup @@ -43,7 +43,7 @@ remove_old() { break fi done - if [ $skip -ne 1 ]; then + if [ $skip -ne 1 ]; then mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR fi fi @@ -78,7 +78,8 @@ for sourceball in "$srcpath"/*$SRCEXT; do packagename=$(basename $sourceball) packagename=${packagename%-*-*$SRCEXT} - if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 ; then + if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 \ + && ! /usr/bin/svn export -q --force "$SVNREPOCOMMUNITY/$packagename" "$packagename" >/dev/null 2>&1 ; then echo "$packagename : no longer in svn. Removing sourceball." >> "$logfile" mv $sourceball $SOURCE_CLEANUP_DESTDIR elif [ -z "$(ls -A "$packagename/repos")" ]; then -- 1.7.1
On Sun, 27 Jun 2010 23:40:38 -0400, Eric Bélanger <snowmaniscool@gmail.com> wrote:
Signed-off-by: Eric Bélanger <snowmaniscool@gmail.com> --- config | 1 + cron-jobs/sourceballs | 2 +- misc-scripts/make-sourceball | 14 +++++++++++--- misc-scripts/sourceballs-cleanup | 5 +++-- 4 files changed, 16 insertions(+), 6 deletions(-)
diff --git a/config b/config index 57ae64b..1cd533f 100644
applied, thanks. -- Pierre Schmitz, https://users.archlinux.de/~pierre
participants (4)
-
Allan McRae
-
Dan McGee
-
Eric Bélanger
-
Pierre Schmitz