- Aur-dev - lists.archlinux.org

[PATCH] Store timestamp and user ID when closing requests
by Lukas Fleischer 23 Nov '19

23 Nov '19

Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- aurweb/git/serve.py | 10 +++++++--- schema/aur-schema.sql | 5 ++++- test/t1200-git-serve.sh | 4 ++-- upgrading/4.9.0.txt | 6 ++++++ web/lib/pkgreqfuncs.inc.php | 2 ++ 5 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 upgrading/4.9.0.txt diff --git a/aurweb/git/serve.py b/aurweb/git/serve.py index d43523c..64d51b9 100755 --- a/aurweb/git/serve.py +++ b/aurweb/git/serve.py @@ -207,18 +207,22 @@ def pkgreq_close(reqid, user, reason, comments, autoclose=False): conn = aurweb.db.Connection() if autoclose: - userid = 0 + userid = None else: cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) userid = cur.fetchone()[0] if userid == 0: raise aurweb.exceptions.InvalidUserException(user) - conn.execute("UPDATE PackageRequests SET Status = ?, ClosureComment = ? " + - "WHERE ID = ?", [status, comments, reqid]) + now = int(time.time()) + conn.execute("UPDATE PackageRequests SET Status = ?, ClosedTS = ?, " + + "ClosedUID = ?, ClosureComment = ? " + + "WHERE ID = ?", [status, now, userid, comments, reqid]) conn.commit() conn.close() + if not userid: + userid = 0 subprocess.Popen((notify_cmd, 'request-close', str(userid), str(reqid), reason)).wait() diff --git a/schema/aur-schema.sql b/schema/aur-schema.sql index f05f40a..fa991ba 100644 --- a/schema/aur-schema.sql +++ b/schema/aur-schema.sql @@ -336,11 +336,14 @@ CREATE TABLE PackageRequests ( Comments TEXT NOT NULL, ClosureComment TEXT NOT NULL, RequestTS BIGINT UNSIGNED NOT NULL DEFAULT 0, + ClosedTS BIGINT UNSIGNED NULL DEFAULT NULL, + ClosedUID INTEGER UNSIGNED NULL DEFAULT NULL, Status TINYINT UNSIGNED NOT NULL DEFAULT 0, PRIMARY KEY (ID), FOREIGN KEY (ReqTypeID) REFERENCES RequestTypes(ID) ON DELETE NO ACTION, FOREIGN KEY (UsersID) REFERENCES Users(ID) ON DELETE SET NULL, - FOREIGN KEY (PackageBaseID) REFERENCES PackageBases(ID) ON DELETE SET NULL + FOREIGN KEY (PackageBaseID) REFERENCES PackageBases(ID) ON DELETE SET NULL, + FOREIGN KEY (ClosedUID) REFERENCES Users(ID) ON DELETE SET NULL ) ENGINE = InnoDB; CREATE INDEX RequestsUsersID ON PackageRequests (UsersID); CREATE INDEX RequestsPackageBaseID ON PackageRequests (PackageBaseID); diff --git a/test/t1200-git-serve.sh b/test/t1200-git-serve.sh index 473065f..e817b2c 100755 --- a/test/t1200-git-serve.sh +++ b/test/t1200-git-serve.sh @@ -381,9 +381,9 @@ test_expect_success "Check whether package requests are closed when disowning." grep "^Subject.*PRQ" sendmail.out >sendmail.parts && test_cmp sendmail.parts expected && cat <<-EOD >expected && - 1|2|3|foobar||4||The user user disowned the package.|0|2 + 1|2||The user user disowned the package. EOD - echo "SELECT * FROM PackageRequests WHERE Status = 2;" | sqlite3 aur.db >actual && + echo "SELECT ID, Status, ClosedUID, ClosureComment FROM PackageRequests WHERE ClosedTS > 0;" | sqlite3 aur.db >actual && test_cmp actual expected ' diff --git a/upgrading/4.9.0.txt b/upgrading/4.9.0.txt new file mode 100644 index 0000000..4c79283 --- /dev/null +++ b/upgrading/4.9.0.txt @@ -0,0 +1,6 @@ +1. Add new columns to store the timestamp and UID when closing requests: + +---- +ALTER TABLE PackageRequests ADD COLUMN ClosedTS BIGINT UNSIGNED NULL DEFAULT NULL; +ALTER TABLE PackageRequests ADD COLUMN ClosedUID INTEGER UNSIGNED NULL DEFAULT NULL; +---- diff --git a/web/lib/pkgreqfuncs.inc.php b/web/lib/pkgreqfuncs.inc.php index 079df38..7fce307 100644 --- a/web/lib/pkgreqfuncs.inc.php +++ b/web/lib/pkgreqfuncs.inc.php @@ -247,6 +247,8 @@ function pkgreq_close($id, $reason, $comments, $auto_close=false) { } $q = "UPDATE PackageRequests SET Status = " . intval($status) . ", "; + $q.= "ClosedTS = " . strval(time()) . ", "; + $q.= "ClosedUID = " . ($uid == 0 ? "NULL" : intval($uid)) . ", "; $q.= "ClosureComment = " . $dbh->quote($comments) . " "; $q.= "WHERE ID = " . intval($id); $dbh->exec($q); -- 2.24.0

1 0

[PATCH 1/2] Upgrade Sharness to 1.1.0
by Lukas Fleischer 23 Nov '19

23 Nov '19

Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- test/sharness.sh | 193 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 153 insertions(+), 40 deletions(-) diff --git a/test/sharness.sh b/test/sharness.sh index 1d57ce9..ace5a5b 100644 --- a/test/sharness.sh +++ b/test/sharness.sh @@ -1,4 +1,4 @@ -#!/bin/sh +# Sharness test framework. # # Copyright (c) 2011-2012 Mathias Lafeldt # Copyright (c) 2005-2012 Git project @@ -22,10 +22,22 @@ SHARNESS_VERSION="1.0.0" export SHARNESS_VERSION # Public: The file extension for tests. By default, it is set to "t". -: ${SHARNESS_TEST_EXTENSION:=t} +: "${SHARNESS_TEST_EXTENSION:=t}" export SHARNESS_TEST_EXTENSION -# Reset TERM to original terminal if found, otherwise save orignal TERM +# Public: Root directory containing tests. Tests can override this variable, +# e.g. for testing Sharness itself. +if test -z "$SHARNESS_TEST_DIRECTORY" +then + SHARNESS_TEST_DIRECTORY=$(pwd) +else + # ensure that SHARNESS_TEST_DIRECTORY is an absolute path so that it + # is valid even if the current working directory is changed + SHARNESS_TEST_DIRECTORY=$(cd "$SHARNESS_TEST_DIRECTORY" && pwd) || exit 1 +fi +export SHARNESS_TEST_DIRECTORY + +# Reset TERM to original terminal if found, otherwise save original TERM [ "x" = "x$SHARNESS_ORIG_TERM" ] && SHARNESS_ORIG_TERM="$TERM" || TERM="$SHARNESS_ORIG_TERM" @@ -33,23 +45,45 @@ export SHARNESS_TEST_EXTENSION export SHARNESS_ORIG_TERM # Export SHELL_PATH -: ${SHELL_PATH:=$SHELL} +: "${SHELL_PATH:=$SHELL}" export SHELL_PATH +# if --tee was passed, write the output not only to the terminal, but +# additionally to the file test-results/$BASENAME.out, too. +case "$SHARNESS_TEST_TEE_STARTED, $* " in +done,*) + # do not redirect again + ;; +*' --tee '*|*' --verbose-log '*) + mkdir -p "$SHARNESS_TEST_DIRECTORY/test-results" + BASE="$SHARNESS_TEST_DIRECTORY/test-results/$(basename "$0" ".$SHARNESS_TEST_EXTENSION")" + + # Make this filename available to the sub-process in case it is using + # --verbose-log. + SHARNESS_TEST_TEE_OUTPUT_FILE="$BASE.out" + export SHARNESS_TEST_TEE_OUTPUT_FILE + + # Truncate before calling "tee -a" to get rid of the results + # from any previous runs. + : >"$SHARNESS_TEST_TEE_OUTPUT_FILE" + + (SHARNESS_TEST_TEE_STARTED="done" ${SHELL_PATH} "$0" "$@" 2>&1; + echo $? >"$BASE.exit") | tee -a "$SHARNESS_TEST_TEE_OUTPUT_FILE" + test "$(cat "$BASE.exit")" = 0 + exit + ;; +esac + # For repeatability, reset the environment to a known state. # TERM is sanitized below, after saving color control sequences. LANG=C LC_ALL=C -PAGER=cat +PAGER="cat" TZ=UTC EDITOR=: export LANG LC_ALL PAGER TZ EDITOR unset VISUAL CDPATH GREP_OPTIONS -# Line feed -LF=' -' - [ "x$TERM" != "xdumb" ] && ( [ -t 1 ] && tput bold >/dev/null 2>&1 && @@ -82,9 +116,14 @@ while test "$#" -ne 0; do chain_lint=; shift ;; --no-color) color=; shift ;; + --tee) + shift ;; # was handled already --root=*) root=$(expr "z$1" : 'z[^=]*=$.*$') shift ;; + --verbose-log) + verbose_log=t + shift ;; *) echo "error: unknown test option '$1'" >&2; exit 1 ;; esac @@ -109,18 +148,25 @@ if test -n "$color"; then say_color_pass=$(tput setaf 2) # green say_color_info=$(tput setaf 6) # cyan say_color_reset=$(tput sgr0) - say_color_="" # no formatting for normal text + say_color_raw="" # no formatting for normal text say_color() { test -z "$1" && test -n "$quiet" && return - eval "say_color_color=\$say_color_$1" + case "$1" in + error) say_color_color=$say_color_error ;; + skip) say_color_color=$say_color_skip ;; + warn) say_color_color=$say_color_warn ;; + pass) say_color_color=$say_color_pass ;; + info) say_color_color=$say_color_info ;; + *) say_color_color=$say_color_raw ;; + esac shift - printf "%s\\n" "$say_color_color$*$say_color_reset" + printf '%s%s%s\n' "$say_color_color" "$*" "$say_color_reset" } else say_color() { test -z "$1" && test -n "$quiet" && return shift - printf "%s\n" "$*" + printf '%s\n' "$*" } fi @@ -137,7 +183,7 @@ say() { say_color info "$*" } -test -n "$test_description" || error "Test script did not set test_description." +test -n "${test_description:-}" || error "Test script did not set test_description." if test "$help" = "t"; then echo "$test_description" @@ -146,7 +192,11 @@ fi exec 5>&1 exec 6<&0 -if test "$verbose" = "t"; then +if test "$verbose_log" = "t" +then + exec 3>>"$SHARNESS_TEST_TEE_OUTPUT_FILE" 4>&3 +elif test "$verbose" = "t" +then exec 4>&2 3>&1 else exec 4>/dev/null 3>/dev/null @@ -177,7 +227,7 @@ trap 'die' EXIT # implicitly by specifying the prerequisite name in calls to test_expect_success # or test_expect_failure. # -# $1 - Name of prerequiste (a simple word, in all capital letters by convention) +# $1 - Name of prerequisite (a simple word, in all capital letters by convention) # # Examples # @@ -214,7 +264,7 @@ test_have_prereq() { # prerequisites can be concatenated with ',' save_IFS=$IFS IFS=, - set -- $* + set -- $@ IFS=$save_IFS total_prereq=0 @@ -231,7 +281,7 @@ test_have_prereq() { negative_prereq= esac - total_prereq=$(($total_prereq + 1)) + total_prereq=$((total_prereq + 1)) case "$satisfied_prereq" in *" $prerequisite "*) satisfied_this_prereq=t @@ -242,7 +292,7 @@ test_have_prereq() { case "$satisfied_this_prereq,$negative_prereq" in t,|,t) - ok_prereq=$(($ok_prereq + 1)) + ok_prereq=$((ok_prereq + 1)) ;; *) # Keep a list of missing prerequisites; restore @@ -263,12 +313,12 @@ test_have_prereq() { # the text_expect_* functions instead. test_ok_() { - test_success=$(($test_success + 1)) - say_color "" "ok $test_count - $@" + test_success=$((test_success + 1)) + say_color "" "ok $test_count - $*" } test_failure_() { - test_failure=$(($test_failure + 1)) + test_failure=$((test_failure + 1)) say_color error "not ok $test_count - $1" shift echo "$@" | sed -e 's/^/# /' @@ -276,13 +326,13 @@ test_failure_() { } test_known_broken_ok_() { - test_fixed=$(($test_fixed + 1)) - say_color error "ok $test_count - $@ # TODO known breakage vanished" + test_fixed=$((test_fixed + 1)) + say_color error "ok $test_count - $* # TODO known breakage vanished" } test_known_broken_failure_() { - test_broken=$(($test_broken + 1)) - say_color warn "not ok $test_count - $@ # TODO known breakage" + test_broken=$((test_broken + 1)) + say_color warn "not ok $test_count - $* # TODO known breakage" } # Public: Execute commands in debug mode. @@ -351,7 +401,7 @@ test_run_() { } test_skip_() { - test_count=$(($test_count + 1)) + test_count=$((test_count + 1)) to_skip= for skp in $SKIP_TESTS; do case $this_test.$test_count in @@ -370,7 +420,7 @@ test_skip_() { of_prereq=" of $test_prereq" fi - say_color skip >&3 "skipping test: $@" + say_color skip >&3 "skipping test: $*" say_color skip "ok $test_count # skip $1 (missing $missing_prereq${of_prereq})" : true ;; @@ -468,6 +518,44 @@ test_expect_failure() { echo >&3 "" } +# Public: Run test commands and expect anything from them. Used when a +# test is not stable or not finished for some reason. +# +# When the test passed, an "ok" message is printed, but the number of +# fixed tests is not incremented. +# +# When it failed, a "not ok ... # TODO known breakage" message is +# printed, and the number of tests still broken is incremented. +# +# Failures from these tests won't cause --immediate to stop. +# +# Usually takes two arguments: +# $1 - Test description +# $2 - Commands to be executed. +# +# With three arguments, the first will be taken to be a prerequisite: +# $1 - Comma-separated list of test prerequisites. The test will be skipped if +# not all of the given prerequisites are set. To negate a prerequisite, +# put a "!" in front of it. +# $2 - Test description +# $3 - Commands to be executed. +# +# Returns nothing. +test_expect_unstable() { + test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq= + test "$#" = 2 || error "bug in the test script: not 2 or 3 parameters to test_expect_unstable" + export test_prereq + if ! test_skip_ "$@"; then + say >&3 "checking unstable test: $2" + if test_run_ "$2" unstable; then + test_ok_ "$1" + else + test_known_broken_failure_ "$1" + fi + fi + echo >&3 "" +} + # Public: Run command and ensure that it fails in a controlled way. # # Use it instead of "! <command>". For example, when <command> dies due to a @@ -560,7 +648,7 @@ test_expect_code() { shift "$@" exit_code=$? - if test $exit_code = $want_code; then + if test "$exit_code" = "$want_code"; then return 0 fi @@ -570,7 +658,7 @@ test_expect_code() { # Public: Compare two files to see if expected output matches actual output. # -# The TEST_CMP variable defines the command used for the comparision; it +# The TEST_CMP variable defines the command used for the comparison; it # defaults to "diff -u". Only when the test script was started with --verbose, # will the command's output, the diff, be printed to the standard output. # @@ -617,14 +705,14 @@ test_seq() { while test "$i" -le "$j" do echo "$i" || return - i=$(expr "$i" + 1) + i=$(("$i" + 1)) done } # Public: Check if the file expected to be empty is indeed empty, and barfs # otherwise. # -# $1 - File to check for emptyness. +# $1 - File to check for emptiness. # # Returns 0 if file is empty, 1 otherwise. test_must_be_empty() { @@ -636,6 +724,36 @@ test_must_be_empty() { fi } +# debugging-friendly alternatives to "test [-f|-d|-e]" +# The commands test the existence or non-existence of $1. $2 can be +# given to provide a more precise diagnosis. +test_path_is_file () { + if ! test -f "$1" + then + echo "File $1 doesn't exist. $2" + false + fi +} + +test_path_is_dir () { + if ! test -d "$1" + then + echo "Directory $1 doesn't exist. $2" + false + fi +} + +# Check if the directory exists and is empty as expected, barf otherwise. +test_dir_is_empty () { + test_path_is_dir "$1" && + if test -n "$(find "$1" -mindepth 1 -maxdepth 1)" + then + echo "Directory '$1' is not empty, it contains:" + ls -la "$1" + return 1 + fi +} + # Public: Schedule cleanup commands to be run unconditionally at the end of a # test. # @@ -723,7 +841,7 @@ test_done() { say_color warn "# still have $test_broken known breakage(s)" fi if test "$test_broken" != 0 || test "$test_fixed" != 0; then - test_remaining=$(( $test_count - $test_broken - $test_fixed )) + test_remaining=$((test_count - test_broken - test_fixed)) msg="remaining $test_remaining test(s)" else test_remaining=$test_count @@ -760,20 +878,15 @@ test_done() { esac } -# Public: Root directory containing tests. Tests can override this variable, -# e.g. for testing Sharness itself. -: ${SHARNESS_TEST_DIRECTORY:=$(pwd)} -export SHARNESS_TEST_DIRECTORY - # Public: Source directory of test code and sharness library. # This directory may be different from the directory in which tests are # being run. -: ${SHARNESS_TEST_SRCDIR:=$(cd $(dirname $0) && pwd)} +: "${SHARNESS_TEST_SRCDIR:=$(cd "$(dirname "$0")" && pwd)}" export SHARNESS_TEST_SRCDIR # Public: Build directory that will be added to PATH. By default, it is set to # the parent directory of SHARNESS_TEST_DIRECTORY. -: ${SHARNESS_BUILD_DIRECTORY:="$SHARNESS_TEST_DIRECTORY/.."} +: "${SHARNESS_BUILD_DIRECTORY:="$SHARNESS_TEST_DIRECTORY/.."}" PATH="$SHARNESS_BUILD_DIRECTORY:$PATH" export PATH SHARNESS_BUILD_DIRECTORY -- 2.24.0

1 1

[PATCH] git-serve: check update hook permissions
by Lukas Fleischer 23 Nov '19

23 Nov '19

Verify that the update hook exists and is executable before running Git to prevent from broken repositories when permissions are broken. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- aurweb/exceptions.py | 6 ++++++ aurweb/git/serve.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/aurweb/exceptions.py b/aurweb/exceptions.py index 664db68..6201528 100644 --- a/aurweb/exceptions.py +++ b/aurweb/exceptions.py @@ -16,6 +16,12 @@ class PermissionDeniedException(AurwebException): super(PermissionDeniedException, self).__init__(msg) +class BrokenUpdateHookException(AurwebException): + def __init__(self, cmd): + msg = 'broken update hook: {:s}'.format(cmd) + super(BrokenUpdateHookException, self).__init__(msg) + + class InvalidUserException(AurwebException): def __init__(self, user): msg = 'unknown user: {:s}'.format(user) diff --git a/aurweb/git/serve.py b/aurweb/git/serve.py index 2882780..d43523c 100755 --- a/aurweb/git/serve.py +++ b/aurweb/git/serve.py @@ -496,6 +496,9 @@ def serve(action, cmdargv, user, privileged, remote_addr): if not privileged and not pkgbase_has_write_access(pkgbase, user): raise aurweb.exceptions.PermissionDeniedException(user) + if not os.access(git_update_cmd, os.R_OK | os.X_OK): + raise aurweb.exceptions.BrokenUpdateHookException(git_update_cmd) + os.environ["AUR_USER"] = user os.environ["AUR_PKGBASE"] = pkgbase os.environ["GIT_NAMESPACE"] = pkgbase -- 2.24.0

1 0

[PATCH] aurjson: use APCu/memcached for rate limiting
by Lukas Fleischer 02 Nov '19

02 Nov '19

There's no need to use permanent storage for rate limiting information; try to keep it in memory if caching is enabled. >From experiments with our live setup, this reduces the number of INSERT/DELETE operations per second from 15 to almost 0. Disk writes on the server hosting the AUR are reduced by 90% (from ~3MB/s to ~300kB/s). Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- web/lib/aurjson.class.php | 47 ++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/web/lib/aurjson.class.php b/web/lib/aurjson.class.php index 1c31a65..0ac586f 100644 --- a/web/lib/aurjson.class.php +++ b/web/lib/aurjson.class.php @@ -152,23 +152,26 @@ class AurJSON { return false; } - $window_length = config_get("ratelimit", "window_length"); $this->update_ratelimit($ip); - $stmt = $this->dbh->prepare(" - SELECT Requests FROM ApiRateLimit - WHERE IP = :ip"); - $stmt->bindParam(":ip", $ip); - $result = $stmt->execute(); - if (!$result) { - return false; - } + $status = false; + $value = get_cache_value('ratelimit:' . $ip, $status); + if (!$status) { + $stmt = $this->dbh->prepare(" + SELECT Requests FROM ApiRateLimit + WHERE IP = :ip"); + $stmt->bindParam(":ip", $ip); + $result = $stmt->execute(); + + if (!$result) { + return false; + } - $row = $stmt->fetch(PDO::FETCH_ASSOC); - if ($row['Requests'] > $limit) { - return true; + $row = $stmt->fetch(PDO::FETCH_ASSOC); + $value = $row['Requests']; } - return false; + + return $value > $limit; } /* @@ -182,9 +185,23 @@ class AurJSON { $window_length = config_get("ratelimit", "window_length"); $db_backend = config_get("database", "backend"); $time = time(); - - // Clean up old windows $deletion_time = $time - $window_length; + + /* Try to use the cache. */ + $status = false; + $value = get_cache_value('ratelimit-ws:' . $ip, $status); + if (!$status || ($status && $value < $deletion_time)) { + if (set_cache_value('ratelimit-ws:' . $ip, $time, $window_length) && + set_cache_value('ratelimit:' . $ip, 1, $window_length)) { + return; + } + } else { + $value = get_cache_value('ratelimit:' . $ip, $status); + if ($status && set_cache_value('ratelimit:' . $ip, $value + 1, $window_length)) + return; + } + + /* Clean up old windows. */ $stmt = $this->dbh->prepare(" DELETE FROM ApiRateLimit WHERE WindowStart < :time"); -- 2.23.0

1 0

[PATCH] Document maintenance tasks and internals
by Lukas Fleischer 01 Nov '19

01 Nov '19

Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- doc/maintenance.txt | 108 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 doc/maintenance.txt diff --git a/doc/maintenance.txt b/doc/maintenance.txt new file mode 100644 index 0000000..d609454 --- /dev/null +++ b/doc/maintenance.txt @@ -0,0 +1,108 @@ +aurweb Maintenance +================== + +Components +---------- + +aurweb has three user-facing components. + +The Git/SSH interface can be used to update package repositories and for basic +package maintenance from the command-line. More details can be found in +`doc/git-interface.txt`. + +The web interface can be used to browse packages, view package details, manage +aurweb accounts, add comments, vote for packages, flag packages, and submit +requests. Trusted Users can update package maintainers and delete/merge +packages. The web interface also includes an area for Trusted Users to post +AUR-related proposals and vote on them. + +The RPC interface can be used to query package information via HTTP. + +Installation +------------ + +The web backend requires a web server with PHP and an SQL database. The Git/SSH +interface requires Python, several Python modules and an up-to-date version of +Git. APCu or memcached can be used to reduce load on the database server. + +All dependencies and the full installation process are described in `INSTALL`. + +Updates +------- + +The `enable-maintenance` option (in the configuration file, usually located at +`/etc/aurweb/config`) can be used to switch aurweb into maintenance mode. This +disables both the Git/SSH interface and the web interface. The +`maintenance-exceptions` variable can be used to reactivate access for certain +IP addresses. Since changes to the database schema might temporarily break +parts of the backend, it is recommended to always enable maintenance mode +before performing an upgrade. + +To simplify the upgrade process, changes in the database schema (and other +changes that require manual interaction) are documented in `upgrading/`. An +exception are additions to the configuration file. It is recommended to always +compare `/etc/aurweb/config` to `conf/config.defaults` when upgrading to a new +release. + +Moreover, the aurweb Python modules and translations need to be reinstalled +with every upgrade. To this end, run `python3 setup.py install` from the aurweb +source tree and run `make install` in the `po/` subdirectory. + +Don't forget to always test all basic features first, then disable maintenance +mode after performing an upgrade. + +Maintenance Scripts +------------------- + +aurweb includes scheduled maintenance routines to perform expensive +computations and clean up the database: + +* aurweb-aurblup parses binary repositories and updates the `OfficialProviders` + table. This table is used to identify AUR packages that depend on packages in + the official repositories. It is also used to prevent users from uploading + packages that are in the official repositories already. + +* aurweb-tuvotereminder sends out reminders to TUs if the voting period for a + TU proposal ends soon. + +* aurweb-popupdate is used to recompute the popularity score of packages. + +* aurweb-pkgmaint automatically removes empty repositories that were created + within the last 24 hours but never populated. + +* aurweb-mkpkglists generates the package list files. + +* aurweb-usermaint removes the last login IP address of all users that did not + login within the past seven days. + +These scripts can be installed by running `python3 setup.py install` and are +usually scheduled using Cron. The current setup is: + +---- +*/5 * * * * aurweb-mkpkglists +1 */2 * * * aurweb-popupdate +2 */2 * * * aurweb-aurblup +3 */2 * * * aurweb-pkgmaint +4 */2 * * * aurweb-usermaint +5 */12 * * * aurweb-tuvotereminder +---- + +Advanced Administrative Features +-------------------------------- + +Trusted Users can set the AUR_OVERWRITE environment variable to enable +non-fast-forward pushes to the Git repositories. This feature is documented in +`doc/git-interface.txt`. + +Rate limiting is used to prevent users from hammering the RPC interface. The +`request_limit` and `window_length` options in the `ratelimit` section of the +configuration file can be used to configure this feature. Recent accesses are +stored in the `ApiRateLimit` table in the database. See commit 27654af (Add +rate limit support to API, 2018-02-01) for details. + +The database contains a `PackageBlacklist` table. Package names added to this +table will be rejected by the SSH/Git interface. This table can only be edited +by a database administrator. + +The `Bans` table can be used to ban certain IP addresses from both the web and +Git/SSH interface. This table can only be accessed by a database administrator. -- 2.23.0

1 0

aur client
by Alberto Salvia Novella 28 Oct '19

28 Oct '19

Since publishing to the AUR is something that all packagers do quite frequently, I have developed a software that reduces the related operations to the bare minimum, making publishing to the AUR instant. Basically it's a SMED <https://www.youtube.com/watch?v=Aoqra9bXAAg> system. The software is called "aur <https://aur.archlinux.org/packages/aur-git>" itself, and you can check its manual out by in the terminal entering: curl --silent https://gitlab.com/es20490446e/aur/raw/master/assets/aur.1 > aur.1; man ./aur.1; rm aur.1 As long as constructive, suggestions are very welcome. When replying please include my email address in the recipients list. Alberto <https://es20490446e.wordpress.com>

1 0

[PATCH] Display popularity with less decimal points
by Lukas Fleischer 26 Oct '19

26 Oct '19

Limit the display to two decimal points for packages with a popularity of at least 0.2. Suggested-by: Allan McRae <allan(a)archlinux.org> Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- web/template/pkg_details.php | 2 +- web/template/pkgbase_details.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/template/pkg_details.php b/web/template/pkg_details.php index 650c245..93b5f1e 100644 --- a/web/template/pkg_details.php +++ b/web/template/pkg_details.php @@ -259,7 +259,7 @@ endif; </tr> <tr> <th><?= __('Popularity') . ': ' ?></th> - <td><?= number_format($popularity, 6) ?></td> + <td><?= number_format($popularity, $popularity >= .2 ? 2 : 6) ?></td> </tr> <tr> <th><?= __('First Submitted') . ': ' ?></th> diff --git a/web/template/pkgbase_details.php b/web/template/pkgbase_details.php index 8a6e2b4..b2ce8cb 100644 --- a/web/template/pkgbase_details.php +++ b/web/template/pkgbase_details.php @@ -108,7 +108,7 @@ endif; </tr> <tr> <th><?= __('Popularity') . ': ' ?></th> - <td><?= number_format($popularity, 6) ?></td> + <td><?= number_format($popularity, $popularity >= .2 ? 2 : 6) ?></td> </tr> <tr> <th><?= __('First Submitted') . ': ' ?></th> -- 2.23.0

1 0

aurweb 4.8.0 released
by Lukas Fleischer 25 Oct '19

25 Oct '19

Dear aurweb contributors and users, This is a bit of a late announcement but we are pleased to announce that aurweb 4.8.0 has been released last week! Several bugs have been fixed, we made some significant performance improvements by caching results of expensive database queries, and a simple CAPTCHA has been added to combat the growing number of spam accounts. For a comprehensive list of changes, please consult the Git log [2]. As usual, bugs should be reported to the aurweb bug tracker [3]. [1] https://aur.archlinux.org/ [2] https://projects.archlinux.org/aurweb.git/log/?id=v4.8.0 [3] https://bugs.archlinux.org/index.php?project=2

1 0

[PATCH] Make package details cache TTL configurable
by Lukas Fleischer 07 Oct '19

07 Oct '19

The TTL for package details can be much longer than for generic values since they never change. Note that when an update is pushed via Git, all packages belonging to that package base are deleted and new packages are created. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- conf/config.defaults | 1 + web/lib/aurjson.class.php | 6 ++++-- web/lib/pkgfuncs.inc.php | 15 ++++++++++----- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/conf/config.defaults b/conf/config.defaults index c8bc3a7..c519eae 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -38,6 +38,7 @@ render-comment-cmd = /usr/local/bin/aurweb-rendercomment localedir = /srv/http/aurweb/aur.git/web/locale/ # memcache or apc cache = none +cache_pkginfo_ttl = 86400 memcache_servers = 127.0.0.1:11211 [ratelimit] diff --git a/web/lib/aurjson.class.php b/web/lib/aurjson.class.php index bfb8cd1..1c31a65 100644 --- a/web/lib/aurjson.class.php +++ b/web/lib/aurjson.class.php @@ -292,7 +292,8 @@ class AurJSON { "FROM Licenses INNER JOIN PackageLicenses " . "ON PackageLicenses.PackageID = " . $pkgid . " " . "AND PackageLicenses.LicenseID = Licenses.ID"; - $rows = db_cache_result($query, 'extended-fields:' . $pkgid, PDO::FETCH_ASSOC); + $ttl = config_get_int('options', 'cache_pkginfo_ttl'); + $rows = db_cache_result($query, 'extended-fields:' . $pkgid, PDO::FETCH_ASSOC, $ttl); $type_map = array( 'depends' => 'Depends', @@ -315,7 +316,8 @@ class AurJSON { $query = "SELECT Keyword FROM PackageKeywords " . "WHERE PackageBaseID = " . intval($base_id) . " " . "ORDER BY Keyword ASC"; - $rows = db_cache_result($query, 'keywords:' . intval($base_id)); + $ttl = config_get_int('options', 'cache_pkginfo_ttl'); + $rows = db_cache_result($query, 'keywords:' . intval($base_id), PDO::FETCH_NUM, $ttl); $data['Keywords'] = array_map(function ($x) { return $x[0]; }, $rows); } diff --git a/web/lib/pkgfuncs.inc.php b/web/lib/pkgfuncs.inc.php index be4bb06..b5035df 100644 --- a/web/lib/pkgfuncs.inc.php +++ b/web/lib/pkgfuncs.inc.php @@ -165,7 +165,8 @@ function pkg_licenses($pkgid) { $q = "SELECT l.Name FROM Licenses l "; $q.= "INNER JOIN PackageLicenses pl ON pl.LicenseID = l.ID "; $q.= "WHERE pl.PackageID = ". $pkgid; - $rows = db_cache_result($q, 'licenses:' . $pkgid); + $ttl = config_get_int('options', 'cache_pkginfo_ttl'); + $rows = db_cache_result($q, 'licenses:' . $pkgid, PDO::FETCH_NUM, $ttl); return array_map(function ($x) { return $x[0]; }, $rows); } @@ -184,7 +185,8 @@ function pkg_groups($pkgid) { $q = "SELECT g.Name FROM `Groups` g "; $q.= "INNER JOIN PackageGroups pg ON pg.GroupID = g.ID "; $q.= "WHERE pg.PackageID = ". $pkgid; - $rows = db_cache_result($q, 'groups:' . $pkgid); + $ttl = config_get_int('options', 'cache_pkginfo_ttl'); + $rows = db_cache_result($q, 'groups:' . $pkgid, PDO::FETCH_NUM, $ttl); return array_map(function ($x) { return $x[0]; }, $rows); } @@ -208,7 +210,8 @@ function pkg_providers($name) { $q.= "UNION "; $q.= "SELECT 0, Name FROM OfficialProviders "; $q.= "WHERE Provides = " . $dbh->quote($name); - return db_cache_result($q, 'providers:' . $name); + $ttl = config_get_int('options', 'cache_pkginfo_ttl'); + return db_cache_result($q, 'providers:' . $name, PDO::FETCH_NUM, $ttl); } /** @@ -231,7 +234,8 @@ function pkg_dependencies($pkgid, $limit) { $q.= "LEFT JOIN DependencyTypes dt ON dt.ID = pd.DepTypeID "; $q.= "WHERE pd.PackageID = ". $pkgid . " "; $q.= "ORDER BY pd.DepName LIMIT " . intval($limit); - return db_cache_result($q, 'dependencies:' . $pkgid); + $ttl = config_get_int('options', 'cache_pkginfo_ttl'); + return db_cache_result($q, 'dependencies:' . $pkgid, PDO::FETCH_NUM, $ttl); } /** @@ -251,7 +255,8 @@ function pkg_relations($pkgid) { $q.= "LEFT JOIN RelationTypes rt ON rt.ID = pr.RelTypeID "; $q.= "WHERE pr.PackageID = ". $pkgid . " "; $q.= "ORDER BY pr.RelName"; - return db_cache_result($q, 'relations:' . $pkgid); + $ttl = config_get_int('options', 'cache_pkginfo_ttl'); + return db_cache_result($q, 'relations:' . $pkgid, PDO::FETCH_NUM, $ttl); } /** -- 2.23.0

1 0

[PATCH] Cache package licenses, groups and relations
by Lukas Fleischer 07 Oct '19

07 Oct '19

Cache more package details if the global caching mechanism is enabled. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- web/lib/pkgfuncs.inc.php | 66 ++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 44 deletions(-) diff --git a/web/lib/pkgfuncs.inc.php b/web/lib/pkgfuncs.inc.php index 3e4b2a8..be4bb06 100644 --- a/web/lib/pkgfuncs.inc.php +++ b/web/lib/pkgfuncs.inc.php @@ -158,22 +158,15 @@ function pkg_from_name($name="") { * @return array All licenses for the package */ function pkg_licenses($pkgid) { - $lics = array(); $pkgid = intval($pkgid); - if ($pkgid > 0) { - $dbh = DB::connect(); - $q = "SELECT l.Name FROM Licenses l "; - $q.= "INNER JOIN PackageLicenses pl ON pl.LicenseID = l.ID "; - $q.= "WHERE pl.PackageID = ". $pkgid; - $result = $dbh->query($q); - if (!$result) { - return array(); - } - while ($row = $result->fetch(PDO::FETCH_COLUMN, 0)) { - $lics[] = $row; - } + if (!$pkgid) { + return array(); } - return $lics; + $q = "SELECT l.Name FROM Licenses l "; + $q.= "INNER JOIN PackageLicenses pl ON pl.LicenseID = l.ID "; + $q.= "WHERE pl.PackageID = ". $pkgid; + $rows = db_cache_result($q, 'licenses:' . $pkgid); + return array_map(function ($x) { return $x[0]; }, $rows); } /** @@ -184,22 +177,15 @@ function pkg_licenses($pkgid) { * @return array All package groups for the package */ function pkg_groups($pkgid) { - $grps = array(); $pkgid = intval($pkgid); - if ($pkgid > 0) { - $dbh = DB::connect(); - $q = "SELECT g.Name FROM `Groups` g "; - $q.= "INNER JOIN PackageGroups pg ON pg.GroupID = g.ID "; - $q.= "WHERE pg.PackageID = ". $pkgid; - $result = $dbh->query($q); - if (!$result) { - return array(); - } - while ($row = $result->fetch(PDO::FETCH_COLUMN, 0)) { - $grps[] = $row; - } + if (!$pkgid) { + return array(); } - return $grps; + $q = "SELECT g.Name FROM `Groups` g "; + $q.= "INNER JOIN PackageGroups pg ON pg.GroupID = g.ID "; + $q.= "WHERE pg.PackageID = ". $pkgid; + $rows = db_cache_result($q, 'groups:' . $pkgid); + return array_map(function ($x) { return $x[0]; }, $rows); } /** @@ -256,24 +242,16 @@ function pkg_dependencies($pkgid, $limit) { * @return array All package relations for the package */ function pkg_relations($pkgid) { - $rels = array(); $pkgid = intval($pkgid); - if ($pkgid > 0) { - $dbh = DB::connect(); - $q = "SELECT pr.RelName, rt.Name, pr.RelCondition, pr.RelArch, p.ID FROM PackageRelations pr "; - $q.= "LEFT JOIN Packages p ON pr.RelName = p.Name "; - $q.= "LEFT JOIN RelationTypes rt ON rt.ID = pr.RelTypeID "; - $q.= "WHERE pr.PackageID = ". $pkgid . " "; - $q.= "ORDER BY pr.RelName"; - $result = $dbh->query($q); - if (!$result) { - return array(); - } - while ($row = $result->fetch(PDO::FETCH_NUM)) { - $rels[] = $row; - } + if (!$pkgid) { + return array(); } - return $rels; + $q = "SELECT pr.RelName, rt.Name, pr.RelCondition, pr.RelArch, p.ID FROM PackageRelations pr "; + $q.= "LEFT JOIN Packages p ON pr.RelName = p.Name "; + $q.= "LEFT JOIN RelationTypes rt ON rt.ID = pr.RelTypeID "; + $q.= "WHERE pr.PackageID = ". $pkgid . " "; + $q.= "ORDER BY pr.RelName"; + return db_cache_result($q, 'relations:' . $pkgid); } /** -- 2.23.0

1 0