- Aur-dev - lists.archlinux.org

Populate DB from aur-schema.sql
by Nodiv Byzero 01 May '18

01 May '18

Hello, When I try to populate DB from aur-schema.sql using mysql command line I see following error: ERROR 1071 (42000) at line 20: Specified key was too long; max key length is 767 bytes Have you seen this one when installed aur-web? Is it because I use Maria DB version 10.1.30? Thanks, Nodiv

2 3

[PATCH] Hash IP for API rate limit
by Florian Pritz 22 Apr '18

22 Apr '18

The hash uses the start of a fixed window as an HMAC key. Therefore the windows stored in the DB also use this so that there can only be one active window for each IP. Signed-off-by: Florian Pritz <bluewind(a)xinu.at> --- schema/aur-schema.sql | 2 +- upgrading/4.7.0.txt | 2 +- web/lib/aurjson.class.php | 12 +++++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/schema/aur-schema.sql b/schema/aur-schema.sql index 79de3f2..45f8993 100644 --- a/schema/aur-schema.sql +++ b/schema/aur-schema.sql @@ -403,7 +403,7 @@ CREATE TABLE AcceptedTerms ( -- Rate limits for API -- CREATE TABLE `ApiRateLimit` ( - IP VARCHAR(45) NOT NULL, + IP VARCHAR(255) NOT NULL, Requests INT(11) NOT NULL, WindowStart BIGINT(20) NOT NULL, PRIMARY KEY (`ip`) diff --git a/upgrading/4.7.0.txt b/upgrading/4.7.0.txt index 820e454..fcdbb4d 100644 --- a/upgrading/4.7.0.txt +++ b/upgrading/4.7.0.txt @@ -2,7 +2,7 @@ --- CREATE TABLE `ApiRateLimit` ( - IP VARCHAR(45) NOT NULL, + IP VARCHAR(255) NOT NULL, Requests INT(11) NOT NULL, WindowStart BIGINT(20) NOT NULL, PRIMARY KEY (`ip`) diff --git a/web/lib/aurjson.class.php b/web/lib/aurjson.class.php index c51e9c2..168c6b7 100644 --- a/web/lib/aurjson.class.php +++ b/web/lib/aurjson.class.php @@ -153,6 +153,9 @@ private function check_ratelimit($ip) { } $window_length = config_get("ratelimit", "window_length"); + $hmac_key = floor(time() / ($window_length)); + $ip = hash_hmac("sha256", $ip, $hmac_key); + $this->update_ratelimit($ip); $stmt = $this->dbh->prepare(" SELECT Requests FROM ApiRateLimit @@ -181,14 +184,13 @@ private function check_ratelimit($ip) { private function update_ratelimit($ip) { $window_length = config_get("ratelimit", "window_length"); $db_backend = config_get("database", "backend"); - $time = time(); + $window_start = floor(time() / ($window_length)); // Clean up old windows - $deletion_time = $time - $window_length; $stmt = $this->dbh->prepare(" DELETE FROM ApiRateLimit WHERE WindowStart < :time"); - $stmt->bindParam(":time", $deletion_time); + $stmt->bindParam(":time", $window_start); $stmt->execute(); if ($db_backend == "mysql") { @@ -198,7 +200,7 @@ private function update_ratelimit($ip) { VALUES (:ip, 1, :window_start) ON DUPLICATE KEY UPDATE Requests=Requests+1"); $stmt->bindParam(":ip", $ip); - $stmt->bindParam(":window_start", $time); + $stmt->bindParam(":window_start", $window_start); $stmt->execute(); } elseif ($db_backend == "sqlite") { $stmt = $this->dbh->prepare(" @@ -206,7 +208,7 @@ private function update_ratelimit($ip) { (IP, Requests, WindowStart) VALUES (:ip, 0, :window_start);"); $stmt->bindParam(":ip", $ip); - $stmt->bindParam(":window_start", $time); + $stmt->bindParam(":window_start", $window_start); $stmt->execute(); $stmt = $this->dbh->prepare(" -- 2.17.0

1 0

[aur-dev][PATCH] config: allow reading both the proto file and the modified config
by Eli Schwartz 22 Apr '18

22 Apr '18

This change allows aurweb configuration to be done via either: - copying config.proto to config and modifying values - creating a new config only containing modified values, next to a config.proto containing unmodified values The motivation for this change is to enable ansible configuration by storing only changed values, and deferring to config.proto otherwise. If a config.proto file does not exist next to /etc/aurweb/config or $AUR_CONFIG, it is ignored and *all* values are expected to live in the modified config file. Signed-off-by: Eli Schwartz <eschwartz(a)archlinux.org> --- aurweb/config.py | 4 ++++ web/lib/confparser.inc.php | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/aurweb/config.py b/aurweb/config.py index a52d942..c333c43 100644 --- a/aurweb/config.py +++ b/aurweb/config.py @@ -13,6 +13,10 @@ def _get_parser(): path = os.environ.get('AUR_CONFIG') else: path = "/etc/aurweb/config" + + if os.path.isfile(path + '.proto'): + with open(path + '.proto') as f: + _parser.read_file(f) _parser.read(path) return _parser diff --git a/web/lib/confparser.inc.php b/web/lib/confparser.inc.php index 499481d..2ed0108 100644 --- a/web/lib/confparser.inc.php +++ b/web/lib/confparser.inc.php @@ -8,11 +8,17 @@ function config_load() { if (!$path) { $path = "/etc/aurweb/config"; } + if (file_exists($path . ".proto")) { + $defaults = parse_ini_file($path . ".proto", true, INI_SCANNER_RAW); + } else { + $defaults = []; + } if (file_exists($path)) { - $AUR_CONFIG = parse_ini_file($path, true, INI_SCANNER_RAW); + $config = parse_ini_file($path, true, INI_SCANNER_RAW); } else { die("aurweb config file not found"); } + $AUR_CONFIG = array_replace_recursive($defaults, $config) } } -- 2.17.0

3 10

[PATCH] Allow manual breaks and horizontal lines in comments
by Lukas Fleischer 08 Apr '18

08 Apr '18

When sanitizing rendered comments, keep <hr> tags and <br> tags. The former are generated when using "---" in Markdown comments, the latter are used when putting two spaces at the end of a line. Fixes FS#56649. --- aurweb/scripts/rendercomment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aurweb/scripts/rendercomment.py b/aurweb/scripts/rendercomment.py index 4b64227..5e18fd5 100755 --- a/aurweb/scripts/rendercomment.py +++ b/aurweb/scripts/rendercomment.py @@ -119,8 +119,8 @@ def main(): FlysprayLinksExtension(), GitCommitsExtension(pkgbase), HeadingExtension()]) - allowed_tags = bleach.sanitizer.ALLOWED_TAGS + \ - ['p', 'pre', 'h4', 'h5', 'h6'] + allowed_tags = (bleach.sanitizer.ALLOWED_TAGS + + ['p', 'pre', 'h4', 'h5', 'h6', 'br', 'hr']) html = bleach.clean(html, tags=allowed_tags) save_rendered_comment(conn, commentid, html) -- 2.17.0

1 0

[PATCH 2/2] serve.py: add support for 'list-voted'
by Simon Legner 05 Apr '18

05 Apr '18

This command lists voted packages. --- aurweb/git/serve.py | 18 ++++++++++++++++++ test/t1200-git-serve.sh | 22 ++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/aurweb/git/serve.py b/aurweb/git/serve.py index 01aea20..3b3967a 100755 --- a/aurweb/git/serve.py +++ b/aurweb/git/serve.py @@ -326,6 +326,19 @@ def pkgbase_unvote(pkgbase, user): conn.commit() +def list_voted(user): + conn = aurweb.db.Connection() + + userid = conn.fetch_userid(user) + + cur = conn.execute("SELECT Name FROM PackageBases JOIN PackageVotes " + + "ON ID = PackageBaseID " + + "WHERE UsersID = ?", [userid]) + for row in cur: + print(row[0]) + conn.close() + + def pkgbase_set_keywords(pkgbase, keywords): pkgbase_id = pkgbase_from_name(pkgbase) if not pkgbase_id: @@ -508,6 +521,9 @@ def serve(action, cmdargv, user, privileged, remote_addr): pkgbase = cmdargv[1] pkgbase_unvote(pkgbase, user) + elif action == 'list-voted': + checkarg(cmdargv) + list_voted(user) elif action == 'set-comaintainers': checkarg_atleast(cmdargv, 'repository name') @@ -515,12 +531,14 @@ def serve(action, cmdargv, user, privileged, remote_addr): userlist = cmdargv[2:] pkgbase_set_comaintainers(pkgbase, userlist, user, privileged) elif action == 'help': + # commands in alphabetical order, git-* at the end cmds = { "adopt <name>": "Adopt a package base.", "disown <name>": "Disown a package base.", "flag <name> <comment>": "Flag a package base out-of-date.", "help": "Show this help message and exit.", "list-repos": "List all your repositories.", + "list-voted": "List voted packages.", "restore <name>": "Restore a deleted package base.", "set-comaintainers <name> [...]": "Set package base co-maintainers.", "set-keywords <name> [...]": "Change package base keywords.", diff --git a/test/t1200-git-serve.sh b/test/t1200-git-serve.sh index 94a5ff6..51db399 100755 --- a/test/t1200-git-serve.sh +++ b/test/t1200-git-serve.sh @@ -466,6 +466,12 @@ test_expect_success "Vote for a package base." ' EOF echo "SELECT NumVotes FROM PackageBases WHERE Name = \"foobar\";" | \ sqlite3 aur.db >actual && + test_cmp expected actual && + SSH_ORIGINAL_COMMAND="list-voted" AUR_USER=user AUR_PRIVILEGED=0 \ + "$GIT_SERVE" 2>&1 >actual && + cat >expected <<-EOF && + foobar + EOF test_cmp expected actual ' @@ -484,6 +490,12 @@ test_expect_success "Vote for a package base twice." ' EOF echo "SELECT NumVotes FROM PackageBases WHERE Name = \"foobar\";" | \ sqlite3 aur.db >actual && + test_cmp expected actual && + SSH_ORIGINAL_COMMAND="list-voted" AUR_USER=user AUR_PRIVILEGED=0 \ + "$GIT_SERVE" 2>&1 >actual && + cat >expected <<-EOF && + foobar + EOF test_cmp expected actual ' @@ -500,6 +512,11 @@ test_expect_success "Remove vote from a package base." ' EOF echo "SELECT NumVotes FROM PackageBases WHERE Name = \"foobar\";" | \ sqlite3 aur.db >actual && + test_cmp expected actual && + SSH_ORIGINAL_COMMAND="list-voted" AUR_USER=user AUR_PRIVILEGED=0 \ + "$GIT_SERVE" 2>&1 >actual && + cat >expected <<-EOF && + EOF test_cmp expected actual ' @@ -518,6 +535,11 @@ test_expect_success "Try to remove the vote again." ' EOF echo "SELECT NumVotes FROM PackageBases WHERE Name = \"foobar\";" | \ sqlite3 aur.db >actual && + test_cmp expected actual && + SSH_ORIGINAL_COMMAND="list-voted" AUR_USER=user AUR_PRIVILEGED=0 \ + "$GIT_SERVE" 2>&1 >actual && + cat >expected <<-EOF && + EOF test_cmp expected actual ' -- 2.16.3

3 2

[PATCH 1/2] db.py: extract commonly used fetch_userid function
by Simon Legner 05 Apr '18

05 Apr '18

--- aurweb/db.py | 7 ++++++ aurweb/git/serve.py | 68 ++++++++++------------------------------------------- 2 files changed, 19 insertions(+), 56 deletions(-) diff --git a/aurweb/db.py b/aurweb/db.py index 0b58197..bbc674b 100644 --- a/aurweb/db.py +++ b/aurweb/db.py @@ -49,3 +49,10 @@ class Connection: def close(self): self._conn.close() + + def fetch_userid(self, user): + cur = self.execute("SELECT ID FROM Users WHERE Username = ?", [user]) + userid = cur.fetchone()[0] + if userid == 0: + raise aurweb.exceptions.InvalidUserException(user) + return userid diff --git a/aurweb/git/serve.py b/aurweb/git/serve.py index 93ff34c..01aea20 100755 --- a/aurweb/git/serve.py +++ b/aurweb/git/serve.py @@ -38,10 +38,7 @@ def pkgbase_exists(pkgbase): def list_repos(user): conn = aurweb.db.Connection() - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - raise aurweb.exceptions.InvalidUserException(user) + userid = conn.fetch_userid(user) cur = conn.execute("SELECT Name, PackagerUID FROM PackageBases " + "WHERE MaintainerUID = ?", [userid]) @@ -58,10 +55,7 @@ def create_pkgbase(pkgbase, user): conn = aurweb.db.Connection() - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - raise aurweb.exceptions.InvalidUserException(user) + userid = conn.fetch_userid(user) now = int(time.time()) cur = conn.execute("INSERT INTO PackageBases (Name, SubmittedTS, " + @@ -90,10 +84,7 @@ def pkgbase_adopt(pkgbase, user, privileged): if not privileged and not cur.fetchone(): raise aurweb.exceptions.PermissionDeniedException(user) - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - raise aurweb.exceptions.InvalidUserException(user) + userid = conn.fetch_userid(user) cur = conn.execute("UPDATE PackageBases SET MaintainerUID = ? " + "WHERE ID = ?", [userid, pkgbase_id]) @@ -138,23 +129,8 @@ def pkgbase_set_comaintainers(pkgbase, userlist, user, privileged): userlist_old = set(pkgbase_get_comaintainers(pkgbase)) - uids_old = set() - for olduser in userlist_old: - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", - [olduser]) - userid = cur.fetchone()[0] - if userid == 0: - raise aurweb.exceptions.InvalidUserException(user) - uids_old.add(userid) - - uids_new = set() - for newuser in userlist: - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", - [newuser]) - userid = cur.fetchone()[0] - if userid == 0: - raise aurweb.exceptions.InvalidUserException(user) - uids_new.add(userid) + uids_old = set([conn.fetch_userid(olduser) for olduser in userlist_old]) + uids_new = set([conn.fetch_userid(newuser) for newuser in userlist]) uids_add = uids_new - uids_old uids_rem = uids_old - uids_new @@ -209,10 +185,7 @@ def pkgreq_close(reqid, user, reason, comments, autoclose=False): if autoclose: userid = 0 else: - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - raise aurweb.exceptions.InvalidUserException(user) + userid = conn.fetch_userid(user) conn.execute("UPDATE PackageRequests SET Status = ?, ClosureComment = ? " + "WHERE ID = ?", [status, comments, reqid]) @@ -250,9 +223,7 @@ def pkgbase_disown(pkgbase, user, privileged): comaintainers = pkgbase_get_comaintainers(pkgbase) if len(comaintainers) > 0: new_maintainer = comaintainers[0] - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", - [new_maintainer]) - new_maintainer_userid = cur.fetchone()[0] + new_maintainer_userid = conn.fetch_userid(new_maintainer) comaintainers.remove(new_maintainer) pkgbase_set_comaintainers(pkgbase, comaintainers, user, privileged) @@ -261,10 +232,7 @@ def pkgbase_disown(pkgbase, user, privileged): conn.commit() - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - raise aurweb.exceptions.InvalidUserException(user) + userid = conn.fetch_userid(user) subprocess.Popen((notify_cmd, 'disown', str(pkgbase_id), str(userid))) @@ -280,10 +248,7 @@ def pkgbase_flag(pkgbase, user, comment): conn = aurweb.db.Connection() - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - raise aurweb.exceptions.InvalidUserException(user) + userid = conn.fetch_userid(user) now = int(time.time()) conn.execute("UPDATE PackageBases SET " + @@ -303,10 +268,7 @@ def pkgbase_unflag(pkgbase, user): conn = aurweb.db.Connection() - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - raise aurweb.exceptions.InvalidUserException(user) + userid = conn.fetch_userid(user) if user in pkgbase_get_comaintainers(pkgbase): conn.execute("UPDATE PackageBases SET OutOfDateTS = NULL " + @@ -326,10 +288,7 @@ def pkgbase_vote(pkgbase, user): conn = aurweb.db.Connection() - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - raise aurweb.exceptions.InvalidUserException(user) + userid = conn.fetch_userid(user) cur = conn.execute("SELECT COUNT(*) FROM PackageVotes " + "WHERE UsersID = ? AND PackageBaseID = ?", @@ -352,10 +311,7 @@ def pkgbase_unvote(pkgbase, user): conn = aurweb.db.Connection() - cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) - userid = cur.fetchone()[0] - if userid == 0: - raise aurweb.exceptions.InvalidUserException(user) + userid = conn.fetch_userid(user) cur = conn.execute("SELECT COUNT(*) FROM PackageVotes " + "WHERE UsersID = ? AND PackageBaseID = ?", -- 2.16.3

2 1

[PATCH 0/2] add SSH command 'list-voted'
by Simon Legner 03 Apr '18

03 Apr '18

This patch adds the command `ssh aur(a)aur.archlinux.org list-voted`. Please CC replies. Simon Legner (2): db.py: extract commonly used fetch_userid function serve.py: add support for 'list-voted' aurweb/db.py | 7 ++++ aurweb/git/serve.py | 86 +++++++++++++++++-------------------------------- test/t1200-git-serve.sh | 22 +++++++++++++ 3 files changed, 59 insertions(+), 56 deletions(-) -- 2.16.3

1 0

[PATCH] Handle empty resultset getting recent 10 packages
by nodivbyzero 21 Mar '18

21 Mar '18

--- web/lib/stats.inc.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/web/lib/stats.inc.php b/web/lib/stats.inc.php index 80619fe..f5692f9 100644 --- a/web/lib/stats.inc.php +++ b/web/lib/stats.inc.php @@ -19,10 +19,12 @@ function updates_table() { $result = $dbh->query($q); $newest_packages = new ArrayObject(); - while ($row = $result->fetch(PDO::FETCH_ASSOC)) { - $newest_packages->append($row); + if ($result) { + while ($row = $result->fetch(PDO::FETCH_ASSOC)) { + $newest_packages->append($row); + } + set_cache_value($key, $newest_packages); } - set_cache_value($key, $newest_packages); } include('stats/updates_table.php'); } -- 2.16.1

3 2

[PATCH] Handle empty resultset getting recent 10 packages
by nodivbyzero 21 Mar '18

21 Mar '18

--- web/lib/stats.inc.php | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/web/lib/stats.inc.php b/web/lib/stats.inc.php index 80619fe..0fa3ec6 100644 --- a/web/lib/stats.inc.php +++ b/web/lib/stats.inc.php @@ -19,10 +19,13 @@ function updates_table() { $result = $dbh->query($q); $newest_packages = new ArrayObject(); - while ($row = $result->fetch(PDO::FETCH_ASSOC)) { - $newest_packages->append($row); - } - set_cache_value($key, $newest_packages); + if ($result) { + + while ($row = $result->fetch(PDO::FETCH_ASSOC)) { + $newest_packages->append($row); + } + set_cache_value($key, $newest_packages); + } } include('stats/updates_table.php'); } -- 2.16.1

2 1

[PATCH] Terminates execution of the script if aurweb config file not found
by nodivbyzero 20 Mar '18

20 Mar '18

--- web/lib/confparser.inc.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/web/lib/confparser.inc.php b/web/lib/confparser.inc.php index e7128be..499481d 100644 --- a/web/lib/confparser.inc.php +++ b/web/lib/confparser.inc.php @@ -8,7 +8,11 @@ function config_load() { if (!$path) { $path = "/etc/aurweb/config"; } - $AUR_CONFIG = parse_ini_file($path, true, INI_SCANNER_RAW); + if (file_exists($path)) { + $AUR_CONFIG = parse_ini_file($path, true, INI_SCANNER_RAW); + } else { + die("aurweb config file not found"); + } } } -- 2.16.1

4 3