- Aur-dev - lists.archlinux.org

Change aur-schema.sql to check if table exists
by Jelle van der Waa 10 Mar '18

10 Mar '18

Would it be possible to change the aur-schema.sql to check if tables exists before creating them? This would be easier for the Ansible role for deploying the AUR, since I can skip the check if the User table exists. I believe it would be as easy as changing it to: CREATE TABLE IF NOT EXISTS tbl_name -- Jelle van der Waa

1 0

[PATCH] Add capability for co-maintainers to disown packages
by Mark Weiman 10 Mar '18

10 Mar '18

Implements FS#53832 Signed-off-by: Mark Weiman <mark.weiman(a)markzz.com> --- web/html/pkgbase.php | 3 +++ web/html/pkgdisown.php | 13 ++++++++++--- web/lib/pkgbasefuncs.inc.php | 12 ++++++++++-- web/template/pkgbase_actions.php | 2 +- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/web/html/pkgbase.php b/web/html/pkgbase.php index 03b0eee..cf9a6c6 100644 --- a/web/html/pkgbase.php +++ b/web/html/pkgbase.php @@ -60,6 +60,9 @@ if (check_token()) { $output = __("The selected packages have not been disowned, check the confirmation checkbox."); $ret = false; } + } elseif (current_action("do_DisownComaintainer")) { + $uid = uid_from_sid($_COOKIE["AURSID"]); + list($ret, $output) = pkgbase_remove_comaintainer($base_id, $uid); } elseif (current_action("do_Vote")) { list($ret, $output) = pkgbase_vote($ids, true); } elseif (current_action("do_UnVote")) { diff --git a/web/html/pkgdisown.php b/web/html/pkgdisown.php index 4b04e85..8da08cf 100644 --- a/web/html/pkgdisown.php +++ b/web/html/pkgdisown.php @@ -7,10 +7,13 @@ include_once("pkgfuncs.inc.php"); html_header(__("Disown Package")); +$action = "do_Disown"; + $maintainer_uids = array(pkgbase_maintainer_uid($base_id)); $comaintainers = pkgbase_get_comaintainers($base_id); +$comaintainer_uids = pkgbase_get_comaintainer_uids(array($base_id)); -if (has_credential(CRED_PKGBASE_DISOWN, $maintainer_uids)): ?> +if (has_credential(CRED_PKGBASE_DISOWN, array_merge($maintainer_uids, $comaintainer_uids))): ?> <div class="box"> <h2><?= __('Disown Package') ?>: <?= htmlspecialchars($pkgbase_name) ?></h2> <p> @@ -23,7 +26,11 @@ if (has_credential(CRED_PKGBASE_DISOWN, $maintainer_uids)): ?> <?php endforeach; ?> </ul> <p> - <?php if (count($comaintainers) > 0 && !has_credential(CRED_PKGBASE_DISOWN)): ?> + + <?php if (in_array(uid_from_sid($_COOKIE["AURSID"]), $comaintainer_uids) && !has_credential(CRED_PKGBASE_DISOWN)): + $action = "do_DisownComaintainer"; ?> + <?= __("By selecting the checkbox, you confirm that you want to no longer be a package co-maintainer.") ?> + <?php elseif (count($comaintainers) > 0 && !has_credential(CRED_PKGBASE_DISOWN)): ?> <?= __('By selecting the checkbox, you confirm that you want to disown the package and transfer ownership to %s%s%s.', '<strong>', $comaintainers[0], '</strong>'); ?> <?php else: ?> @@ -40,7 +47,7 @@ if (has_credential(CRED_PKGBASE_DISOWN, $maintainer_uids)): ?> <?php endif; ?> <p><label class="confirmation"><input type="checkbox" name="confirm" value="1" /> <?= __("Confirm to disown the package") ?></label</p> - <p><input type="submit" class="button" name="do_Disown" value="<?= __("Disown") ?>" /></p> + <p><input type="submit" class="button" name="<?= $action ?>" value="<?= __("Disown") ?>" /></p> </fieldset> </form> </div> diff --git a/web/lib/pkgbasefuncs.inc.php b/web/lib/pkgbasefuncs.inc.php index ff1bc90..16f95da 100644 --- a/web/lib/pkgbasefuncs.inc.php +++ b/web/lib/pkgbasefuncs.inc.php @@ -1158,11 +1158,12 @@ function pkgbase_get_comaintainer_uids($base_ids) { * * @param int $base_id The package base ID to update the co-maintainers of * @param array $users Array of co-maintainer user names + * @param boolean $override Override credential check if true * * @return array Tuple of success/failure indicator and error message */ -function pkgbase_set_comaintainers($base_id, $users) { - if (!has_credential(CRED_PKGBASE_EDIT_COMAINTAINERS, array(pkgbase_maintainer_uid($base_id)))) { +function pkgbase_set_comaintainers($base_id, $users, $override=false) { + if (!$override && !has_credential(CRED_PKGBASE_EDIT_COMAINTAINERS, array(pkgbase_maintainer_uid($base_id)))) { return array(false, __("You are not allowed to manage co-maintainers of this package base.")); } @@ -1213,3 +1214,10 @@ function pkgbase_set_comaintainers($base_id, $users) { return array(true, __("The package base co-maintainers have been updated.")); } + +function pkgbase_remove_comaintainer($base_id, $uid) { + $uname = username_from_id($uid); + $names = pkgbase_get_comaintainers($base_id); + $names = array_diff($names, array($uname)); + return pkgbase_set_comaintainers($base_id, $names, true); +} diff --git a/web/template/pkgbase_actions.php b/web/template/pkgbase_actions.php index d3f0592..5eee547 100644 --- a/web/template/pkgbase_actions.php +++ b/web/template/pkgbase_actions.php @@ -41,7 +41,7 @@ <?php if ($uid && $row["MaintainerUID"] === NULL): ?> <li><?= html_action_form($base_uri . 'adopt/', "do_Adopt", __('Adopt Package')) ?></li> - <?php elseif (has_credential(CRED_PKGBASE_DISOWN, array($row["MaintainerUID"]))): ?> + <?php elseif (has_credential(CRED_PKGBASE_DISOWN, array_merge(array($row["MaintainerUID"]), pkgbase_get_comaintainer_uids(array($base_id))))): ?> <li><?= html_action_form($base_uri . 'disown/', "do_Disown", __('Disown Package')) ?></li> <?php endif; ?> </ul> -- 2.16.1

4 6

[PATCH 0/1] Update cache code to INI style configuration
by Jelle van der Waa 10 Mar '18

10 Mar '18

This updates the cache code to make it possible to enable memcache for the cache of MySQL data. I couldn't get APC working and can't figure out if Arch still supports it, I'd suggest getting rid of it. Would you accept more patches to use memcache caching? Jelle van der Waa (1): Update cache code to INI style configuration INSTALL | 6 ++++++ conf/config.proto | 3 +++ web/lib/cachefuncs.inc.php | 10 +++------- 3 files changed, 12 insertions(+), 7 deletions(-) -- 2.16.2

2 2

[PATCH] Remove unused variable $dbh in pkgbase_display_details
by Jelle van der Waa 10 Mar '18

10 Mar '18

Signed-off-by: Jelle van der Waa <jelle(a)vdwaa.nl> --- web/lib/pkgbasefuncs.inc.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/web/lib/pkgbasefuncs.inc.php b/web/lib/pkgbasefuncs.inc.php index ff1bc90..ed4f3d0 100644 --- a/web/lib/pkgbasefuncs.inc.php +++ b/web/lib/pkgbasefuncs.inc.php @@ -264,8 +264,6 @@ function pkgbase_get_details($base_id) { * @return void */ function pkgbase_display_details($base_id, $row, $SID="") { - $dbh = DB::connect(); - if (isset($row['error'])) { print "<p>" . $row['error'] . "</p>\n"; } -- 2.16.2

2 1

alad@mailbox.org
by Actionless Loveless 08 Mar '18

08 Mar '18

Currently AUR web interface is able to resolve the dependencies in case if the dependendent package is provided by another package (see python2-catkin-pkg in the dependencies list): https://aur.archlinux.org/packages/ros-indigo-catkin/ However AUR RPC is not allowing querying by Provides field but only by `name`, `name-desc` and `maintainer` fields: https://aur.archlinux.org/rpc.php The discussion about that problem (https://github.com/ actionless/pikaur/issues/53) resulted in at least 3 possible ways to resolve that issue: - adding name-desc-provides filter - adding provides filter - when searching by name match not only Name field but also Provides field Please let me know if you're OK with one of the following approaches or propose another one. And if you fine with the idea let me know if you can implement such filter or I should send a patch. Also related ticket: https://bugs.archlinux.org/task/49090 All best, Yauheni Kirylau

3 13

Extending the RPC to include "provides" info
by Actionless Loveless 08 Mar '18

08 Mar '18

Hi Mikael! Are you already sitting on the patch for that feature? On Thu, Mar 8, 2018 at 6:24 AM, Mikael Blomstrand <mbloms(a)kth.se> wrote: > I'm pretty new to this mailing list. But I'm been thinking the same thing. > Talked about it in a previous mail. https://lists.archlinux. > org/pipermail/aur-general/2018-March/034001.html > > I think it's a great idea. When searching by name, i think the rpc should > at least also search by the "replaces" attribute too. Provides would be a > separate query but it could also be the same. > > Cheers, > Mikael > > tors 8 mars 2018 kl. 03:25 skrev Actionless Loveless < > actionless.loveless(a)gmail.com>: > >> Currently AUR web interface is able to resolve the dependencies in case if >> the dependendent package is provided by another package (see >> python2-catkin-pkg in the dependencies list): >> https://aur.archlinux.org/packages/ros-indigo-catkin/ >> >> However AUR RPC is not allowing querying by Provides field but only by >> `name`, >> `name-desc` and `maintainer` fields: >> https://aur.archlinux.org/rpc.php >> >> The discussion about that problem (https://github.com/ >> actionless/pikaur/issues/53) resulted in at least 3 possible ways to >> resolve that issue: >> - adding name-desc-provides filter >> - adding provides filter >> - when searching by name match not only Name field but also Provides field >> >> Please let me know if you're OK with one of the following approaches or >> propose another one. And if you fine with the idea let me know if you can >> implement such filter or I should send a patch. >> Also related ticket: https://bugs.archlinux.org/task/49090 >> >> >> All best, Yauheni Kirylau >> >

3 4

Searching the AUR API by dependent packages?
by Baptiste Jonglez 23 Feb '18

23 Feb '18

Hi, I wanted to use the AUR API to list packages that depend on a specific package. But it seems that the API does not implement this kind of search. The use-case is the following: when I bump a package in [community], I would like to list all packages in the AUR that potentially need a rebuild. Would this be a welcome addition to the API? If it is, I may try to hack this myself (but I have no guarantee about available time, and I stopped writing PHP and raw SQL years ago) The API I have in mind is the following: /rpc/?type=search&by=depends&arg=ocaml → returns all packages that depend on ocaml /rpc/?type=search&by=makedepends&arg=ocaml → returns all packages that make-depend on ocaml /rpc/?type=search&by=optdepends&arg=ocaml → returns all packages that opt-depend on ocaml Thanks, Baptiste

2 9

[PATCH] Add rate limit support to API
by Florian Pritz 23 Feb '18

23 Feb '18

This allows us to prevent users from hammering the API every few seconds to check if any of their packages were updated. Real world users check as often as every 5 or 10 seconds. Signed-off-by: Florian Pritz <bluewind(a)xinu.at> --- Basic idea for a rate limiting solution. Currently the cleanup of old entries is done on each api request. That may be a bit excessive, but I didn't find a cronjob to put it and given the table is indexed, the query is probably fast enough. At least for a PoC this should be fine. Tell me what you think. conf/config.proto | 4 +++ upgrading/4.7.0.txt | 11 ++++++++ web/lib/aurjson.class.php | 71 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 upgrading/4.7.0.txt diff --git a/conf/config.proto b/conf/config.proto index 1750929..934d369 100644 --- a/conf/config.proto +++ b/conf/config.proto @@ -36,6 +36,10 @@ enable-maintenance = 1 maintenance-exceptions = 127.0.0.1 render-comment-cmd = /usr/local/bin/aurweb-rendercomment +[ratelimit] +request_limit = 4000 +window_length = 86400 + [notifications] notify-cmd = /usr/local/bin/aurweb-notify sendmail = /usr/bin/sendmail diff --git a/upgrading/4.7.0.txt b/upgrading/4.7.0.txt new file mode 100644 index 0000000..1b51cd2 --- /dev/null +++ b/upgrading/4.7.0.txt @@ -0,0 +1,11 @@ +1. Add ApiRateLimit table: + +--- +CREATE TABLE `ApiRateLimit` ( + `ip` varchar(45) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + `requests` int(11) NOT NULL, + `window_start` bigint(20) NOT NULL, + PRIMARY KEY (`ip`), + KEY `window_start_idx` (`window_start`) +) ENGINE=InnoDB; +--- diff --git a/web/lib/aurjson.class.php b/web/lib/aurjson.class.php index 9eeaafd..8a6dd7b 100644 --- a/web/lib/aurjson.class.php +++ b/web/lib/aurjson.class.php @@ -96,6 +96,11 @@ public function handle($http_data) { $this->dbh = DB::connect(); + if ($this->check_ratelimit($_SERVER['REMOTE_ADDR'])) { + header("HTTP/1.1 429 Too Many Requests"); + return $this->json_error('Rate limit reached'); + } + $type = str_replace('-', '_', $http_data['type']); if ($type == 'info' && $this->version >= 5) { $type = 'multiinfo'; @@ -130,6 +135,72 @@ public function handle($http_data) { } } + /* + * Check if an IP needs to be rate limited. + * + * @param $ip IP of the current request + * + * @return true if IP needs to be rate limited, false otherwise. + */ + private function check_ratelimit($ip) { + $limit = config_get("ratelimit", "request_limit"); + $window_length = config_get("ratelimit", "window_length"); + $this->update_ratelimit($ip); + $stmt = $this->dbh->prepare(" + SELECT requests,window_start FROM ApiRateLimit + WHERE ip = :ip"); + $stmt->bindParam(":ip", $ip); + $result = $stmt->execute(); + + if (!$result) { + return false; + } + + $row = $stmt->fetch(PDO::FETCH_ASSOC); + + if ($row['window_start'] < time() - $window_length) { + $stmt = $this->dbh->prepare(" + DELETE FROM ApiRateLimit + WHERE ip = :ip"); + $stmt->bindParam(":ip", $ip); + $stmt->execute(); + return false; + } + + if ($row['requests'] > $limit) { + return true; + } + return false; + } + + /* + * Update a rate limit for an IP by increasing it's requests value by one. + * + * @param $ip IP of the current request + * + * @return void + */ + private function update_ratelimit($ip) { + $window_length = config_get("ratelimit", "window_length"); + $time = time(); + $deletion_time = $time - $window_length; + $stmt = $this->dbh->prepare(" + INSERT INTO ApiRateLimit + (ip, requests, window_start) + VALUES (:ip, 0, :window_start) + ON DUPLICATE KEY UPDATE requests=requests+1"); + $stmt->bindParam(":ip", $ip); + $stmt->bindParam(":window_start", $time); + $stmt->execute(); + + // TODO: this should move into some cronjob + $stmt = $this->dbh->prepare(" + DELETE FROM ApiRateLimit + WHERE window_start < :time"); + $stmt->bindParam(":time", $deletion_time); + $stmt->execute(); + } + /* * Returns a JSON formatted error string. * -- 2.15.1

2 5

[PATCH] Remove disjunction in pkg_providers query
by Florian Pritz 23 Feb '18

23 Feb '18

For some reason, running the SELECT .. WHERE .. OR .. query takes e.g. 58ms on a randomly generated db for some dependency name. Splitting the OR into two dedicated queries and UNIONing the result takes only 0.42ms. On the Arch Linux installation, searching for the providers of e.g. mongodb takes >=110ms when not cached by the query cache. The new query takes <1ms even when not cached. Signed-off-by: Florian Pritz <bluewind(a)xinu.at> --- web/lib/pkgfuncs.inc.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/web/lib/pkgfuncs.inc.php b/web/lib/pkgfuncs.inc.php index d022ebe..1dd8481 100644 --- a/web/lib/pkgfuncs.inc.php +++ b/web/lib/pkgfuncs.inc.php @@ -212,10 +212,12 @@ function pkg_groups($pkgid) { function pkg_providers($name) { $dbh = DB::connect(); $q = "SELECT p.ID, p.Name FROM Packages p "; + $q.= "WHERE p.Name = " . $dbh->quote($name) . " "; + $q.= "UNION "; + $q = "SELECT p.ID, p.Name FROM Packages p "; $q.= "LEFT JOIN PackageRelations pr ON pr.PackageID = p.ID "; $q.= "LEFT JOIN RelationTypes rt ON rt.ID = pr.RelTypeID "; - $q.= "WHERE p.Name = " . $dbh->quote($name) . " "; - $q.= "OR (rt.Name = 'provides' "; + $q.= "WHERE (rt.Name = 'provides' "; $q.= "AND pr.RelName = " . $dbh->quote($name) . ")"; $q.= "UNION "; $q.= "SELECT 0, Name FROM OfficialProviders "; -- 2.16.2

3 2

[PATCH] Add missing instruction for testing aurweb with a sqlite database
by Baptiste Jonglez 28 Jan '18

28 Jan '18

From: Baptiste Jonglez <git(a)bitsofnetworks.org> Also clarify what to change in the configuration. Signed-off-by: Baptiste Jonglez <git(a)bitsofnetworks.org> --- TESTING | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/TESTING b/TESTING index d2c97d9..fca23d7 100644 --- a/TESTING +++ b/TESTING @@ -13,7 +13,9 @@ INSTALL. # pacman -S php php-sqlite sqlite -3) Prepare the testing database: +3) Uncomment "extension=pdo_sqlite" in /etc/php/php.ini + +4) Prepare the testing database: $ cd /path/to/aurweb/schema $ make @@ -21,12 +23,12 @@ INSTALL. $ sqlite3 ../aurweb.sqlite3 < aur-schema-sqlite.sql $ sqlite3 ../aurweb.sqlite3 < out.sql -4) Copy conf/config.proto to conf/config and adjust the configuration +5) Copy conf/config.proto to conf/config and adjust the configuration (pay attention to disable_http_login, enable_maintenance and aur_location). - Be sure to change backend to sqlite and name to the file location of your - created test database. + Be sure to set "backend" to "sqlite" and "name" to the file location of your + created test database. An absolute path is recommended. -5) Run the PHP built-in web server: +6) Run the PHP built-in web server: $ AUR_CONFIG='/path/to/aurweb/conf/config' php -S localhost:8080 -t /path/to/aurweb/web/html -- 2.16.1

2 2