- Aur-dev - lists.archlinux.org

[aur-dev] [PATCH] Track providers in the official repositories
by Lukas Fleischer 22 Oct '15

22 Oct '15

Maintain a list of virtual provisions of packages from the official binary package repositories. The list can be updated using the aurblup script, e.g. via a cronjob. This allows for adding proper links to package dependencies: If an AUR package depends on a package from the official repositories (or on a name provided by a package from the official repositories), add a link to the corresponding archweb package details page. If an AUR package depends on another AUR package (or on a name provided by another AUR package), add a link to the corresponding aurweb package details page. Otherwise, just display the name and do not add a link at all. Fixes FS#46549. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- schema/aur-schema.sql | 10 ++++++++ scripts/aurblup.py | 13 ++++++++++ upgrading/4.2.0.txt | 11 +++++++++ web/lib/pkgfuncs.inc.php | 64 ++++++++++++++++++++++++++++++++++++------------ 4 files changed, 82 insertions(+), 16 deletions(-) create mode 100644 upgrading/4.2.0.txt diff --git a/schema/aur-schema.sql b/schema/aur-schema.sql index 53dc468..0a0c806 100644 --- a/schema/aur-schema.sql +++ b/schema/aur-schema.sql @@ -301,6 +301,16 @@ CREATE TABLE PackageBlacklist ( UNIQUE (Name) ) ENGINE = InnoDB; +-- Providers in the official repositories +-- +CREATE TABLE OfficialProviders ( + ID INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, + Name VARCHAR(64) NOT NULL, + Provides VARCHAR(64) NOT NULL, + PRIMARY KEY (ID) +) ENGINE = InnoDB; +CREATE UNIQUE INDEX ProviderNameProvides ON OfficialProviders (Name, Provides); + -- Define package request types -- CREATE TABLE RequestTypes ( diff --git a/scripts/aurblup.py b/scripts/aurblup.py index d6d0c3c..5397528 100755 --- a/scripts/aurblup.py +++ b/scripts/aurblup.py @@ -18,6 +18,7 @@ sync_dbs = config.get('aurblup', 'sync-dbs').split(' ') servers = config.get('aurblup', 'servers').split(' ') blacklist = set() +providers = set() h = pyalpm.Handle("/", db_path) for sync_db in sync_dbs: @@ -30,6 +31,8 @@ for sync_db in sync_dbs: for pkg in repo.pkgcache: blacklist.add(pkg.name) [blacklist.add(x) for x in pkg.replaces] + providers.add((pkg.name, pkg.name)) + [providers.add((pkg.name, x)) for x in pkg.provides] db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, passwd=aur_db_pass, db=aur_db_name, @@ -44,5 +47,15 @@ for pkg in blacklist.difference(oldblacklist): for pkg in oldblacklist.difference(blacklist): cur.execute("DELETE FROM PackageBlacklist WHERE Name = %s", [pkg]) +cur.execute("SELECT Name, Provides FROM OfficialProviders") +oldproviders = set(cur.fetchall()) + +for pkg, provides in providers.difference(oldproviders): + cur.execute("INSERT INTO OfficialProviders (Name, Provides) " + "VALUES (%s, %s)", [pkg, provides]) +for pkg, provides in oldproviders.difference(providers): + cur.execute("DELETE FROM OfficialProviders " + "WHERE Name = %s AND Provides = %s", [pkg, provides]) + db.commit() db.close() diff --git a/upgrading/4.2.0.txt b/upgrading/4.2.0.txt new file mode 100644 index 0000000..37cbeae --- /dev/null +++ b/upgrading/4.2.0.txt @@ -0,0 +1,11 @@ +1. Add a new table to store providers from official packages: + +---- +CREATE TABLE OfficialProviders ( + ID INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, + Name VARCHAR(64) NOT NULL, + Provides VARCHAR(64) NOT NULL, + PRIMARY KEY (ID) +) ENGINE = InnoDB; +CREATE UNIQUE INDEX ProviderNameProvides ON OfficialProviders (Name, Provides); +---- diff --git a/web/lib/pkgfuncs.inc.php b/web/lib/pkgfuncs.inc.php index 66bc249..717085d 100644 --- a/web/lib/pkgfuncs.inc.php +++ b/web/lib/pkgfuncs.inc.php @@ -168,6 +168,9 @@ function pkg_providers($name) { $q.= "INNER JOIN RelationTypes rt ON rt.ID = pr.RelTypeID "; $q.= "WHERE rt.Name = 'provides' "; $q.= "AND pr.RelName = " . $dbh->quote($name); + $q.= "UNION "; + $q.= "SELECT 0, Name FROM OfficialProviders "; + $q.= "WHERE Provides = " . $dbh->quote($name); $result = $dbh->query($q); if (!$result) { @@ -281,6 +284,29 @@ function pkg_deplink_annotation($type, $arch, $desc=false) { } /** + * Get the HTML code to display a package provider link + * + * @param string $name The name of the provider + * @param bool $official True if the package is in the official repositories + * + * @return string The HTML code of the link to display + */ +function pkg_provider_link($name, $official) { + $link = '<a href="'; + if ($official) { + $link .= 'https://www.archlinux.org/packages/?q=' . + urlencode($name); + } else { + $link .= htmlspecialchars(get_pkg_uri($name), ENT_QUOTES); + } + $link .= '" title="' . __('View packages details for') . ' '; + $link .= htmlspecialchars($name) . '">'; + $link .= htmlspecialchars($name) . '</a>'; + + return $link; +} + +/** * Get the HTML code to display a package dependency link * * @param string $name The name of the dependency @@ -312,31 +338,37 @@ function pkg_depend_link($name, $type, $cond, $arch, $pkg_id) { $providers = pkg_providers($name); } + $link = htmlspecialchars($name); + foreach ($providers as $provider) { + if ($provider[1] == $name) { + $is_official = ($provider[0] == 0); + $name = $provider[1]; + $link = pkg_provider_link($name, $is_official); + break; + } + } + $link .= ' ' . htmlspecialchars($cond); + + foreach ($providers as $key => $provider) { + if ($provider[1] == $name) { + unset($providers[$key]); + } + } + if (count($providers) > 0) { - $link = htmlspecialchars($name) . ' '; $link .= '('; foreach ($providers as $provider) { + $is_official = ($provider[0] == 0); $name = $provider[1]; - $link .= '<a href="'; - $link .= htmlspecialchars(get_pkg_uri($name), ENT_QUOTES); - $link .= '" title="' . __('View packages details for') .' ' . htmlspecialchars($name) . '">'; - $link .= htmlspecialchars($name) . '</a>, '; + $link .= pkg_provider_link($name, $is_official) . ', '; } $link = substr($link, 0, -2); $link .= ')'; - } else { - $link = '<a href="'; - if (is_null($pkg_id)) { - $link .= 'https://www.archlinux.org/packages/?q=' . urlencode($name); - } else { - $link .= htmlspecialchars(get_pkg_uri($name), ENT_QUOTES); - } - $link .= '" title="' . __('View packages details for') .' ' . htmlspecialchars($name) . '">'; - $link .= htmlspecialchars($name) . '</a>'; - $link .= htmlspecialchars($cond); } - return $link . pkg_deplink_annotation($type, $arch, $desc); + $link .= pkg_deplink_annotation($type, $arch, $desc); + + return $link; } /** -- 2.6.1

3 3

[aur-dev] [PATCH 1/2] Check comment length in the backend
by Lukas Fleischer 22 Oct '15

22 Oct '15

Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- web/html/pkgbase.php | 7 +------ web/lib/pkgbasefuncs.inc.php | 4 ++++ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/web/html/pkgbase.php b/web/html/pkgbase.php index b98dc64..cbbf3cc 100644 --- a/web/html/pkgbase.php +++ b/web/html/pkgbase.php @@ -49,12 +49,7 @@ $output = ""; $fragment = ""; if (check_token()) { if (current_action("do_Flag")) { - if (strlen($_POST['comments']) >= 3) { - list($ret, $output) = pkgbase_flag($ids, $_POST['comments']); - } else { - $output = __("The selected packages have not been flagged, please enter a comment."); - $ret = false; - } + list($ret, $output) = pkgbase_flag($ids, $_POST['comments']); } elseif (current_action("do_UnFlag")) { list($ret, $output) = pkgbase_unflag($ids); } elseif (current_action("do_Adopt")) { diff --git a/web/lib/pkgbasefuncs.inc.php b/web/lib/pkgbasefuncs.inc.php index aad9d14..afccc7d 100644 --- a/web/lib/pkgbasefuncs.inc.php +++ b/web/lib/pkgbasefuncs.inc.php @@ -330,6 +330,10 @@ function pkgbase_flag($base_ids, $comment) { return array(false, __("You did not select any packages to flag.")); } + if (strlen($comment) < 3) { + return array(false, __("The selected packages have not been flagged, please enter a comment.")); + } + $uid = uid_from_sid($_COOKIE['AURSID']); $dbh = DB::connect(); -- 2.6.1

1 1

[aur-dev] [PATCH 1/2] Refactor pkgbase_set_comaintainers()
by Lukas Fleischer 17 Oct '15

17 Oct '15

This makes the code slightly more efficient and allows for easily determining the users that were added/removed to the co-maintainer list. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- web/lib/pkgbasefuncs.inc.php | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/web/lib/pkgbasefuncs.inc.php b/web/lib/pkgbasefuncs.inc.php index cb756f6..bb9d241 100644 --- a/web/lib/pkgbasefuncs.inc.php +++ b/web/lib/pkgbasefuncs.inc.php @@ -1023,7 +1023,7 @@ function pkgbase_set_comaintainers($base_id, $users) { $dbh = DB::connect(); - $uids = array(); + $uids_new = array(); foreach($users as $user) { $q = "SELECT ID FROM Users "; $q .= "WHERE UserName = " . $dbh->quote($user); @@ -1034,18 +1034,32 @@ function pkgbase_set_comaintainers($base_id, $users) { return array(false, __("Invalid user name: %s", $user)); } - $uids[] = $uid; + $uids_new[] = $uid; } - $q = sprintf("DELETE FROM PackageComaintainers WHERE PackageBaseID = %d", $base_id); - $dbh->exec($q); + $q = sprintf("SELECT UsersID FROM PackageComaintainers WHERE PackageBaseID = %d", $base_id); + $result = $dbh->query($q); + $uids_old = $result->fetchAll(PDO::FETCH_COLUMN, 0); + + $uids_add = array_diff($uids_new, $uids_old); + $uids_rem = array_diff($uids_old, $uids_new); $i = 1; - foreach ($uids as $uid) { - $q = sprintf("INSERT INTO PackageComaintainers (PackageBaseID, UsersID, Priority) VALUES (%d, %d, %d)", $base_id, $uid, $i); + foreach ($uids_new as $uid) { + if (in_array($uid, $uids_add)) { + $q = sprintf("INSERT INTO PackageComaintainers (PackageBaseID, UsersID, Priority) VALUES (%d, %d, %d)", $base_id, $uid, $i); + } else { + $q = sprintf("UPDATE PackageComaintainers SET Priority = %d WHERE PackageBaseID = %d AND UsersID = %d", $i, $base_id, $uid); + } + $dbh->exec($q); $i++; } + foreach ($uids_rem as $uid) { + $q = sprintf("DELETE FROM PackageComaintainers WHERE PackageBaseID = %d AND UsersID = %d", $base_id, $uid); + $dbh->exec($q); + } + return array(true, __("The package base co-maintainers have been updated.")); } -- 2.6.1

1 1

[aur-dev] [PATCH 1/2] notify: Split out email header generation
by Lukas Fleischer 17 Oct '15

17 Oct '15

Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- scripts/notify.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/scripts/notify.py b/scripts/notify.py index 56c15d5..a0ddbda 100755 --- a/scripts/notify.py +++ b/scripts/notify.py @@ -26,7 +26,13 @@ sender = config.get('notifications', 'sender') reply_to = config.get('notifications', 'reply-to') -def send_notification(to, subject, body, refs, cc=None, reference=None): +def headers_cc(cclist): + return {'Cc': str.join(', ', cclist)} + +def headers_reply(thread_id): + return {'In-Reply-To': thread_id, 'References': thread_id} + +def send_notification(to, subject, body, refs, headers={}): body = '\n'.join([textwrap.fill(line) for line in body.splitlines()]) body += '\n\n' + refs @@ -37,12 +43,8 @@ def send_notification(to, subject, body, refs, cc=None, reference=None): msg['Reply-to'] = reply_to msg['To'] = recipient - if cc: - msg['Cc'] = str.join(', ', cc) - - if reference: - msg['In-Reply-To'] = reference - msg['References'] = reference + for key, value in header.items(): + msg[key] = value p = subprocess.Popen([sendmail, '-t', '-oi'], stdin=subprocess.PIPE) p.communicate(msg.as_bytes()) @@ -130,8 +132,9 @@ def comment(cur, uid, pkgbase_id): refs = '[1] ' + user_uri + '\n' refs += '[2] ' + pkgbase_uri thread_id = '<pkg-notifications-' + pkgbase + '@aur.archlinux.org>' + headers = headers_reply(thread_id) - send_notification(to, subject, body, refs, reference=thread_id) + send_notification(to, subject, body, refs, headers) def flag(cur, uid, pkgbase_id): user = username_from_id(cur, uid) @@ -207,8 +210,9 @@ def request_open(cur, uid, reqid, reqtype, pkgbase_id, merge_into=None): refs = '[1] ' + user_uri + '\n' refs += '[2] ' + pkgbase_uri + '\n' thread_id = '<pkg-request-' + reqid + '@aur.archlinux.org>' + headers = headers_reply(thread_id) + headers_cc(cc) - send_notification(to, subject, body, refs, cc, thread_id) + send_notification(to, subject, body, refs, headers) def request_close(cur, uid, reqid, reason): user = username_from_id(cur, uid) @@ -227,8 +231,9 @@ def request_close(cur, uid, reqid, reason): body += ':\n\n' + text refs = '[1] ' + user_uri thread_id = '<pkg-request-' + reqid + '@aur.archlinux.org>' + headers = headers_reply(thread_id) + headers_cc(cc) - send_notification(to, subject, body, refs, cc, thread_id) + send_notification(to, subject, body, refs, headers) if __name__ == '__main__': -- 2.6.1

1 2

[aur-dev] [PATCH] Make copyright notice translatable
by Lukas Fleischer 17 Oct '15

17 Oct '15

Fixes FS#46747. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- web/template/footer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/template/footer.php b/web/template/footer.php index 1b3c936..7e505a3 100644 --- a/web/template/footer.php +++ b/web/template/footer.php @@ -5,7 +5,7 @@ <?php if ($ver): ?> aurweb <a href="https://projects.archlinux.org/aurweb.git/log/?h=<?= htmlspecialchars($ver, ENT_QUOTES) ?>"><?= htmlspecialchars($ver) ?></a> <?php endif; ?> - Copyright © 2004-<?= date("Y") ?> aurweb Development Team. + <?= __('Copyright %s 2004-%d aurweb Development Team.', '©', date('Y')) <?= __('AUR packages are user produced content. Any use of the provided files is at your own risk.') ?> </div> </div> -- 2.6.1

1 0

[aur-dev] [PATCH] notify: Fix welcome/flag notifications
by Lukas Fleischer 17 Oct '15

17 Oct '15

Fixes two regressions introduced in commit 6681e56 (notify: Do not wrap references, 2015-10-03). Fixes FS#46742. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- scripts/notify.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/notify.py b/scripts/notify.py index e9026ba..1be0117 100755 --- a/scripts/notify.py +++ b/scripts/notify.py @@ -111,7 +111,7 @@ def welcome(cur, uid): 'password for your new account, please click the link [1] below. ' \ 'If the link does not work, try copying and pasting it into your ' \ 'browser.' - body = '[1] ' + aur_location + '/passreset/?resetkey=' + resetkey + refs = '[1] ' + aur_location + '/passreset/?resetkey=' + resetkey send_notification([to], subject, body, refs) @@ -151,11 +151,11 @@ def flag(cur, uid, pkgbase_id): subject = 'AUR Out-of-date Notification for %s' % (pkgbase) body = 'Your package %s [1] has been flagged out-of-date by %s [2]:' % \ (pkgbase, user) - body += '\n\n' + text + '\n\n' - body += '[1] ' + pkgbase_uri + '\n' - body += '[2] ' + user_uri + body += '\n\n' + text + refs = '[1] ' + pkgbase_uri + '\n' + refs += '[2] ' + user_uri - send_notification(to, subject, body) + send_notification(to, subject, body, refs) def delete(cur, uid, old_pkgbase_id, new_pkgbase_id=None): user = username_from_id(cur, uid) -- 2.6.1

1 0

[aur-dev] Setting keywords from the command line
by Lukas Fleischer 16 Oct '15

16 Oct '15

Hi, Currently, in order to change the keywords of a package base, users need to log into the web interface and submit a form on the package base details page. It would be nice to be able to change keywords without using a web browser, see FS#45627 [1]. Now, there are two options: 1. Add a set-keywords command to the aurweb SSH interface such that users can run `ssh aur.archlinux.org set-keywords foobar keyword1 keyword2 keyword3` to change keywords (or still use the web interface). 2. Remove the option from the web interface and introduce a new file (.KEYWORDS or something more general like .AURINFO, .AURMETADATA) that is parsed in the Git update hook. Obviously, with the second option, one would get a history of keyword changes. On the other hand, some might argue that this will clutter the commit history (although keyword changes should not happen too often) and that this changes makes the AUR package repositories special in a certain way (currently there is nothing AUR-specific in the Git repositories itself, an implementation of the second idea will add such information). Are there any opinions on which option should be implemented? Any other advantages/disadvantages I did not think of? Regards, Lukas [1] https://bugs.archlinux.org/task/45627

3 2

[aur-dev] [PATCH] Minor correction in INSTALL
by Ian D. Scott 10 Oct '15

10 Oct '15

--- INSTALL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL b/INSTALL index d68fa26..b151e27 100644 --- a/INSTALL +++ b/INSTALL @@ -46,7 +46,7 @@ Setup on Arch Linux # mkdir /srv/http/aurweb/aur.git/ # cd /srv/http/aurweb/aur.git/ # git init --bare - # ln -s ../../git-interface/git-update.py hooks/update + # ln -s ../git-interface/git-update.py hooks/update # chown -R aur . 7) Install the git-auth wrapper script: -- 2.6.1

2 4

[aur-dev] [PATCH] aurjson: Rename the search_by parameter to "by"
by Lukas Fleischer 10 Oct '15

10 Oct '15

This parameter is only supported by the search command. We do not need to repeat ourselves. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- doc/rpc.txt | 13 ++++++++----- web/lib/aurjson.class.php | 14 +++++++++----- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/doc/rpc.txt b/doc/rpc.txt index ee1fa1e..f353ff0 100644 --- a/doc/rpc.txt +++ b/doc/rpc.txt @@ -5,11 +5,14 @@ Package Search -------------- Package searches can be performed by issuing HTTP GET requests of the form -+/rpc/?v=5&type=search&search_by=_by_&arg=_keywords_+ where _by_ is either -`name` (search by package name only), `name-desc` (search by package name and -description) or `maintainer` (search by package maintainer) and _keywords_ is -the search argument. The _search_by_ parameter can be skipped and defaults to -`name-desc`. ++/rpc/?v=5&type=search&by=_field_&arg=_keywords_+ where _keywords_ is the +search argument and _field_ is one of the following values: + +* `name` (search by package name only) +* `name-desc` (search by package name and description) +* `maintainer` (search by package maintainer) + +The _by_ parameter can be skipped and defaults to `name-desc`. If a maintainer search is performed and the search argument is left empty, a list of orphan packages is returned. diff --git a/web/lib/aurjson.class.php b/web/lib/aurjson.class.php index 2bf2e7a..1548232 100644 --- a/web/lib/aurjson.class.php +++ b/web/lib/aurjson.class.php @@ -86,8 +86,12 @@ class AurJSON { if (!in_array($http_data['type'], self::$exposed_methods)) { return $this->json_error('Incorrect request type specified.'); } - if (isset($http_data['search_by']) && !in_array($http_data['search_by'], self::$exposed_fields)) { - return $this->json_error('Incorrect search_by field specified.'); + + if (isset($http_data['search_by']) && !isset($http_data['by'])) { + $http_data['by'] = $http_data['search_by']; + } + if (isset($http_data['by']) && !in_array($http_data['by'], self::$exposed_fields)) { + return $this->json_error('Incorrect by field specified.'); } $this->dbh = DB::connect(); @@ -362,8 +366,8 @@ class AurJSON { private function search($http_data) { $keyword_string = $http_data['arg']; - if (isset($http_data['search_by'])) { - $search_by = $http_data['search_by']; + if (isset($http_data['by'])) { + $search_by = $http_data['by']; } else { $search_by = 'name-desc'; } @@ -455,7 +459,7 @@ class AurJSON { * @return mixed Returns an array of value data containing the package data */ private function msearch($http_data) { - $http_data['search_by'] = 'maintainer'; + $http_data['by'] = 'maintainer'; return $this->search($http_data); } -- 2.6.1

1 0

[aur-dev] [PATCH 1/1] use square brackets for annotations in pkg dependencies
by Christian Hesse 10 Oct '15

10 Oct '15

From: Christian Hesse <mail(a)eworm.de> Currently a package depending on `markdown` will have a listed dependency that looks like this: markdown (discount) (make) Where `dicount` is a package that provides `markdown`, make is an annotation to express the type of dependency. Let's use square brackets for annotations in pkg dependencies, so this looks like: markdown (discount) [make] Signed-off-by: Christian Hesse <mail(a)eworm.de> --- web/lib/pkgfuncs.inc.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/lib/pkgfuncs.inc.php b/web/lib/pkgfuncs.inc.php index 66bc249..b1a271b 100644 --- a/web/lib/pkgfuncs.inc.php +++ b/web/lib/pkgfuncs.inc.php @@ -253,7 +253,7 @@ function pkg_deplink_annotation($type, $arch, $desc=false) { return ''; } - $link = ' ('; + $link = ' ['; if ($type == 'makedepends') { $link .= 'make'; @@ -271,7 +271,7 @@ function pkg_deplink_annotation($type, $arch, $desc=false) { $link .= htmlspecialchars($arch); } - $link .= ')'; + $link .= ']'; if ($type == 'optdepends' && $desc) { $link .= ' – ' . htmlspecialchars($desc) . ' '; } -- 2.6.1

2 1