October 2015 - Aur-dev - lists.archlinux.org

[aur-dev] [PATCH] aurjson: Allow underscores in JSONP callback names
by Lukas Fleischer 24 Oct '15

24 Oct '15

Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- web/lib/aurjson.class.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/lib/aurjson.class.php b/web/lib/aurjson.class.php index 1548232..9097035 100644 --- a/web/lib/aurjson.class.php +++ b/web/lib/aurjson.class.php @@ -119,7 +119,7 @@ class AurJSON { if (isset($http_data['callback'])) { $callback = $http_data['callback']; - if (!preg_match('/^[a-zA-Z0-9().]{1,128}$/D', $callback)) { + if (!preg_match('/^[a-zA-Z0-9()_.]{1,128}$/D', $callback)) { return $this->json_error('Invalid callback name.'); } header('content-type: text/javascript'); -- 2.6.2

1 0

[aur-dev] [PATCH] git-serve: Add support for setting keywords
by Lukas Fleischer 24 Oct '15

24 Oct '15

This allows for setting keywords using the SSH interface. The syntax is `config <pkgbase> keywords <keyword1> <keyword2>...`. Implements FS#45627. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- git-interface/git-serve.py | 55 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/git-interface/git-serve.py b/git-interface/git-serve.py index 45c9a01..8bd6aa8 100755 --- a/git-interface/git-serve.py +++ b/git-interface/git-serve.py @@ -25,17 +25,19 @@ ssh_cmdline = config.get('serve', 'ssh-cmdline') enable_maintenance = config.getboolean('options', 'enable-maintenance') maintenance_exc = config.get('options', 'maintenance-exceptions').split() -def pkgbase_exists(pkgbase): +def pkgbase_from_name(pkgbase): db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, passwd=aur_db_pass, db=aur_db_name, unix_socket=aur_db_socket) cur = db.cursor() + cur.execute("SELECT ID FROM PackageBases WHERE Name = %s", [pkgbase]) + db.close() - cur.execute("SELECT COUNT(*) FROM PackageBases WHERE Name = %s ", - [pkgbase]) + row = cur.fetchone() + return row[0] if row else None - db.close() - return (cur.fetchone()[0] > 0) +def pkgbase_exists(pkgbase): + return (pkgbase_from_name(pkgbase) > 0) def list_repos(user): db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, @@ -81,6 +83,25 @@ def create_pkgbase(pkgbase, user): db.commit() db.close() +def pkgbase_config_keywords(pkgbase, keywords): + pkgbase_id = pkgbase_from_name(pkgbase) + if not pkgbase_id: + die('{:s}: package base not found: {:s}'.format(action, pkgbase)) + + db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket) + cur = db.cursor() + + cur.execute("DELETE FROM PackageKeywords WHERE PackageBaseID = %s", + [pkgbase_id]) + for keyword in keywords: + cur.execute("INSERT INTO PackageKeywords (PackageBaseID, Keyword) " + "VALUES (%s, %s)", [pkgbase_id, keyword]) + + db.commit() + db.close() + def check_permissions(pkgbase, user): db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, passwd=aur_db_pass, db=aur_db_name, @@ -143,6 +164,17 @@ if action == 'git-upload-pack' or action == 'git-receive-pack': os.environ["GIT_NAMESPACE"] = pkgbase cmd = action + " '" + repo_path + "'" os.execl(git_shell_cmd, git_shell_cmd, '-c', cmd) +elif action == 'config': + if len(cmdargv) < 2: + die_with_help("{:s}: missing repository name".format(action)) + if len(cmdargv) < 3: + die_with_help("{:s}: missing option name".format(action)) + pkgbase = cmdargv[1] + option = cmdargv[2] + + if option == 'keywords': + pkgbase_config_keywords(pkgbase, cmdargv[3:]) + elif action == 'list-repos': if len(cmdargv) > 1: die_with_help("{:s}: too many arguments".format(action)) @@ -172,11 +204,12 @@ elif action == 'restore': os.execl(git_update_cmd, git_update_cmd, 'restore') elif action == 'help': die("Commands:\n" + - " help Show this help message and exit.\n" + - " list-repos List all your repositories.\n" + - " restore <name> Restore a deleted package base.\n" + - " setup-repo <name> Create an empty repository.\n" + - " git-receive-pack Internal command used with Git.\n" + - " git-upload-pack Internal command used with Git.") + " config <name> <param>... Change package base settings.\n" + + " help Show this help message and exit.\n" + + " list-repos List all your repositories.\n" + + " restore <name> Restore a deleted package base.\n" + + " setup-repo <name> Create an empty repository.\n" + + " git-receive-pack Internal command used with Git.\n" + + " git-upload-pack Internal command used with Git.") else: die_with_help("invalid command: {:s}".format(action)) -- 2.6.2

1 0

[aur-dev] [PATCH] Track providers in the official repositories
by Lukas Fleischer 22 Oct '15

22 Oct '15

Maintain a list of virtual provisions of packages from the official binary package repositories. The list can be updated using the aurblup script, e.g. via a cronjob. This allows for adding proper links to package dependencies: If an AUR package depends on a package from the official repositories (or on a name provided by a package from the official repositories), add a link to the corresponding archweb package details page. If an AUR package depends on another AUR package (or on a name provided by another AUR package), add a link to the corresponding aurweb package details page. Otherwise, just display the name and do not add a link at all. Fixes FS#46549. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- schema/aur-schema.sql | 10 ++++++++ scripts/aurblup.py | 13 ++++++++++ upgrading/4.2.0.txt | 11 +++++++++ web/lib/pkgfuncs.inc.php | 64 ++++++++++++++++++++++++++++++++++++------------ 4 files changed, 82 insertions(+), 16 deletions(-) create mode 100644 upgrading/4.2.0.txt diff --git a/schema/aur-schema.sql b/schema/aur-schema.sql index 53dc468..0a0c806 100644 --- a/schema/aur-schema.sql +++ b/schema/aur-schema.sql @@ -301,6 +301,16 @@ CREATE TABLE PackageBlacklist ( UNIQUE (Name) ) ENGINE = InnoDB; +-- Providers in the official repositories +-- +CREATE TABLE OfficialProviders ( + ID INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, + Name VARCHAR(64) NOT NULL, + Provides VARCHAR(64) NOT NULL, + PRIMARY KEY (ID) +) ENGINE = InnoDB; +CREATE UNIQUE INDEX ProviderNameProvides ON OfficialProviders (Name, Provides); + -- Define package request types -- CREATE TABLE RequestTypes ( diff --git a/scripts/aurblup.py b/scripts/aurblup.py index d6d0c3c..5397528 100755 --- a/scripts/aurblup.py +++ b/scripts/aurblup.py @@ -18,6 +18,7 @@ sync_dbs = config.get('aurblup', 'sync-dbs').split(' ') servers = config.get('aurblup', 'servers').split(' ') blacklist = set() +providers = set() h = pyalpm.Handle("/", db_path) for sync_db in sync_dbs: @@ -30,6 +31,8 @@ for sync_db in sync_dbs: for pkg in repo.pkgcache: blacklist.add(pkg.name) [blacklist.add(x) for x in pkg.replaces] + providers.add((pkg.name, pkg.name)) + [providers.add((pkg.name, x)) for x in pkg.provides] db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, passwd=aur_db_pass, db=aur_db_name, @@ -44,5 +47,15 @@ for pkg in blacklist.difference(oldblacklist): for pkg in oldblacklist.difference(blacklist): cur.execute("DELETE FROM PackageBlacklist WHERE Name = %s", [pkg]) +cur.execute("SELECT Name, Provides FROM OfficialProviders") +oldproviders = set(cur.fetchall()) + +for pkg, provides in providers.difference(oldproviders): + cur.execute("INSERT INTO OfficialProviders (Name, Provides) " + "VALUES (%s, %s)", [pkg, provides]) +for pkg, provides in oldproviders.difference(providers): + cur.execute("DELETE FROM OfficialProviders " + "WHERE Name = %s AND Provides = %s", [pkg, provides]) + db.commit() db.close() diff --git a/upgrading/4.2.0.txt b/upgrading/4.2.0.txt new file mode 100644 index 0000000..37cbeae --- /dev/null +++ b/upgrading/4.2.0.txt @@ -0,0 +1,11 @@ +1. Add a new table to store providers from official packages: + +---- +CREATE TABLE OfficialProviders ( + ID INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, + Name VARCHAR(64) NOT NULL, + Provides VARCHAR(64) NOT NULL, + PRIMARY KEY (ID) +) ENGINE = InnoDB; +CREATE UNIQUE INDEX ProviderNameProvides ON OfficialProviders (Name, Provides); +---- diff --git a/web/lib/pkgfuncs.inc.php b/web/lib/pkgfuncs.inc.php index 66bc249..717085d 100644 --- a/web/lib/pkgfuncs.inc.php +++ b/web/lib/pkgfuncs.inc.php @@ -168,6 +168,9 @@ function pkg_providers($name) { $q.= "INNER JOIN RelationTypes rt ON rt.ID = pr.RelTypeID "; $q.= "WHERE rt.Name = 'provides' "; $q.= "AND pr.RelName = " . $dbh->quote($name); + $q.= "UNION "; + $q.= "SELECT 0, Name FROM OfficialProviders "; + $q.= "WHERE Provides = " . $dbh->quote($name); $result = $dbh->query($q); if (!$result) { @@ -281,6 +284,29 @@ function pkg_deplink_annotation($type, $arch, $desc=false) { } /** + * Get the HTML code to display a package provider link + * + * @param string $name The name of the provider + * @param bool $official True if the package is in the official repositories + * + * @return string The HTML code of the link to display + */ +function pkg_provider_link($name, $official) { + $link = '<a href="'; + if ($official) { + $link .= 'https://www.archlinux.org/packages/?q=' . + urlencode($name); + } else { + $link .= htmlspecialchars(get_pkg_uri($name), ENT_QUOTES); + } + $link .= '" title="' . __('View packages details for') . ' '; + $link .= htmlspecialchars($name) . '">'; + $link .= htmlspecialchars($name) . '</a>'; + + return $link; +} + +/** * Get the HTML code to display a package dependency link * * @param string $name The name of the dependency @@ -312,31 +338,37 @@ function pkg_depend_link($name, $type, $cond, $arch, $pkg_id) { $providers = pkg_providers($name); } + $link = htmlspecialchars($name); + foreach ($providers as $provider) { + if ($provider[1] == $name) { + $is_official = ($provider[0] == 0); + $name = $provider[1]; + $link = pkg_provider_link($name, $is_official); + break; + } + } + $link .= ' ' . htmlspecialchars($cond); + + foreach ($providers as $key => $provider) { + if ($provider[1] == $name) { + unset($providers[$key]); + } + } + if (count($providers) > 0) { - $link = htmlspecialchars($name) . ' '; $link .= '('; foreach ($providers as $provider) { + $is_official = ($provider[0] == 0); $name = $provider[1]; - $link .= '<a href="'; - $link .= htmlspecialchars(get_pkg_uri($name), ENT_QUOTES); - $link .= '" title="' . __('View packages details for') .' ' . htmlspecialchars($name) . '">'; - $link .= htmlspecialchars($name) . '</a>, '; + $link .= pkg_provider_link($name, $is_official) . ', '; } $link = substr($link, 0, -2); $link .= ')'; - } else { - $link = '<a href="'; - if (is_null($pkg_id)) { - $link .= 'https://www.archlinux.org/packages/?q=' . urlencode($name); - } else { - $link .= htmlspecialchars(get_pkg_uri($name), ENT_QUOTES); - } - $link .= '" title="' . __('View packages details for') .' ' . htmlspecialchars($name) . '">'; - $link .= htmlspecialchars($name) . '</a>'; - $link .= htmlspecialchars($cond); } - return $link . pkg_deplink_annotation($type, $arch, $desc); + $link .= pkg_deplink_annotation($type, $arch, $desc); + + return $link; } /** -- 2.6.1

3 3

[aur-dev] [PATCH 1/2] Check comment length in the backend
by Lukas Fleischer 22 Oct '15

22 Oct '15

Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- web/html/pkgbase.php | 7 +------ web/lib/pkgbasefuncs.inc.php | 4 ++++ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/web/html/pkgbase.php b/web/html/pkgbase.php index b98dc64..cbbf3cc 100644 --- a/web/html/pkgbase.php +++ b/web/html/pkgbase.php @@ -49,12 +49,7 @@ $output = ""; $fragment = ""; if (check_token()) { if (current_action("do_Flag")) { - if (strlen($_POST['comments']) >= 3) { - list($ret, $output) = pkgbase_flag($ids, $_POST['comments']); - } else { - $output = __("The selected packages have not been flagged, please enter a comment."); - $ret = false; - } + list($ret, $output) = pkgbase_flag($ids, $_POST['comments']); } elseif (current_action("do_UnFlag")) { list($ret, $output) = pkgbase_unflag($ids); } elseif (current_action("do_Adopt")) { diff --git a/web/lib/pkgbasefuncs.inc.php b/web/lib/pkgbasefuncs.inc.php index aad9d14..afccc7d 100644 --- a/web/lib/pkgbasefuncs.inc.php +++ b/web/lib/pkgbasefuncs.inc.php @@ -330,6 +330,10 @@ function pkgbase_flag($base_ids, $comment) { return array(false, __("You did not select any packages to flag.")); } + if (strlen($comment) < 3) { + return array(false, __("The selected packages have not been flagged, please enter a comment.")); + } + $uid = uid_from_sid($_COOKIE['AURSID']); $dbh = DB::connect(); -- 2.6.1

1 1

[aur-dev] [PATCH 1/2] Refactor pkgbase_set_comaintainers()
by Lukas Fleischer 17 Oct '15

17 Oct '15

This makes the code slightly more efficient and allows for easily determining the users that were added/removed to the co-maintainer list. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- web/lib/pkgbasefuncs.inc.php | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/web/lib/pkgbasefuncs.inc.php b/web/lib/pkgbasefuncs.inc.php index cb756f6..bb9d241 100644 --- a/web/lib/pkgbasefuncs.inc.php +++ b/web/lib/pkgbasefuncs.inc.php @@ -1023,7 +1023,7 @@ function pkgbase_set_comaintainers($base_id, $users) { $dbh = DB::connect(); - $uids = array(); + $uids_new = array(); foreach($users as $user) { $q = "SELECT ID FROM Users "; $q .= "WHERE UserName = " . $dbh->quote($user); @@ -1034,18 +1034,32 @@ function pkgbase_set_comaintainers($base_id, $users) { return array(false, __("Invalid user name: %s", $user)); } - $uids[] = $uid; + $uids_new[] = $uid; } - $q = sprintf("DELETE FROM PackageComaintainers WHERE PackageBaseID = %d", $base_id); - $dbh->exec($q); + $q = sprintf("SELECT UsersID FROM PackageComaintainers WHERE PackageBaseID = %d", $base_id); + $result = $dbh->query($q); + $uids_old = $result->fetchAll(PDO::FETCH_COLUMN, 0); + + $uids_add = array_diff($uids_new, $uids_old); + $uids_rem = array_diff($uids_old, $uids_new); $i = 1; - foreach ($uids as $uid) { - $q = sprintf("INSERT INTO PackageComaintainers (PackageBaseID, UsersID, Priority) VALUES (%d, %d, %d)", $base_id, $uid, $i); + foreach ($uids_new as $uid) { + if (in_array($uid, $uids_add)) { + $q = sprintf("INSERT INTO PackageComaintainers (PackageBaseID, UsersID, Priority) VALUES (%d, %d, %d)", $base_id, $uid, $i); + } else { + $q = sprintf("UPDATE PackageComaintainers SET Priority = %d WHERE PackageBaseID = %d AND UsersID = %d", $i, $base_id, $uid); + } + $dbh->exec($q); $i++; } + foreach ($uids_rem as $uid) { + $q = sprintf("DELETE FROM PackageComaintainers WHERE PackageBaseID = %d AND UsersID = %d", $base_id, $uid); + $dbh->exec($q); + } + return array(true, __("The package base co-maintainers have been updated.")); } -- 2.6.1

1 1

[aur-dev] [PATCH 1/2] notify: Split out email header generation
by Lukas Fleischer 17 Oct '15

17 Oct '15

Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- scripts/notify.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/scripts/notify.py b/scripts/notify.py index 56c15d5..a0ddbda 100755 --- a/scripts/notify.py +++ b/scripts/notify.py @@ -26,7 +26,13 @@ sender = config.get('notifications', 'sender') reply_to = config.get('notifications', 'reply-to') -def send_notification(to, subject, body, refs, cc=None, reference=None): +def headers_cc(cclist): + return {'Cc': str.join(', ', cclist)} + +def headers_reply(thread_id): + return {'In-Reply-To': thread_id, 'References': thread_id} + +def send_notification(to, subject, body, refs, headers={}): body = '\n'.join([textwrap.fill(line) for line in body.splitlines()]) body += '\n\n' + refs @@ -37,12 +43,8 @@ def send_notification(to, subject, body, refs, cc=None, reference=None): msg['Reply-to'] = reply_to msg['To'] = recipient - if cc: - msg['Cc'] = str.join(', ', cc) - - if reference: - msg['In-Reply-To'] = reference - msg['References'] = reference + for key, value in header.items(): + msg[key] = value p = subprocess.Popen([sendmail, '-t', '-oi'], stdin=subprocess.PIPE) p.communicate(msg.as_bytes()) @@ -130,8 +132,9 @@ def comment(cur, uid, pkgbase_id): refs = '[1] ' + user_uri + '\n' refs += '[2] ' + pkgbase_uri thread_id = '<pkg-notifications-' + pkgbase + '@aur.archlinux.org>' + headers = headers_reply(thread_id) - send_notification(to, subject, body, refs, reference=thread_id) + send_notification(to, subject, body, refs, headers) def flag(cur, uid, pkgbase_id): user = username_from_id(cur, uid) @@ -207,8 +210,9 @@ def request_open(cur, uid, reqid, reqtype, pkgbase_id, merge_into=None): refs = '[1] ' + user_uri + '\n' refs += '[2] ' + pkgbase_uri + '\n' thread_id = '<pkg-request-' + reqid + '@aur.archlinux.org>' + headers = headers_reply(thread_id) + headers_cc(cc) - send_notification(to, subject, body, refs, cc, thread_id) + send_notification(to, subject, body, refs, headers) def request_close(cur, uid, reqid, reason): user = username_from_id(cur, uid) @@ -227,8 +231,9 @@ def request_close(cur, uid, reqid, reason): body += ':\n\n' + text refs = '[1] ' + user_uri thread_id = '<pkg-request-' + reqid + '@aur.archlinux.org>' + headers = headers_reply(thread_id) + headers_cc(cc) - send_notification(to, subject, body, refs, cc, thread_id) + send_notification(to, subject, body, refs, headers) if __name__ == '__main__': -- 2.6.1

1 2

[aur-dev] [PATCH] Make copyright notice translatable
by Lukas Fleischer 17 Oct '15

17 Oct '15

Fixes FS#46747. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- web/template/footer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/template/footer.php b/web/template/footer.php index 1b3c936..7e505a3 100644 --- a/web/template/footer.php +++ b/web/template/footer.php @@ -5,7 +5,7 @@ <?php if ($ver): ?> aurweb <a href="https://projects.archlinux.org/aurweb.git/log/?h=<?= htmlspecialchars($ver, ENT_QUOTES) ?>"><?= htmlspecialchars($ver) ?></a> <?php endif; ?> - Copyright © 2004-<?= date("Y") ?> aurweb Development Team. + <?= __('Copyright %s 2004-%d aurweb Development Team.', '©', date('Y')) <?= __('AUR packages are user produced content. Any use of the provided files is at your own risk.') ?> </div> </div> -- 2.6.1

1 0

[aur-dev] [PATCH] notify: Fix welcome/flag notifications
by Lukas Fleischer 17 Oct '15

17 Oct '15

Fixes two regressions introduced in commit 6681e56 (notify: Do not wrap references, 2015-10-03). Fixes FS#46742. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- scripts/notify.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/notify.py b/scripts/notify.py index e9026ba..1be0117 100755 --- a/scripts/notify.py +++ b/scripts/notify.py @@ -111,7 +111,7 @@ def welcome(cur, uid): 'password for your new account, please click the link [1] below. ' \ 'If the link does not work, try copying and pasting it into your ' \ 'browser.' - body = '[1] ' + aur_location + '/passreset/?resetkey=' + resetkey + refs = '[1] ' + aur_location + '/passreset/?resetkey=' + resetkey send_notification([to], subject, body, refs) @@ -151,11 +151,11 @@ def flag(cur, uid, pkgbase_id): subject = 'AUR Out-of-date Notification for %s' % (pkgbase) body = 'Your package %s [1] has been flagged out-of-date by %s [2]:' % \ (pkgbase, user) - body += '\n\n' + text + '\n\n' - body += '[1] ' + pkgbase_uri + '\n' - body += '[2] ' + user_uri + body += '\n\n' + text + refs = '[1] ' + pkgbase_uri + '\n' + refs += '[2] ' + user_uri - send_notification(to, subject, body) + send_notification(to, subject, body, refs) def delete(cur, uid, old_pkgbase_id, new_pkgbase_id=None): user = username_from_id(cur, uid) -- 2.6.1

1 0

[aur-dev] Setting keywords from the command line
by Lukas Fleischer 16 Oct '15

16 Oct '15

Hi, Currently, in order to change the keywords of a package base, users need to log into the web interface and submit a form on the package base details page. It would be nice to be able to change keywords without using a web browser, see FS#45627 [1]. Now, there are two options: 1. Add a set-keywords command to the aurweb SSH interface such that users can run `ssh aur.archlinux.org set-keywords foobar keyword1 keyword2 keyword3` to change keywords (or still use the web interface). 2. Remove the option from the web interface and introduce a new file (.KEYWORDS or something more general like .AURINFO, .AURMETADATA) that is parsed in the Git update hook. Obviously, with the second option, one would get a history of keyword changes. On the other hand, some might argue that this will clutter the commit history (although keyword changes should not happen too often) and that this changes makes the AUR package repositories special in a certain way (currently there is nothing AUR-specific in the Git repositories itself, an implementation of the second idea will add such information). Are there any opinions on which option should be implemented? Any other advantages/disadvantages I did not think of? Regards, Lukas [1] https://bugs.archlinux.org/task/45627

3 2

[aur-dev] [PATCH] Minor correction in INSTALL
by Ian D. Scott 10 Oct '15

10 Oct '15

--- INSTALL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL b/INSTALL index d68fa26..b151e27 100644 --- a/INSTALL +++ b/INSTALL @@ -46,7 +46,7 @@ Setup on Arch Linux # mkdir /srv/http/aurweb/aur.git/ # cd /srv/http/aurweb/aur.git/ # git init --bare - # ln -s ../../git-interface/git-update.py hooks/update + # ln -s ../git-interface/git-update.py hooks/update # chown -R aur . 7) Install the git-auth wrapper script: -- 2.6.1

2 4