April 2017 - Aur-dev - lists.archlinux.org

[aur-dev] [PATCH] Add support for Terms of Service documents
by Lukas Fleischer 30 Apr '17

30 Apr '17

This allows for adding Terms of Service documents to the database that registered users need to accept before using the AUR. A revision field can be used to indicate whether a document was updated. If it is increased, all users are again asked to accept the new terms. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- schema/aur-schema.sql | 20 +++++++++++++ upgrading/4.6.0.txt | 20 +++++++++++++ web/html/tos.php | 50 ++++++++++++++++++++++++++++++++ web/lib/acctfuncs.inc.php | 74 +++++++++++++++++++++++++++++++++++++++++++++++ web/lib/aur.inc.php | 23 +++++++++++++++ web/lib/routing.inc.php | 1 + 6 files changed, 188 insertions(+) create mode 100644 web/html/tos.php diff --git a/schema/aur-schema.sql b/schema/aur-schema.sql index e584165..45272bb 100644 --- a/schema/aur-schema.sql +++ b/schema/aur-schema.sql @@ -379,3 +379,23 @@ CREATE TABLE Bans ( BanTS TIMESTAMP NOT NULL, PRIMARY KEY (IPAddress) ) ENGINE = InnoDB; + +-- Terms and Conditions +-- +CREATE TABLE Terms ( + ID INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, + Description VARCHAR(255) NOT NULL, + URL VARCHAR(8000) NOT NULL, + Revision INTEGER UNSIGNED NOT NULL DEFAULT 1, + PRIMARY KEY (ID) +) ENGINE = InnoDB; + +-- Terms and Conditions accepted by users +-- +CREATE TABLE AcceptedTerms ( + UsersID INTEGER UNSIGNED NOT NULL, + TermsID INTEGER UNSIGNED NOT NULL, + Revision INTEGER UNSIGNED NOT NULL DEFAULT 0, + FOREIGN KEY (UsersID) REFERENCES Users(ID) ON DELETE CASCADE, + FOREIGN KEY (TermsID) REFERENCES Terms(ID) ON DELETE CASCADE +) ENGINE = InnoDB; diff --git a/upgrading/4.6.0.txt b/upgrading/4.6.0.txt index b051bac..816409d 100644 --- a/upgrading/4.6.0.txt +++ b/upgrading/4.6.0.txt @@ -15,3 +15,23 @@ UPDATE PackageDepends --- ALTER TABLE PackageComments ADD COLUMN RenderedComment TEXT NOT NULL; --- + +3. Add Terms and AcceptedTerms tables: + +--- +CREATE TABLE Terms ( + ID INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, + Description VARCHAR(255) NOT NULL, + URL VARCHAR(8000) NOT NULL, + Revision INTEGER UNSIGNED NOT NULL DEFAULT 1, + PRIMARY KEY (ID) +) ENGINE = InnoDB; + +CREATE TABLE AcceptedTerms ( + UsersID INTEGER UNSIGNED NOT NULL, + TermsID INTEGER UNSIGNED NOT NULL, + Revision INTEGER UNSIGNED NOT NULL DEFAULT 0, + FOREIGN KEY (UsersID) REFERENCES Users(ID) ON DELETE CASCADE, + FOREIGN KEY (TermsID) REFERENCES Terms(ID) ON DELETE CASCADE +) ENGINE = InnoDB; +--- diff --git a/web/html/tos.php b/web/html/tos.php new file mode 100644 index 0000000..135fa1e --- /dev/null +++ b/web/html/tos.php @@ -0,0 +1,50 @@ +<?php +set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); + +include_once("aur.inc.php"); + +$uid = uid_from_sid($_COOKIE["AURSID"]); + +if (isset($_POST["accept"]) && $_POST["accept"]) { + accept_terms($uid, $_POST["rev"]); + header("Location: " . get_uri('/')); +} + +$terms = fetch_updated_terms($uid); +if (!$terms) { + header("Location: " . get_uri('/')); +} + +html_header('AUR ' . __("Terms of Service")); +?> +<div id="dev-login" class="box"> + <h2>AUR <?= __('Terms of Service') ?></h2> + <?php if (isset($_COOKIE["AURSID"])): ?> + <form method="post" action="<?= get_uri('/tos') ?>"> + <fieldset> + <p> + <?= __("Logged-in as: %s", '<strong>' . username_from_sid($_COOKIE["AURSID"]) . '</strong>'); ?> + </p> + <p> + <?= __("The following documents have been updated. Please review them carefully:"); ?> + </p> + <ul> + <?php foreach($terms as $row): ?> + <li><a href="<?= urlencode(sprintf($row["URL"], $row["Revision"])) ?>"><?= htmlspecialchars($row["Description"]) ?></a> (<?= __('revision %d', $row["Revision"]) ?>)</li> + <?php endforeach; ?> + </ul> + <p> + <?php foreach($terms as $row): ?> + <input type="hidden" name="rev[<?= $row["ID"] ?>]" value="<?= $row["Revision"] ?>" /> + <?php endforeach; ?> + <input type="checkbox" name="accept" /> <?= __("I accept the terms and conditions above."); ?> + </p> + <p> + <input type="submit" name="submit" value="<?= __("Continue") ?>" /> + </p> + </fieldset> + </form> + <?php endif; ?> +</div> +<?php +html_footer(AURWEB_VERSION); diff --git a/web/lib/acctfuncs.inc.php b/web/lib/acctfuncs.inc.php index 22b3ca8..e45d735 100644 --- a/web/lib/acctfuncs.inc.php +++ b/web/lib/acctfuncs.inc.php @@ -1325,3 +1325,77 @@ function notify($params) { return proc_close($p); } + +/* + * Obtain a list of terms a given user has not yet accepted. + * + * @param int $uid The ID of the user to obtain terms for. + * + * @return array A list of terms the user has not yet accepted. + */ +function fetch_updated_terms($uid) { + $dbh = DB::connect(); + + $q = "SELECT ID, Terms.Revision, Description, URL "; + $q .= "FROM Terms LEFT JOIN AcceptedTerms "; + $q .= "ON AcceptedTerms.TermsID = Terms.ID "; + $q .= "AND AcceptedTerms.UsersID = " . intval($uid) . " "; + $q .= "WHERE AcceptedTerms.Revision IS NULL OR "; + $q .= "AcceptedTerms.Revision < Terms.Revision"; + + $result = $dbh->query($q); + + if ($result) { + return $result->fetchAll(); + } else { + return array(); + } +} + +/* + * Accept a list of given terms. + * + * @param int $uid The ID of the user to accept the terms. + * @param array $termrev An array mapping each term to the accepted revision. + * + * @return void + */ +function accept_terms($uid, $termrev) { + $dbh = DB::connect(); + + $q = "SELECT TermsID, Revision FROM AcceptedTerms "; + $q .= "WHERE UsersID = " . intval($uid); + + $result = $dbh->query($q); + + if (!$result) { + return; + } + + $termrev_update = array(); + while ($row = $result->fetch(PDO::FETCH_ASSOC)) { + $id = $row['TermsID']; + if (!array_key_exists($id, $termrev)) { + continue; + } + if ($row['Revision'] < $termrev[$id]) { + $termrev_update[$id] = $termrev[$id]; + } + } + $termrev_add = array_diff_key($termrev, $termrev_update); + + foreach ($termrev_add as $id => $rev) { + $q = "INSERT INTO AcceptedTerms (TermsID, UsersID, Revision) "; + $q .= "VALUES (" . intval($id) . ", " . intval($uid) . ", "; + $q .= intval($rev) . ")"; + $dbh->exec($q); + } + + foreach ($termrev_update as $id => $rev) { + $q = "UPDATE AcceptedTerms "; + $q .= "SET Revision = " . intval($rev) . " "; + $q .= "WHERE TermsID = " . intval($id) . " AND "; + $q .= "UsersID = " . intval($uid); + $dbh->exec($q); + } +} diff --git a/web/lib/aur.inc.php b/web/lib/aur.inc.php index 063de8f..ce569ea 100644 --- a/web/lib/aur.inc.php +++ b/web/lib/aur.inc.php @@ -22,6 +22,7 @@ include_once('timezone.inc.php'); set_tz(); check_sid(); +check_tos(); /** * Check if a visitor is logged in @@ -92,6 +93,28 @@ function check_sid() { } /** + * Redirect user to the Terms of Service agreement if there are updated terms. + * + * @return void + */ +function check_tos() { + if (!isset($_COOKIE["AURSID"])) { + return; + } + + $path = $_SERVER['PATH_INFO']; + $route = get_route($path); + if (!$route || $route == "tos.php") { + return; + } + + if (count(fetch_updated_terms(uid_from_sid($_COOKIE["AURSID"]))) > 0) { + header('Location: ' . get_uri('/tos')); + exit(); + } +} + +/** * Verify the supplied CSRF token matches expected token * * @return bool True if the CSRF token is the same as the cookie SID, otherwise false diff --git a/web/lib/routing.inc.php b/web/lib/routing.inc.php index 8c45c62..7d9750a 100644 --- a/web/lib/routing.inc.php +++ b/web/lib/routing.inc.php @@ -16,6 +16,7 @@ $ROUTES = array( '/passreset' => 'passreset.php', '/rpc' => 'rpc.php', '/rss' => 'rss.php', + '/tos' => 'tos.php', '/tu' => 'tu.php', '/addvote' => 'addvote.php', ); -- 2.12.2

1 1

[aur-dev] [PATCH] Call check_sid() from a central location
by Lukas Fleischer 27 Apr '17

27 Apr '17

Instead of calling check_sid() from every single PHP script representing a web page, add the call to aur.inc.php which is sourced by all of them. Also, remove set_lang() calls from the scripts since these are also already included in aur.inc.php. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- web/html/account.php | 3 --- web/html/addvote.php | 2 -- web/html/comaintainers.php | 3 --- web/html/commentedit.php | 3 --- web/html/home.php | 3 --- web/html/login.php | 2 -- web/html/logout.php | 1 - web/html/packages.php | 2 -- web/html/passreset.php | 3 --- web/html/pkgbase.php | 2 -- web/html/pkgdel.php | 3 --- web/html/pkgdisown.php | 3 --- web/html/pkgflag.php | 3 --- web/html/pkgflagcomment.php | 3 --- web/html/pkgmerge.php | 3 --- web/html/pkgreq.php | 3 --- web/html/register.php | 3 --- web/html/tu.php | 2 -- web/lib/aur.inc.php | 2 ++ 19 files changed, 2 insertions(+), 47 deletions(-) diff --git a/web/html/account.php b/web/html/account.php index 549f852..c30a89a 100644 --- a/web/html/account.php +++ b/web/html/account.php @@ -5,9 +5,6 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once('aur.inc.php'); # access AUR common functions include_once('acctfuncs.inc.php'); # access Account specific functions -set_lang(); # this sets up the visitor's language -check_sid(); # see if they're still logged in - $action = in_request("Action"); $need_userinfo = array( diff --git a/web/html/addvote.php b/web/html/addvote.php index d152941..3672c03 100644 --- a/web/html/addvote.php +++ b/web/html/addvote.php @@ -3,8 +3,6 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); -set_lang(); -check_sid(); $title = __("Add Proposal"); diff --git a/web/html/comaintainers.php b/web/html/comaintainers.php index 23d8d19..9733558 100644 --- a/web/html/comaintainers.php +++ b/web/html/comaintainers.php @@ -5,9 +5,6 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); include_once("pkgbasefuncs.inc.php"); -set_lang(); -check_sid(); - if (!isset($base_id) || !has_credential(CRED_PKGBASE_EDIT_COMAINTAINERS, array(pkgbase_maintainer_uid($base_id)))) { header('Location: /'); exit(); diff --git a/web/html/commentedit.php b/web/html/commentedit.php index 2a0628e..53ec289 100644 --- a/web/html/commentedit.php +++ b/web/html/commentedit.php @@ -5,9 +5,6 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); include_once("pkgbasefuncs.inc.php"); -set_lang(); -check_sid(); - $comment_id = intval($_REQUEST['comment_id']); list($user_id, $comment) = comment_by_id($comment_id); diff --git a/web/html/home.php b/web/html/home.php index 2675491..0ce89f4 100644 --- a/web/html/home.php +++ b/web/html/home.php @@ -3,9 +3,6 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); -set_lang(); -check_sid(); - include_once('stats.inc.php'); if (isset($_COOKIE["AURSID"])) { diff --git a/web/html/login.php b/web/html/login.php index 7345439..a55ce05 100644 --- a/web/html/login.php +++ b/web/html/login.php @@ -2,8 +2,6 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); -set_lang(); -check_sid(); $disable_http_login = config_get_bool('options', 'disable_http_login'); if (!$disable_http_login || (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'])) { diff --git a/web/html/logout.php b/web/html/logout.php index 5e8e8f4..1402200 100644 --- a/web/html/logout.php +++ b/web/html/logout.php @@ -5,7 +5,6 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); # access AUR common functions include_once("acctfuncs.inc.php"); # access AUR common functions - # if they've got a cookie, log them out - need to do this before # sending any HTML output. # diff --git a/web/html/packages.php b/web/html/packages.php index 7d5b207..db9606d 100644 --- a/web/html/packages.php +++ b/web/html/packages.php @@ -3,9 +3,7 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); # access AUR common functions -set_lang(); # this sets up the visitor's language include_once('pkgfuncs.inc.php'); # package specific functions -check_sid(); # see if they're still logged in # Retrieve package ID and name, unless initialized by the routing framework if (!isset($pkgid) || !isset($pkgname)) { diff --git a/web/html/passreset.php b/web/html/passreset.php index e89967d..9e7cee8 100644 --- a/web/html/passreset.php +++ b/web/html/passreset.php @@ -4,9 +4,6 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); # access AUR common functions -set_lang(); # this sets up the visitor's language -check_sid(); # see if they're still logged in - if (isset($_COOKIE["AURSID"])) { header('Location: /'); exit(); diff --git a/web/html/pkgbase.php b/web/html/pkgbase.php index a593af1..03b0eee 100644 --- a/web/html/pkgbase.php +++ b/web/html/pkgbase.php @@ -3,9 +3,7 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); -set_lang(); include_once('pkgfuncs.inc.php'); -check_sid(); /* * Retrieve package base ID and name, unless initialized by the routing diff --git a/web/html/pkgdel.php b/web/html/pkgdel.php index 591ccce..4620beb 100644 --- a/web/html/pkgdel.php +++ b/web/html/pkgdel.php @@ -5,9 +5,6 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); include_once("pkgfuncs.inc.php"); -set_lang(); -check_sid(); - html_header(__("Package Deletion")); if (has_credential(CRED_PKGBASE_DELETE)): ?> diff --git a/web/html/pkgdisown.php b/web/html/pkgdisown.php index 036fe6d..4b04e85 100644 --- a/web/html/pkgdisown.php +++ b/web/html/pkgdisown.php @@ -5,9 +5,6 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); include_once("pkgfuncs.inc.php"); -set_lang(); -check_sid(); - html_header(__("Disown Package")); $maintainer_uids = array(pkgbase_maintainer_uid($base_id)); diff --git a/web/html/pkgflag.php b/web/html/pkgflag.php index 25c8ccb..61346b9 100644 --- a/web/html/pkgflag.php +++ b/web/html/pkgflag.php @@ -5,9 +5,6 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); include_once("pkgfuncs.inc.php"); -set_lang(); -check_sid(); - /* Grab the list of package base IDs to be operated on. */ $ids = array(); if (isset($_POST['IDs'])) { diff --git a/web/html/pkgflagcomment.php b/web/html/pkgflagcomment.php index 98680ef..c0363c8 100644 --- a/web/html/pkgflagcomment.php +++ b/web/html/pkgflagcomment.php @@ -5,9 +5,6 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); include_once("pkgbasefuncs.inc.php"); -set_lang(); -check_sid(); - if (!isset($base_id)) { header('Location: /'); exit(); diff --git a/web/html/pkgmerge.php b/web/html/pkgmerge.php index e8e7ca9..6ee7423 100644 --- a/web/html/pkgmerge.php +++ b/web/html/pkgmerge.php @@ -5,9 +5,6 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); include_once("pkgfuncs.inc.php"); -set_lang(); -check_sid(); - html_header(__("Package Merging")); if (has_credential(CRED_PKGBASE_DELETE)): ?> diff --git a/web/html/pkgreq.php b/web/html/pkgreq.php index f981c25..d2f9461 100644 --- a/web/html/pkgreq.php +++ b/web/html/pkgreq.php @@ -5,9 +5,6 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); include_once("pkgfuncs.inc.php"); -set_lang(); -check_sid(); - if (isset($base_id)) { if (!has_credential(CRED_PKGREQ_FILE)) { header('Location: /'); diff --git a/web/html/register.php b/web/html/register.php index 843fea9..368999a 100644 --- a/web/html/register.php +++ b/web/html/register.php @@ -5,9 +5,6 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once('aur.inc.php'); # access AUR common functions include_once('acctfuncs.inc.php'); # access Account specific functions -set_lang(); # this sets up the visitor's language -check_sid(); # see if they're still logged in - if (isset($_COOKIE["AURSID"])) { header('Location: /'); exit(); diff --git a/web/html/tu.php b/web/html/tu.php index cc77d62..34e64ea 100644 --- a/web/html/tu.php +++ b/web/html/tu.php @@ -3,8 +3,6 @@ set_include_path(get_include_path() . PATH_SEPARATOR . '../lib'); include_once("aur.inc.php"); -set_lang(); -check_sid(); $title = __("Trusted User"); diff --git a/web/lib/aur.inc.php b/web/lib/aur.inc.php index d58df40..063de8f 100644 --- a/web/lib/aur.inc.php +++ b/web/lib/aur.inc.php @@ -21,6 +21,8 @@ include_once("credentials.inc.php"); include_once('timezone.inc.php'); set_tz(); +check_sid(); + /** * Check if a visitor is logged in * -- 2.12.2

1 0

[aur-dev] [PATCH] Support headings in package comments
by Lukas Fleischer 26 Apr '17

26 Apr '17

Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- aurweb/scripts/rendercomment.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/aurweb/scripts/rendercomment.py b/aurweb/scripts/rendercomment.py index f32c827..9c0aa6a 100755 --- a/aurweb/scripts/rendercomment.py +++ b/aurweb/scripts/rendercomment.py @@ -67,6 +67,20 @@ class GitCommitsExtension(markdown.extensions.Extension): md.preprocessors.add('git-commits', preprocessor, '_end') +class HeadingTreeprocessor(markdown.treeprocessors.Treeprocessor): + def run(self, doc): + for elem in doc: + if elem.tag == 'h1': + elem.tag = 'h5' + elif elem.tag in ['h2', 'h3', 'h4', 'h5']: + elem.tag = 'h6' + + +class HeadingExtension(markdown.extensions.Extension): + def extendMarkdown(self, md, md_globals): + md.treeprocessors.add('heading', HeadingTreeprocessor(md), '_end') + + def get_comment(conn, commentid): cur = conn.execute('SELECT PackageComments.Comments, PackageBases.Name ' 'FROM PackageComments INNER JOIN PackageBases ' @@ -88,8 +102,10 @@ def main(): text, pkgbase = get_comment(conn, commentid) html = markdown.markdown(text, extensions=['fenced_code', LinkifyExtension(), - GitCommitsExtension(pkgbase)]) - allowed_tags = bleach.sanitizer.ALLOWED_TAGS + ['p', 'pre'] + GitCommitsExtension(pkgbase), + HeadingExtension()]) + allowed_tags = bleach.sanitizer.ALLOWED_TAGS + \ + ['p', 'pre', 'h4', 'h5', 'h6'] html = bleach.clean(html, tags=allowed_tags) save_rendered_comment(conn, commentid, html) -- 2.12.2

1 0

[aur-dev] [PATCH 1/3] t1300: Factor out package dumping
by Lukas Fleischer 25 Apr '17

25 Apr '17

Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- test/t1300-git-update.sh | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/test/t1300-git-update.sh b/test/t1300-git-update.sh index a65ca3a..dbe6029 100755 --- a/test/t1300-git-update.sh +++ b/test/t1300-git-update.sh @@ -4,6 +4,14 @@ test_description='git-update tests' . ./setup.sh +dump_package_info() { + for t in Packages Licenses PackageLicenses Groups PackageGroups \ + PackageDepends PackageRelations PackageSources \ + PackageNotifications; do + echo "SELECT * FROM $t;" | sqlite3 aur.db + done +} + test_expect_success 'Test update hook on a fresh repository.' ' old=0000000000000000000000000000000000000000 && new=$(git -C aur.git rev-parse HEAD^) && @@ -16,12 +24,7 @@ test_expect_success 'Test update hook on a fresh repository.' ' 1|1|python-pygit2|| 1|1 EOF - >actual && - for t in Packages Licenses PackageLicenses Groups PackageGroups \ - PackageDepends PackageRelations PackageSources \ - PackageNotifications; do - echo "SELECT * FROM $t;" | sqlite3 aur.db >>actual - done && + dump_package_info >actual && test_cmp expected actual ' @@ -44,12 +47,7 @@ test_expect_success 'Test update hook on another fresh repository.' ' 1|1 2|1 EOF - >actual && - for t in Packages Licenses PackageLicenses Groups PackageGroups \ - PackageDepends PackageRelations PackageSources \ - PackageNotifications; do - echo "SELECT * FROM $t;" | sqlite3 aur.db >>actual - done && + dump_package_info >actual && test_cmp expected actual ' @@ -70,12 +68,7 @@ test_expect_success 'Test update hook on an updated repository.' ' 1|1 2|1 EOF - >actual && - for t in Packages Licenses PackageLicenses Groups PackageGroups \ - PackageDepends PackageRelations PackageSources \ - PackageNotifications; do - echo "SELECT * FROM $t;" | sqlite3 aur.db >>actual - done && + dump_package_info >actual && test_cmp expected actual ' @@ -94,12 +87,7 @@ test_expect_success 'Test restore mode.' ' 1|1 2|1 EOF - >actual && - for t in Packages Licenses PackageLicenses Groups PackageGroups \ - PackageDepends PackageRelations PackageSources \ - PackageNotifications; do - echo "SELECT * FROM $t;" | sqlite3 aur.db >>actual - done && + dump_package_info >actual && test_cmp expected actual ' -- 2.12.2

1 2

[aur-dev] [PATCH] Fix the comment collapse feature
by Lukas Fleischer 24 Apr '17

24 Apr '17

In commit 4abde89 (Use JavaScript to collapse long comments, 2017-04-19), support for collapsing/expanding long comments was added. This was broken by the recent Markdown support since comments no longer live inside a single HTML paragraph. Fix this by wrapping each comment in another div container. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- web/html/packages.php | 23 ++++++++++++----------- web/template/pkg_comments.php | 16 +++++++++------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/web/html/packages.php b/web/html/packages.php index 8d76c76..7d5b207 100644 --- a/web/html/packages.php +++ b/web/html/packages.php @@ -74,32 +74,33 @@ function collapseDependsList(list) { function collapseComment(div) { var linkid = div.attr('id') + 'link', - par = div.find('p'), - height = par.height(), + inner = div.find('div'), + height = inner.height(), maxheight = 200; if (height <= maxheight) return; - par.css({ 'overflow': 'hidden', 'height': maxheight + 'px' }); - par.addClass('collapsed'); - par.after('<p><a id="' + linkid + '" href="#">Show More…</a></p>'); + inner.css({ 'overflow': 'hidden', 'height': maxheight + 'px' }); + inner.addClass('collapsed'); + inner.after('<p><a id="' + linkid + '" href="#">Show More…</a></p>'); $('#' + linkid).click(function(event) { + var inner = $(this).parent().parent().find('div'); var newheight; - if (par.hasClass('collapsed')) { - par.css({ 'height': 'auto' }); - newheight = par.height(); - par.css({ 'height': maxheight }); + if (inner.hasClass('collapsed')) { + inner.css({ 'height': 'auto' }); + newheight = inner.height(); + inner.css({ 'height': maxheight }); $(this).text('Collapse'); } else { newheight = maxheight; $(this).text('Show More…'); } - par.animate({ 'height': newheight }); - par.toggleClass('collapsed'); + inner.animate({ 'height': newheight }); + inner.toggleClass('collapsed'); event.preventDefault(); }); } diff --git a/web/template/pkg_comments.php b/web/template/pkg_comments.php index f973b74..7d9bedc 100644 --- a/web/template/pkg_comments.php +++ b/web/template/pkg_comments.php @@ -103,13 +103,15 @@ if (!isset($count)) { <?php endif; ?> </h4> <div id="<?= isset($pinned) ? "pinned-" : "comment-" ?><?= $row['ID'] ?>-content" class="article-content<?php if ($is_deleted): ?> comment-deleted<?php endif; ?>"> - <?php if (!empty($row['RenderedComment'])): ?> - <?= $row['RenderedComment'] ?> - <?php else: ?> - <p> - <?= parse_comment($row['Comments']) ?> - </p> - <?php endif; ?> + <div> + <?php if (!empty($row['RenderedComment'])): ?> + <?= $row['RenderedComment'] ?> + <?php else: ?> + <p> + <?= parse_comment($row['Comments']) ?> + </p> + <?php endif; ?> + </div> </div> <?php endwhile; ?> -- 2.12.2

1 0

[aur-dev] [PATCH 1/2] Do not keep line breaks in comments
by Lukas Fleischer 24 Apr '17

24 Apr '17

With the new Markdown support, text paragraphs are now properly converted to HTML paragraphs, so we no longer need to keep line breaks. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- aurweb/scripts/rendercomment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aurweb/scripts/rendercomment.py b/aurweb/scripts/rendercomment.py index 659e18a..c23ef8d 100755 --- a/aurweb/scripts/rendercomment.py +++ b/aurweb/scripts/rendercomment.py @@ -86,9 +86,9 @@ def main(): conn = aurweb.db.Connection() text, pkgbase = get_comment(conn, commentid) - html = markdown.markdown(text, extensions=['nl2br', LinkifyExtension(), + html = markdown.markdown(text, extensions=[LinkifyExtension(), GitCommitsExtension(pkgbase)]) - allowed_tags = bleach.sanitizer.ALLOWED_TAGS + ['p', 'br'] + allowed_tags = bleach.sanitizer.ALLOWED_TAGS + ['p'] html = bleach.clean(html, tags=allowed_tags) save_rendered_comment(conn, commentid, html) -- 2.12.2

1 1

[aur-dev] [PATCH 0/4] Add Markdown syntax to comments
by Lukas Fleischer 24 Apr '17

24 Apr '17

This patch series moves comment rendering to a separate Python script, adds markdown support and also makes Git commit identifiers clickable. Comments welcome! Lukas Fleischer (4): Render comments when storing them in the database Add Markdown support to package comments Reintroduce backwards-compatible hyperlink syntax Make references to Git commits clickable aurweb/scripts/rendercomment.py | 100 ++++++++++++++++++++++++++++++++++++++++ conf/config.proto | 2 + schema/aur-schema.sql | 1 + setup.py | 1 + test/setup.sh | 1 + test/t2600-rendercomment.sh | 23 +++++++++ upgrading/4.6.0.txt | 6 +++ web/lib/pkgbasefuncs.inc.php | 40 ++++++++++++++-- web/template/pkg_comments.php | 4 ++ 9 files changed, 174 insertions(+), 4 deletions(-) create mode 100755 aurweb/scripts/rendercomment.py create mode 100755 test/t2600-rendercomment.sh -- 2.12.2

1 5

[aur-dev] [PATCH] Rerender package comments after editing
by Lukas Fleischer 23 Apr '17

23 Apr '17

Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- web/lib/pkgbasefuncs.inc.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/web/lib/pkgbasefuncs.inc.php b/web/lib/pkgbasefuncs.inc.php index 3e78309..ff1bc90 100644 --- a/web/lib/pkgbasefuncs.inc.php +++ b/web/lib/pkgbasefuncs.inc.php @@ -1041,6 +1041,9 @@ function pkgbase_edit_comment($comment) { $q.= "EditedTS = " . strval(time()) . " "; $q.= "WHERE ID = ".intval($comment_id); $dbh->exec($q); + + render_comment($comment_id); + return array(true, __("Comment has been edited.")); } else { return array(false, __("You are not allowed to edit this comment.")); -- 2.12.2

1 0

[aur-dev] [PATCH] t2600: Test Markdown and HTML sanitizing
by Lukas Fleischer 23 Apr '17

23 Apr '17

Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- test/t2600-rendercomment.sh | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/test/t2600-rendercomment.sh b/test/t2600-rendercomment.sh index 6db9467..8da1529 100755 --- a/test/t2600-rendercomment.sh +++ b/test/t2600-rendercomment.sh @@ -21,4 +21,46 @@ test_expect_success 'Test comment rendering.' ' test_cmp actual expected ' +test_expect_success 'Test Markdown conversion.' ' + cat <<-EOD | sqlite3 aur.db && + INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (2, 1, "*Hello* [world](https://www.archlinux.org/)!", ""); + EOD + "$RENDERCOMMENT" 2 && + cat <<-EOD >expected && + <p><em>Hello</em> <a href="https://www.archlinux.org/">world</a>!</p> + EOD + cat <<-EOD | sqlite3 aur.db >actual && + SELECT RenderedComment FROM PackageComments WHERE ID = 2; + EOD + test_cmp actual expected +' + +test_expect_success 'Test HTML sanitizing.' ' + cat <<-EOD | sqlite3 aur.db && + INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (3, 1, "<script>alert(""XSS!"");</script>", ""); + EOD + "$RENDERCOMMENT" 3 && + cat <<-EOD >expected && + <script>alert("XSS!");</script> + EOD + cat <<-EOD | sqlite3 aur.db >actual && + SELECT RenderedComment FROM PackageComments WHERE ID = 3; + EOD + test_cmp actual expected +' + +test_expect_success 'Test link conversion.' ' + cat <<-EOD | sqlite3 aur.db && + INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (4, 1, "Visit https://www.archlinux.org/.", ""); + EOD + "$RENDERCOMMENT" 4 && + cat <<-EOD >expected && + <p>Visit <a href="https://www.archlinux.org/">https://www.archlinux.org/</a>.</p> + EOD + cat <<-EOD | sqlite3 aur.db >actual && + SELECT RenderedComment FROM PackageComments WHERE ID = 4; + EOD + test_cmp actual expected +' + test_done -- 2.12.2

1 0

[aur-dev] [PATCH] Replace task references in comments with actual links
by Ragnis Armus 23 Apr '17

23 Apr '17

Search comments for references to Flyspray tasks (in the format FS#???) and make them into clickable links. Signed-off-by: Ragnis Armus <ragnis(a)aragnis.com> --- conf/config.proto | 1 + web/lib/aur.inc.php | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/conf/config.proto b/conf/config.proto index df10b99..39257f6 100644 --- a/conf/config.proto +++ b/conf/config.proto @@ -33,6 +33,7 @@ log_uri = https://aur.archlinux.org/cgit/aur.git/log/?h=%s snapshot_uri = /cgit/aur.git/snapshot/%s.tar.gz enable-maintenance = 1 maintenance-exceptions = 127.0.0.1 +fs_task_uri = https://bugs.archlinux.org/task/%d [notifications] notify-cmd = /usr/local/bin/aurweb-notify diff --git a/web/lib/aur.inc.php b/web/lib/aur.inc.php index d58df40..418b7cc 100644 --- a/web/lib/aur.inc.php +++ b/web/lib/aur.inc.php @@ -557,6 +557,35 @@ function comment_by_id($comment_id) { } /** + * Process comment body replacing any references to tasks with an actual link + * + * @param string $text The input text + * + * @return string The text with any task references replaces with HTML links + */ +function parse_comment_flyspray_links($text) { + $matches = preg_split('/(?<=\W|^)FS#(\d+)(?=\W|$)/', $text, -1, + PREG_SPLIT_DELIM_CAPTURE); + + $output = ''; + + for ($i = 0; $i < count($matches); $i++) { + if ($i % 2) { + # link the tasks + $url = sprintf(config_get('options', 'fs_task_uri'), $matches[$i]); + + $output .= '<a href="' . htmlspecialchars($url) . + '" rel="nofollow">FS#' . htmlspecialchars($matches[$i]) . '</a>'; + } else { + # keep everything else + $output .= $matches[$i]; + } + } + + return $output; +} + +/** * Process submitted comments so any links can be followed * * @param string $comment Raw user submitted package comment @@ -579,7 +608,10 @@ function parse_comment($comment) { } else { # convert everything else - $html .= nl2br(htmlspecialchars($matches[$i])); + $value = nl2br(htmlspecialchars($matches[$i])); + $value = parse_comment_flyspray_links($value); + + $html .= $value; } } -- 2.12.2

3 2

[aur-dev] [PATCH] Use JavaScript to collapse long comments
by Lukas Fleischer 20 Apr '17

20 Apr '17

Instead of using CSS to limit the height of package comments as implemented in 7b13203 (Limit comment height to 15 lines, 2016-03-12), use JavaScript to collapse long comments and add a link to expand them. Clicking the same link twice results in the corresponding comment being collapsed again. If JavaScript is disabled, the full comments are shown (without any possibility to collapse or expand). Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- web/html/css/aurweb.css | 5 ----- web/html/packages.php | 35 +++++++++++++++++++++++++++++++++++ web/template/pkg_comments.php | 2 +- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/web/html/css/aurweb.css b/web/html/css/aurweb.css index f777ab8..f5e1037 100644 --- a/web/html/css/aurweb.css +++ b/web/html/css/aurweb.css @@ -148,8 +148,3 @@ label.confirmation, color: red; font-weight: bold; } - -#news div p { - max-height: 15em; - overflow: auto; -} diff --git a/web/html/packages.php b/web/html/packages.php index 113a114..8d76c76 100644 --- a/web/html/packages.php +++ b/web/html/packages.php @@ -72,10 +72,45 @@ function collapseDependsList(list) { }); } +function collapseComment(div) { + var linkid = div.attr('id') + 'link', + par = div.find('p'), + height = par.height(), + maxheight = 200; + + if (height <= maxheight) + return; + + par.css({ 'overflow': 'hidden', 'height': maxheight + 'px' }); + par.addClass('collapsed'); + par.after('<p><a id="' + linkid + '" href="#">Show More…</a></p>'); + + $('#' + linkid).click(function(event) { + var newheight; + + if (par.hasClass('collapsed')) { + par.css({ 'height': 'auto' }); + newheight = par.height(); + par.css({ 'height': maxheight }); + $(this).text('Collapse'); + } else { + newheight = maxheight; + $(this).text('Show More…'); + } + + par.animate({ 'height': newheight }); + par.toggleClass('collapsed'); + event.preventDefault(); + }); +} + $(document).ready(function() { collapseDependsList("#pkgdepslist"); collapseDependsList("#pkgreqslist"); collapseDependsList("#pkgsrcslist"); + $(".article-content").each(function() { + collapseComment($(this)); + }); }); </script> diff --git a/web/template/pkg_comments.php b/web/template/pkg_comments.php index fee1898..c23ec42 100644 --- a/web/template/pkg_comments.php +++ b/web/template/pkg_comments.php @@ -102,7 +102,7 @@ if (!isset($count)) { </form> <?php endif; ?> </h4> - <div class="article-content<?php if ($is_deleted): ?> comment-deleted<?php endif; ?>"> + <div id="<?= isset($pinned) ? "pinned-" : "comment-" ?><?= $row['ID'] ?>-content" class="article-content<?php if ($is_deleted): ?> comment-deleted<?php endif; ?>"> <p> <?= parse_comment($row['Comments']) ?> </p> -- 2.12.2

1 0

[aur-dev] [PATCH] Store dependency descriptions in a separate column
by Lukas Fleischer 19 Apr '17

19 Apr '17

Split optional dependency descriptions from dependency names before storing them in the database and use a separate column to store the descriptions. This allows us to simplify and optimize the SQL queries in pkg_dependencies() as well as pkg_required(). Suggested-by: Florian Pritz <bluewind(a)xinu.at> Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- aurweb/git/update.py | 16 +++++++--------- schema/aur-schema.sql | 1 + upgrading/4.6.0.txt | 11 +++++++++++ web/lib/pkgfuncs.inc.php | 33 ++++++++------------------------- web/template/pkg_details.php | 2 +- 5 files changed, 28 insertions(+), 35 deletions(-) create mode 100644 upgrading/4.6.0.txt diff --git a/aurweb/git/update.py b/aurweb/git/update.py index 532db92..09a57ef 100755 --- a/aurweb/git/update.py +++ b/aurweb/git/update.py @@ -52,10 +52,7 @@ def parse_dep(depstring): depname = re.sub(r'(<|=|>).*', '', dep) depcond = dep[len(depname):] - if (desc): - return (depname + ': ' + desc, depcond) - else: - return (depname, depcond) + return (depname, desc, depcond) def create_pkgbase(conn, pkgbase, user): @@ -141,12 +138,13 @@ def save_metadata(metadata, conn, user): [deptype]) deptypeid = cur.fetchone()[0] for dep_info in extract_arch_fields(pkginfo, deptype): - depname, depcond = parse_dep(dep_info['value']) + depname, depdesc, depcond = parse_dep(dep_info['value']) deparch = dep_info['arch'] conn.execute("INSERT INTO PackageDepends (PackageID, " + - "DepTypeID, DepName, DepCondition, DepArch) " + - "VALUES (?, ?, ?, ?, ?)", - [pkgid, deptypeid, depname, depcond, deparch]) + "DepTypeID, DepName, DepDesc, DepCondition, " + + "DepArch) VALUES (?, ?, ?, ?, ?, ?)", + [pkgid, deptypeid, depname, depdesc, depcond, + deparch]) # Add package relations (conflicts, provides, replaces). for reltype in ('conflicts', 'provides', 'replaces'): @@ -154,7 +152,7 @@ def save_metadata(metadata, conn, user): [reltype]) reltypeid = cur.fetchone()[0] for rel_info in extract_arch_fields(pkginfo, reltype): - relname, relcond = parse_dep(rel_info['value']) + relname, _, relcond = parse_dep(rel_info['value']) relarch = rel_info['arch'] conn.execute("INSERT INTO PackageRelations (PackageID, " + "RelTypeID, RelName, RelCondition, RelArch) " + diff --git a/schema/aur-schema.sql b/schema/aur-schema.sql index 89167b3..be6f3e5 100644 --- a/schema/aur-schema.sql +++ b/schema/aur-schema.sql @@ -187,6 +187,7 @@ CREATE TABLE PackageDepends ( PackageID INTEGER UNSIGNED NOT NULL, DepTypeID TINYINT UNSIGNED NOT NULL, DepName VARCHAR(255) NOT NULL, + DepDesc VARCHAR(255) NULL DEFAULT NULL, DepCondition VARCHAR(255), DepArch VARCHAR(255) NULL DEFAULT NULL, FOREIGN KEY (PackageID) REFERENCES Packages(ID) ON DELETE CASCADE, diff --git a/upgrading/4.6.0.txt b/upgrading/4.6.0.txt new file mode 100644 index 0000000..45740f4 --- /dev/null +++ b/upgrading/4.6.0.txt @@ -0,0 +1,11 @@ +1. Add DepDesc column to PackageDepends and split dependency names: + +--- +ALTER TABLE PackageDepends ADD COLUMN DepDesc VARCHAR(255) NULL DEFAULT NULL; +UPDATE PackageDepends + SET DepDesc = SUBSTRING(DepName FROM POSITION(': ' IN DepName) + 2) + WHERE POSITION(': ' IN DepName) > 0; +UPDATE PackageDepends + SET DepName = SUBSTRING(DepName FROM 1 FOR POSITION(': ' IN DepName) - 1) + WHERE POSITION(': ' IN DepName) > 0; +--- diff --git a/web/lib/pkgfuncs.inc.php b/web/lib/pkgfuncs.inc.php index adb21f6..5c48a8e 100644 --- a/web/lib/pkgfuncs.inc.php +++ b/web/lib/pkgfuncs.inc.php @@ -239,9 +239,10 @@ function pkg_dependencies($pkgid, $limit) { $pkgid = intval($pkgid); if ($pkgid > 0) { $dbh = DB::connect(); - $q = "SELECT pd.DepName, dt.Name, pd.DepCondition, pd.DepArch, p.ID FROM PackageDepends pd "; + $q = "SELECT pd.DepName, dt.Name, pd.DepDesc, "; + $q.= "pd.DepCondition, pd.DepArch, p.ID "; + $q.= "FROM PackageDepends pd "; $q.= "LEFT JOIN Packages p ON pd.DepName = p.Name "; - $q.= "OR SUBSTRING(pd.DepName FROM 1 FOR POSITION(': ' IN pd.DepName) - 1) = p.Name "; $q.= "LEFT JOIN DependencyTypes dt ON dt.ID = pd.DepTypeID "; $q.= "WHERE pd.PackageID = ". $pkgid . " "; $q.= "ORDER BY pd.DepName LIMIT " . intval($limit); @@ -354,21 +355,14 @@ function pkg_provider_link($name, $official) { * * @param string $name The name of the dependency * @param string $type The name of the dependency type + * @param string $desc The (optional) description of the dependency * @param string $cond The package dependency condition string * @param string $arch The package dependency architecture * @param int $pkg_id The package of the package to display the dependency for * * @return string The HTML code of the label to display */ -function pkg_depend_link($name, $type, $cond, $arch, $pkg_id) { - if ($type == 'optdepends' && strpos($name, ':') !== false) { - $tokens = explode(':', $name, 2); - $name = $tokens[0]; - $desc = $tokens[1]; - } else { - $desc = ''; - } - +function pkg_depend_link($name, $type, $desc, $cond, $arch, $pkg_id) { /* * TODO: We currently perform one SQL query per nonexistent package * dependency. It would be much better if we could annotate dependency @@ -432,25 +426,14 @@ function pkg_depend_link($name, $type, $cond, $arch, $pkg_id) { * @return string The HTML code of the link to display */ function pkg_requiredby_link($name, $depends, $type, $arch, $pkgname) { - if ($type == 'optdepends' && strpos($name, ':') !== false) { - $tokens = explode(':', $name, 2); - $name = $tokens[0]; - } - $link = '<a href="'; $link .= htmlspecialchars(get_pkg_uri($name), ENT_QUOTES); $link .= '" title="' . __('View packages details for') .' ' . htmlspecialchars($name) . '">'; $link .= htmlspecialchars($name) . '</a>'; if ($depends != $pkgname) { - $depname = $depends; - if (strpos($depends, ':') !== false) { - $tokens = explode(':', $depname, 2); - $depname = $tokens[0]; - } - $link .= ' <span class="virtual-dep">('; - $link .= __('requires %s', htmlspecialchars($depname)); + $link .= __('requires %s', htmlspecialchars($depends)); $link .= ')</span>'; } @@ -522,11 +505,11 @@ function pkg_required($name="", $provides, $limit) { $name_list .= ',' . $dbh->quote($p[0]); } - $q = "SELECT p.Name, pd.DepName, dt.Name, pd.DepArch FROM PackageDepends pd "; + $q = "SELECT p.Name, pd.DepName, dt.Name, pd.DepArch "; + $q.= "FROM PackageDepends pd "; $q.= "LEFT JOIN Packages p ON p.ID = pd.PackageID "; $q.= "LEFT JOIN DependencyTypes dt ON dt.ID = pd.DepTypeID "; $q.= "WHERE pd.DepName IN (" . $name_list . ") "; - $q.= "OR SUBSTRING(pd.DepName FROM 1 FOR POSITION(': ' IN pd.DepName) - 1) IN (" . $name_list . ") "; $q.= "ORDER BY p.Name LIMIT " . intval($limit); $result = $dbh->query($q); if (!$result) {return array();} diff --git a/web/template/pkg_details.php b/web/template/pkg_details.php index ed8974a..8a93417 100644 --- a/web/template/pkg_details.php +++ b/web/template/pkg_details.php @@ -277,7 +277,7 @@ endif; <?php if (count($deps) > 0): ?> <ul id="pkgdepslist"> <?php while (list($k, $darr) = each($deps)): ?> - <li><?= pkg_depend_link($darr[0], $darr[1], $darr[2], $darr[3], $darr[4]); ?></li> + <li><?= pkg_depend_link($darr[0], $darr[1], $darr[2], $darr[3], $darr[4], $darr[5]); ?></li> <?php endwhile; ?> </ul> <?php endif; ?> -- 2.12.2

2 1

[aur-dev] [PATCH 1/4] Check query return value in pkgbase_user_notify()
by Lukas Fleischer 18 Apr '17

18 Apr '17

Instead of unconditionally calling fetch on the return value of query(), error out early if the value evaluates to false. Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- web/lib/pkgbasefuncs.inc.php | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/web/lib/pkgbasefuncs.inc.php b/web/lib/pkgbasefuncs.inc.php index cd4b271..57933e8 100644 --- a/web/lib/pkgbasefuncs.inc.php +++ b/web/lib/pkgbasefuncs.inc.php @@ -852,12 +852,11 @@ function pkgbase_user_notify($uid, $base_id) { $q.= " AND PackageBaseID = " . $dbh->quote($base_id); $result = $dbh->query($q); - if ($result->fetch(PDO::FETCH_NUM)) { - return true; - } - else { + if (!$result) { return false; } + + return ($result->fetch(PDO::FETCH_NUM) > 0); } /** -- 2.12.2

1 3

[aur-dev] [PATCH 1/2] schema: Remove invalid default values for TEXT columns
by Florian Pritz 16 Apr '17

16 Apr '17

When running in strict mode, mysql throws an error upon encountering these definitions since they are not supported. References: https://dev.mysql.com/doc/refman/5.7/en/data-type-defaults.html Signed-off-by: Florian Pritz <bluewind(a)xinu.at> --- schema/aur-schema.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/schema/aur-schema.sql b/schema/aur-schema.sql index b42b45d..b3e4ecb 100644 --- a/schema/aur-schema.sql +++ b/schema/aur-schema.sql @@ -80,7 +80,7 @@ CREATE TABLE PackageBases ( NumVotes INTEGER UNSIGNED NOT NULL DEFAULT 0, Popularity DECIMAL(10,6) UNSIGNED NOT NULL DEFAULT 0, OutOfDateTS BIGINT UNSIGNED NULL DEFAULT NULL, - FlaggerComment TEXT NOT NULL DEFAULT '', + FlaggerComment TEXT NOT NULL, SubmittedTS BIGINT UNSIGNED NOT NULL, ModifiedTS BIGINT UNSIGNED NOT NULL, FlaggerUID INTEGER UNSIGNED NULL DEFAULT NULL, -- who flagged the package out-of-date? @@ -253,7 +253,7 @@ CREATE TABLE PackageComments ( ID BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, PackageBaseID INTEGER UNSIGNED NOT NULL, UsersID INTEGER UNSIGNED NULL DEFAULT NULL, - Comments TEXT NOT NULL DEFAULT '', + Comments TEXT NOT NULL, CommentTS BIGINT UNSIGNED NOT NULL DEFAULT 0, EditedTS BIGINT UNSIGNED NULL DEFAULT NULL, EditedUsersID INTEGER UNSIGNED NULL DEFAULT NULL, @@ -331,8 +331,8 @@ CREATE TABLE PackageRequests ( PackageBaseName VARCHAR(255) NOT NULL, MergeBaseName VARCHAR(255) NULL, UsersID INTEGER UNSIGNED NULL DEFAULT NULL, - Comments TEXT NOT NULL DEFAULT '', - ClosureComment TEXT NOT NULL DEFAULT '', + Comments TEXT NOT NULL, + ClosureComment TEXT NOT NULL, RequestTS BIGINT UNSIGNED NOT NULL DEFAULT 0, Status TINYINT UNSIGNED NOT NULL DEFAULT 0, PRIMARY KEY (ID), -- 2.12.2

2 2

[aur-dev] [PATCH 1/4] Add sqlite3 schema for testing databases
by Mark Weiman 12 Apr '17

12 Apr '17

The existing aur-schema.sql file is specific to MySQL, this patch adds a second schema file for use to create sqlite3 testing databases. Signed-off-by: Mark Weiman <mark.weiman(a)markzz.com> --- schema/aur-schema-sqlite.sql | 368 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 368 insertions(+) create mode 100644 schema/aur-schema-sqlite.sql diff --git a/schema/aur-schema-sqlite.sql b/schema/aur-schema-sqlite.sql new file mode 100644 index 0000000..5d655e9 --- /dev/null +++ b/schema/aur-schema-sqlite.sql @@ -0,0 +1,368 @@ +-- The SQLite database layout for the AUR. Certain data +-- is also included such as AccountTypes, etc. +-- + +-- Define the Account Types for the AUR. +-- +CREATE TABLE AccountTypes ( + ID TINYINT UNSIGNED NOT NULL, + AccountType VARCHAR(32) NOT NULL DEFAULT '', + PRIMARY KEY (ID) +); +INSERT INTO AccountTypes (ID, AccountType) VALUES (1, 'User'); +INSERT INTO AccountTypes (ID, AccountType) VALUES (2, 'Trusted User'); +INSERT INTO AccountTypes (ID, AccountType) VALUES (3, 'Developer'); +INSERT INTO AccountTypes (ID, AccountType) VALUES (4, 'Trusted User & Developer'); + + +-- User information for each user regardless of type. +-- +CREATE TABLE Users ( + ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + AccountTypeID TINYINT UNSIGNED NOT NULL DEFAULT 1, + Suspended TINYINT UNSIGNED NOT NULL DEFAULT 0, + Username VARCHAR(32) NOT NULL, + Email VARCHAR(254) NOT NULL, + HideEmail TINYINT UNSIGNED NOT NULL DEFAULT 0, + Passwd CHAR(32) NOT NULL, + Salt CHAR(32) NOT NULL DEFAULT '', + ResetKey CHAR(32) NOT NULL DEFAULT '', + RealName VARCHAR(64) NOT NULL DEFAULT '', + LangPreference VARCHAR(6) NOT NULL DEFAULT 'en', + Timezone VARCHAR(32) NOT NULL DEFAULT 'UTC', + Homepage TEXT NULL DEFAULT NULL, + IRCNick VARCHAR(32) NOT NULL DEFAULT '', + PGPKey VARCHAR(40) NULL DEFAULT NULL, + LastLogin BIGINT UNSIGNED NOT NULL DEFAULT 0, + LastLoginIPAddress VARCHAR(45) NULL DEFAULT NULL, + LastSSHLogin BIGINT UNSIGNED NOT NULL DEFAULT 0, + LastSSHLoginIPAddress VARCHAR(45) NULL DEFAULT NULL, + InactivityTS BIGINT UNSIGNED NOT NULL DEFAULT 0, + RegistrationTS TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + CommentNotify TINYINT(1) NOT NULL DEFAULT 1, + UpdateNotify TINYINT(1) NOT NULL DEFAULT 0, + OwnershipNotify TINYINT(1) NOT NULL DEFAULT 1, + UNIQUE (Username), + UNIQUE (Email), + FOREIGN KEY (AccountTypeID) REFERENCES AccountTypes(ID) ON DELETE NO ACTION +); +CREATE INDEX UsersAccountTypeID ON Users (AccountTypeID); + + +-- SSH public keys used for the aurweb SSH/Git interface. +-- +CREATE TABLE SSHPubKeys ( + UserID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + Fingerprint VARCHAR(44) NOT NULL, + PubKey VARCHAR(4096) NOT NULL, + FOREIGN KEY (UserID) REFERENCES Users(ID) ON DELETE CASCADE +); + + +-- Track Users logging in/out of AUR web site. +-- +CREATE TABLE Sessions ( + UsersID INTEGER NOT NULL, + SessionID CHAR(32) NOT NULL, + LastUpdateTS BIGINT UNSIGNED NOT NULL, + FOREIGN KEY (UsersID) REFERENCES Users(ID) ON DELETE CASCADE, + UNIQUE (SessionID) +); + + +-- Information on package bases +-- +CREATE TABLE PackageBases ( + ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + Name VARCHAR(255) NOT NULL, + NumVotes INTEGER UNSIGNED NOT NULL DEFAULT 0, + Popularity DECIMAL(10,6) NOT NULL DEFAULT 0, + OutOfDateTS BIGINT UNSIGNED NULL DEFAULT NULL, + FlaggerComment TEXT NOT NULL DEFAULT '', + SubmittedTS BIGINT UNSIGNED NOT NULL, + ModifiedTS BIGINT UNSIGNED NOT NULL, + FlaggerUID INTEGER UNSIGNED NULL DEFAULT NULL, -- who flagged the package out-of-date? + SubmitterUID INTEGER UNSIGNED NULL DEFAULT NULL, -- who submitted it? + MaintainerUID INTEGER UNSIGNED NULL DEFAULT NULL, -- User + PackagerUID INTEGER UNSIGNED NULL DEFAULT NULL, -- Last packager + UNIQUE (Name), + FOREIGN KEY (FlaggerUID) REFERENCES Users(ID) ON DELETE SET NULL, + -- deleting a user will cause packages to be orphaned, not deleted + FOREIGN KEY (SubmitterUID) REFERENCES Users(ID) ON DELETE SET NULL, + FOREIGN KEY (MaintainerUID) REFERENCES Users(ID) ON DELETE SET NULL, + FOREIGN KEY (PackagerUID) REFERENCES Users(ID) ON DELETE SET NULL +); +CREATE INDEX BasesNumVotes ON PackageBases (NumVotes); +CREATE INDEX BasesSubmitterUID ON PackageBases (SubmitterUID); +CREATE INDEX BasesMaintainerUID ON PackageBases (MaintainerUID); +CREATE INDEX BasesPackagerUID ON PackageBases (PackagerUID); + + +-- Keywords of package bases +-- +CREATE TABLE PackageKeywords ( + PackageBaseID INTEGER UNSIGNED NOT NULL, + Keyword VARCHAR(255) NOT NULL DEFAULT '', + PRIMARY KEY (PackageBaseID, Keyword), + FOREIGN KEY (PackageBaseID) REFERENCES PackageBases(ID) ON DELETE CASCADE +); + + +-- Information about the actual packages +-- +CREATE TABLE Packages ( + ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + PackageBaseID INTEGER UNSIGNED NOT NULL, + Name VARCHAR(255) NOT NULL, + Version VARCHAR(255) NOT NULL DEFAULT '', + Description VARCHAR(255) NULL DEFAULT NULL, + URL VARCHAR(8000) NULL DEFAULT NULL, + UNIQUE (Name), + FOREIGN KEY (PackageBaseID) REFERENCES PackageBases(ID) ON DELETE CASCADE +); + + +-- Information about licenses +-- +CREATE TABLE Licenses ( + ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + Name VARCHAR(255) NOT NULL, + UNIQUE (Name) +); + + +-- Information about package-license-relations +-- +CREATE TABLE PackageLicenses ( + PackageID INTEGER UNSIGNED NOT NULL, + LicenseID INTEGER UNSIGNED NOT NULL, + PRIMARY KEY (PackageID, LicenseID), + FOREIGN KEY (PackageID) REFERENCES Packages(ID) ON DELETE CASCADE, + FOREIGN KEY (LicenseID) REFERENCES Licenses(ID) ON DELETE CASCADE +); + + +-- Information about groups +-- +CREATE TABLE Groups ( + ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + Name VARCHAR(255) NOT NULL, + UNIQUE (Name) +); + + +-- Information about package-group-relations +-- +CREATE TABLE PackageGroups ( + PackageID INTEGER UNSIGNED NOT NULL, + GroupID INTEGER UNSIGNED NOT NULL, + PRIMARY KEY (PackageID, GroupID), + FOREIGN KEY (PackageID) REFERENCES Packages(ID) ON DELETE CASCADE, + FOREIGN KEY (GroupID) REFERENCES Groups(ID) ON DELETE CASCADE +); + + +-- Define the package dependency types +-- +CREATE TABLE DependencyTypes ( + ID TINYINT UNSIGNED NOT NULL, + Name VARCHAR(32) NOT NULL DEFAULT '', + PRIMARY KEY (ID) +); +INSERT INTO DependencyTypes VALUES (1, 'depends'); +INSERT INTO DependencyTypes VALUES (2, 'makedepends'); +INSERT INTO DependencyTypes VALUES (3, 'checkdepends'); +INSERT INTO DependencyTypes VALUES (4, 'optdepends'); + + +-- Track which dependencies a package has +-- +CREATE TABLE PackageDepends ( + PackageID INTEGER UNSIGNED NOT NULL, + DepTypeID TINYINT UNSIGNED NOT NULL, + DepName VARCHAR(255) NOT NULL, + DepCondition VARCHAR(255), + DepArch VARCHAR(255) NULL DEFAULT NULL, + FOREIGN KEY (PackageID) REFERENCES Packages(ID) ON DELETE CASCADE, + FOREIGN KEY (DepTypeID) REFERENCES DependencyTypes(ID) ON DELETE NO ACTION +); +CREATE INDEX DependsPackageID ON PackageDepends (PackageID); +CREATE INDEX DependsDepName ON PackageDepends (DepName); + + +-- Define the package relation types +-- +CREATE TABLE RelationTypes ( + ID TINYINT UNSIGNED NOT NULL, + Name VARCHAR(32) NOT NULL DEFAULT '', + PRIMARY KEY (ID) +); +INSERT INTO RelationTypes VALUES (1, 'conflicts'); +INSERT INTO RelationTypes VALUES (2, 'provides'); +INSERT INTO RelationTypes VALUES (3, 'replaces'); + + +-- Track which conflicts, provides and replaces a package has +-- +CREATE TABLE PackageRelations ( + PackageID INTEGER UNSIGNED NOT NULL, + RelTypeID TINYINT UNSIGNED NOT NULL, + RelName VARCHAR(255) NOT NULL, + RelCondition VARCHAR(255), + RelArch VARCHAR(255) NULL DEFAULT NULL, + FOREIGN KEY (PackageID) REFERENCES Packages(ID) ON DELETE CASCADE, + FOREIGN KEY (RelTypeID) REFERENCES RelationTypes(ID) ON DELETE NO ACTION +); +CREATE INDEX RelationsPackageID ON PackageRelations (PackageID); +CREATE INDEX RelationsRelName ON PackageRelations (RelName); + + +-- Track which sources a package has +-- +CREATE TABLE PackageSources ( + PackageID INTEGER UNSIGNED NOT NULL, + Source VARCHAR(8000) NOT NULL DEFAULT '/dev/null', + SourceArch VARCHAR(255) NULL DEFAULT NULL, + FOREIGN KEY (PackageID) REFERENCES Packages(ID) ON DELETE CASCADE +); +CREATE INDEX SourcesPackageID ON PackageSources (PackageID); + + +-- Track votes for packages +-- +CREATE TABLE PackageVotes ( + UsersID INTEGER UNSIGNED NOT NULL, + PackageBaseID INTEGER UNSIGNED NOT NULL, + VoteTS BIGINT UNSIGNED NULL DEFAULT NULL, + FOREIGN KEY (UsersID) REFERENCES Users(ID) ON DELETE CASCADE, + FOREIGN KEY (PackageBaseID) REFERENCES PackageBases(ID) ON DELETE CASCADE +); +CREATE UNIQUE INDEX VoteUsersIDPackageID ON PackageVotes (UsersID, PackageBaseID); +CREATE INDEX VotesUsersID ON PackageVotes (UsersID); +CREATE INDEX VotesPackageBaseID ON PackageVotes (PackageBaseID); + +-- Record comments for packages +-- +CREATE TABLE PackageComments ( + ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + PackageBaseID INTEGER UNSIGNED NOT NULL, + UsersID INTEGER UNSIGNED NULL DEFAULT NULL, + Comments TEXT NOT NULL DEFAULT '', + CommentTS BIGINT UNSIGNED NOT NULL DEFAULT 0, + EditedTS BIGINT UNSIGNED NULL DEFAULT NULL, + EditedUsersID INTEGER UNSIGNED NULL DEFAULT NULL, + DelTS BIGINT UNSIGNED NULL DEFAULT NULL, + DelUsersID INTEGER UNSIGNED NULL DEFAULT NULL, + PinnedTS BIGINT UNSIGNED NOT NULL DEFAULT 0, + FOREIGN KEY (UsersID) REFERENCES Users(ID) ON DELETE SET NULL, + FOREIGN KEY (EditedUsersID) REFERENCES Users(ID) ON DELETE SET NULL, + FOREIGN KEY (DelUsersID) REFERENCES Users(ID) ON DELETE CASCADE, + FOREIGN KEY (PackageBaseID) REFERENCES PackageBases(ID) ON DELETE CASCADE +); +CREATE INDEX CommentsUsersID ON PackageComments (UsersID); +CREATE INDEX CommentsPackageBaseID ON PackageComments (PackageBaseID); + +-- Package base co-maintainers +-- +CREATE TABLE PackageComaintainers ( + UsersID INTEGER UNSIGNED NOT NULL, + PackageBaseID INTEGER UNSIGNED NOT NULL, + Priority INTEGER UNSIGNED NOT NULL, + FOREIGN KEY (UsersID) REFERENCES Users(ID) ON DELETE CASCADE, + FOREIGN KEY (PackageBaseID) REFERENCES PackageBases(ID) ON DELETE CASCADE +); +CREATE INDEX ComaintainersUsersID ON PackageComaintainers (UsersID); +CREATE INDEX ComaintainersPackageBaseID ON PackageComaintainers (PackageBaseID); + +-- Package base notifications +-- +CREATE TABLE PackageNotifications ( + PackageBaseID INTEGER UNSIGNED NOT NULL, + UserID INTEGER UNSIGNED NOT NULL, + FOREIGN KEY (PackageBaseID) REFERENCES PackageBases(ID) ON DELETE CASCADE, + FOREIGN KEY (UserID) REFERENCES Users(ID) ON DELETE CASCADE +); +CREATE UNIQUE INDEX NotifyUserIDPkgID ON PackageNotifications (UserID, PackageBaseID); + +-- Package name blacklist +-- +CREATE TABLE PackageBlacklist ( + ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + Name VARCHAR(64) NOT NULL, + UNIQUE (Name) +); + +-- Providers in the official repositories +-- +CREATE TABLE OfficialProviders ( + ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + Name VARCHAR(64) NOT NULL, + Repo VARCHAR(64) NOT NULL, + Provides VARCHAR(64) NOT NULL +); +CREATE UNIQUE INDEX ProviderNameProvides ON OfficialProviders (Name, Provides); + +-- Define package request types +-- +CREATE TABLE RequestTypes ( + ID TINYINT UNSIGNED NOT NULL, + Name VARCHAR(32) NOT NULL DEFAULT '', + PRIMARY KEY (ID) +); +INSERT INTO RequestTypes VALUES (1, 'deletion'); +INSERT INTO RequestTypes VALUES (2, 'orphan'); +INSERT INTO RequestTypes VALUES (3, 'merge'); + +-- Package requests +-- +CREATE TABLE PackageRequests ( + ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + ReqTypeID TINYINT UNSIGNED NOT NULL, + PackageBaseID INTEGER UNSIGNED NULL, + PackageBaseName VARCHAR(255) NOT NULL, + MergeBaseName VARCHAR(255) NULL, + UsersID INTEGER UNSIGNED NULL DEFAULT NULL, + Comments TEXT NOT NULL DEFAULT '', + ClosureComment TEXT NOT NULL DEFAULT '', + RequestTS BIGINT UNSIGNED NOT NULL DEFAULT 0, + Status TINYINT UNSIGNED NOT NULL DEFAULT 0, + FOREIGN KEY (ReqTypeID) REFERENCES RequestTypes(ID) ON DELETE NO ACTION, + FOREIGN KEY (UsersID) REFERENCES Users(ID) ON DELETE SET NULL, + FOREIGN KEY (PackageBaseID) REFERENCES PackageBases(ID) ON DELETE SET NULL +); +CREATE INDEX RequestsUsersID ON PackageRequests (UsersID); +CREATE INDEX RequestsPackageBaseID ON PackageRequests (PackageBaseID); + +-- Vote information +-- +CREATE TABLE IF NOT EXISTS TU_VoteInfo ( + ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + Agenda TEXT NOT NULL, + User VARCHAR(32) NOT NULL, + Submitted BIGINT UNSIGNED NOT NULL, + End BIGINT UNSIGNED NOT NULL, + Quorum DECIMAL(2, 2) NOT NULL, + SubmitterID INTEGER UNSIGNED NOT NULL, + Yes TINYINT UNSIGNED NOT NULL DEFAULT '0', + No TINYINT UNSIGNED NOT NULL DEFAULT '0', + Abstain TINYINT UNSIGNED NOT NULL DEFAULT '0', + ActiveTUs TINYINT UNSIGNED NOT NULL DEFAULT '0', + FOREIGN KEY (SubmitterID) REFERENCES Users(ID) ON DELETE CASCADE +); + +-- Individual vote records +-- +CREATE TABLE IF NOT EXISTS TU_Votes ( + VoteID INTEGER UNSIGNED NOT NULL, + UserID INTEGER UNSIGNED NOT NULL, + FOREIGN KEY (VoteID) REFERENCES TU_VoteInfo(ID) ON DELETE CASCADE, + FOREIGN KEY (UserID) REFERENCES Users(ID) ON DELETE CASCADE +); + +-- Malicious user banning +-- +CREATE TABLE Bans ( + IPAddress VARCHAR(45) NULL DEFAULT NULL, + BanTS TIMESTAMP NOT NULL, + PRIMARY KEY (IPAddress) +); -- 2.12.0

2 8

[aur-dev] [PATCH] git-update: Check for missing pkgname entries
by Lukas Fleischer 06 Apr '17

06 Apr '17

Reject commits containing .SRCINFO files without any pkgname entries. Suggested-by: Bruno Pagani <bruno.n.pagani(a)gmail.com> Signed-off-by: Lukas Fleischer <lfleischer(a)archlinux.org> --- aurweb/git/update.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aurweb/git/update.py b/aurweb/git/update.py index 3b84eb5..532db92 100755 --- a/aurweb/git/update.py +++ b/aurweb/git/update.py @@ -308,6 +308,9 @@ def main(): die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase), str(commit.id)) + if not metadata['packages']: + die_commit('missing pkgname entry', str(commit.id)) + for pkgname in set(metadata['packages'].keys()): pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) -- 2.12.2

1 0