[aur-dev] [PATCH 0/4] Add Markdown syntax to comments
This patch series moves comment rendering to a separate Python script, adds markdown support and also makes Git commit identifiers clickable. Comments welcome! Lukas Fleischer (4): Render comments when storing them in the database Add Markdown support to package comments Reintroduce backwards-compatible hyperlink syntax Make references to Git commits clickable aurweb/scripts/rendercomment.py | 100 ++++++++++++++++++++++++++++++++++++++++ conf/config.proto | 2 + schema/aur-schema.sql | 1 + setup.py | 1 + test/setup.sh | 1 + test/t2600-rendercomment.sh | 23 +++++++++ upgrading/4.6.0.txt | 6 +++ web/lib/pkgbasefuncs.inc.php | 40 ++++++++++++++-- web/template/pkg_comments.php | 4 ++ 9 files changed, 174 insertions(+), 4 deletions(-) create mode 100755 aurweb/scripts/rendercomment.py create mode 100755 test/t2600-rendercomment.sh -- 2.12.2
Instead of converting package comments from plain text to HTML code when they are displayed, do the conversion when the comment is posted and store the rendered result in the database. The conversion itself is done by a Python script which uses Bleach for sanitizing the text. Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org> --- aurweb/scripts/rendercomment.py | 35 +++++++++++++++++++++++++++++++++++ conf/config.proto | 1 + schema/aur-schema.sql | 1 + setup.py | 1 + test/setup.sh | 1 + test/t2600-rendercomment.sh | 22 ++++++++++++++++++++++ upgrading/4.6.0.txt | 6 ++++++ web/lib/pkgbasefuncs.inc.php | 40 ++++++++++++++++++++++++++++++++++++---- web/template/pkg_comments.php | 4 ++++ 9 files changed, 107 insertions(+), 4 deletions(-) create mode 100755 aurweb/scripts/rendercomment.py create mode 100755 test/t2600-rendercomment.sh diff --git a/aurweb/scripts/rendercomment.py b/aurweb/scripts/rendercomment.py new file mode 100755 index 0000000..593cd36 --- /dev/null +++ b/aurweb/scripts/rendercomment.py @@ -0,0 +1,35 @@ +#!/usr/bin/python3 + +import sys +import bleach + +import aurweb.db + + +def get_comment(conn, commentid): + cur = conn.execute('SELECT Comments FROM PackageComments WHERE ID = ?', + [commentid]) + return cur.fetchone()[0] + + +def save_rendered_comment(conn, commentid, html): + conn.execute('UPDATE PackageComments SET RenderedComment = ? WHERE ID = ?', + [html, commentid]) + + +def main(): + commentid = int(sys.argv[1]) + + conn = aurweb.db.Connection() + + html = get_comment(conn, commentid) + html = html.replace('\n', '<br>') + html = bleach.clean(html, tags=['br']) + save_rendered_comment(conn, commentid, html) + + conn.commit() + conn.close() + + +if __name__ == '__main__': + main() diff --git a/conf/config.proto b/conf/config.proto index df10b99..094d821 100644 --- a/conf/config.proto +++ b/conf/config.proto @@ -33,6 +33,7 @@ log_uri = https://aur.archlinux.org/cgit/aur.git/log/?h=%s snapshot_uri = /cgit/aur.git/snapshot/%s.tar.gz enable-maintenance = 1 maintenance-exceptions = 127.0.0.1 +render-comment-cmd = /usr/local/bin/aurweb-rendercomment [notifications] notify-cmd = /usr/local/bin/aurweb-notify diff --git a/schema/aur-schema.sql b/schema/aur-schema.sql index be6f3e5..e584165 100644 --- a/schema/aur-schema.sql +++ b/schema/aur-schema.sql @@ -255,6 +255,7 @@ CREATE TABLE PackageComments ( PackageBaseID INTEGER UNSIGNED NOT NULL, UsersID INTEGER UNSIGNED NULL DEFAULT NULL, Comments TEXT NOT NULL, + RenderedComment TEXT NOT NULL, CommentTS BIGINT UNSIGNED NOT NULL DEFAULT 0, EditedTS BIGINT UNSIGNED NULL DEFAULT NULL, EditedUsersID INTEGER UNSIGNED NULL DEFAULT NULL, diff --git a/setup.py b/setup.py index 99dbfed..9d10cc1 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ setup( 'aurweb-notify = aurweb.scripts.notify:main', 'aurweb-pkgmaint = aurweb.scripts.pkgmaint:main', 'aurweb-popupdate = aurweb.scripts.popupdate:main', + 'aurweb-rendercomment = aurweb.scripts.rendercomment:main', 'aurweb-tuvotereminder = aurweb.scripts.tuvotereminder:main', ], }, diff --git a/test/setup.sh b/test/setup.sh index b71e73e..f29695a 100644 --- a/test/setup.sh +++ b/test/setup.sh @@ -16,6 +16,7 @@ TUVOTEREMINDER="$TOPLEVEL/aurweb/scripts/tuvotereminder.py" PKGMAINT="$TOPLEVEL/aurweb/scripts/pkgmaint.py" AURBLUP="$TOPLEVEL/aurweb/scripts/aurblup.py" NOTIFY="$TOPLEVEL/aurweb/scripts/notify.py" +RENDERCOMMENT="$TOPLEVEL/aurweb/scripts/rendercomment.py" # Create the configuration file and a dummy notification script. cat >config <<-EOF diff --git a/test/t2600-rendercomment.sh b/test/t2600-rendercomment.sh new file mode 100755 index 0000000..8d79336 --- /dev/null +++ b/test/t2600-rendercomment.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +test_description='rendercomment tests' + +. ./setup.sh + +test_expect_success 'Test comment rendering.' ' + cat <<-EOD | sqlite3 aur.db && + INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (1, 1, "Hello world! + This is a comment.", ""); + EOD + "$RENDERCOMMENT" 1 && + cat <<-EOD >expected && + Hello world!<br>This is a comment. + EOD + cat <<-EOD | sqlite3 aur.db >actual && + SELECT RenderedComment FROM PackageComments WHERE ID = 1; + EOD + test_cmp actual expected +' + +test_done diff --git a/upgrading/4.6.0.txt b/upgrading/4.6.0.txt index 45740f4..b051bac 100644 --- a/upgrading/4.6.0.txt +++ b/upgrading/4.6.0.txt @@ -9,3 +9,9 @@ UPDATE PackageDepends SET DepName = SUBSTRING(DepName FROM 1 FOR POSITION(': ' IN DepName) - 1) WHERE POSITION(': ' IN DepName) > 0; --- + +2. Add RenderedComment column to PackageComments: + +--- +ALTER TABLE PackageComments ADD COLUMN RenderedComment TEXT NOT NULL; +--- diff --git a/web/lib/pkgbasefuncs.inc.php b/web/lib/pkgbasefuncs.inc.php index 57933e8..3e78309 100644 --- a/web/lib/pkgbasefuncs.inc.php +++ b/web/lib/pkgbasefuncs.inc.php @@ -54,7 +54,7 @@ function pkgbase_comments($base_id, $limit, $include_deleted, $only_pinned=false $dbh = DB::connect(); $q = "SELECT PackageComments.ID, A.UserName AS UserName, UsersID, Comments, "; $q.= "PackageBaseID, CommentTS, DelTS, EditedTS, B.UserName AS EditUserName, "; - $q.= "DelUsersID, C.UserName AS DelUserName, "; + $q.= "DelUsersID, C.UserName AS DelUserName, RenderedComment, "; $q.= "PinnedTS FROM PackageComments "; $q.= "LEFT JOIN Users A ON PackageComments.UsersID = A.ID "; $q.= "LEFT JOIN Users B ON PackageComments.EditedUsersID = B.ID "; @@ -79,6 +79,36 @@ function pkgbase_comments($base_id, $limit, $include_deleted, $only_pinned=false return $result->fetchAll(); } +/* + * Invoke the comment rendering script. + * + * @param int $id ID of the comment to render + * + * @return void + */ +function render_comment($id) { + $cmd = config_get('options', 'render-comment-cmd'); + $cmd .= ' ' . intval($id); + + $descspec = array( + 0 => array('pipe', 'r'), + 1 => array('pipe', 'w'), + 2 => array('pipe', 'w') + ); + + $p = proc_open($cmd, $descspec, $pipes); + + if (!is_resource($p)) { + return false; + } + + fclose($pipes[0]); + fclose($pipes[1]); + fclose($pipes[2]); + + return proc_close($p); +} + /** * Add a comment to a package page and send out appropriate notifications * @@ -96,12 +126,14 @@ function pkgbase_add_comment($base_id, $uid, $comment) { } $q = "INSERT INTO PackageComments "; - $q.= "(PackageBaseID, UsersID, Comments, CommentTS) VALUES ("; - $q.= intval($base_id) . ", " . $uid . ", "; - $q.= $dbh->quote($comment) . ", " . strval(time()) . ")"; + $q.= "(PackageBaseID, UsersID, Comments, RenderedComment, CommentTS) "; + $q.= "VALUES (" . intval($base_id) . ", " . $uid . ", "; + $q.= $dbh->quote($comment) . ", '', " . strval(time()) . ")"; $dbh->exec($q); $comment_id = $dbh->lastInsertId(); + render_comment($comment_id); + notify(array('comment', $uid, $base_id, $comment_id)); return array(true, __('Comment has been added.')); diff --git a/web/template/pkg_comments.php b/web/template/pkg_comments.php index c23ec42..f973b74 100644 --- a/web/template/pkg_comments.php +++ b/web/template/pkg_comments.php @@ -103,9 +103,13 @@ if (!isset($count)) { <?php endif; ?> </h4> <div id="<?= isset($pinned) ? "pinned-" : "comment-" ?><?= $row['ID'] ?>-content" class="article-content<?php if ($is_deleted): ?> comment-deleted<?php endif; ?>"> + <?php if (!empty($row['RenderedComment'])): ?> + <?= $row['RenderedComment'] ?> + <?php else: ?> <p> <?= parse_comment($row['Comments']) ?> </p> + <?php endif; ?> </div> <?php endwhile; ?> -- 2.12.2
Support Markdown syntax in package comments. Among other things, this makes it easier to paste command line output and patches. Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org> --- aurweb/scripts/rendercomment.py | 8 +++++--- test/t2600-rendercomment.sh | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/aurweb/scripts/rendercomment.py b/aurweb/scripts/rendercomment.py index 593cd36..7e8a16b 100755 --- a/aurweb/scripts/rendercomment.py +++ b/aurweb/scripts/rendercomment.py @@ -2,6 +2,7 @@ import sys import bleach +import markdown import aurweb.db @@ -22,9 +23,10 @@ def main(): conn = aurweb.db.Connection() - html = get_comment(conn, commentid) - html = html.replace('\n', '<br>') - html = bleach.clean(html, tags=['br']) + text = get_comment(conn, commentid) + html = markdown.markdown(text, extensions=['nl2br']) + allowed_tags = bleach.sanitizer.ALLOWED_TAGS + ['p', 'br'] + html = bleach.clean(html, tags=allowed_tags) save_rendered_comment(conn, commentid, html) conn.commit() diff --git a/test/t2600-rendercomment.sh b/test/t2600-rendercomment.sh index 8d79336..50a5adb 100755 --- a/test/t2600-rendercomment.sh +++ b/test/t2600-rendercomment.sh @@ -11,7 +11,8 @@ test_expect_success 'Test comment rendering.' ' EOD "$RENDERCOMMENT" 1 && cat <<-EOD >expected && - Hello world!<br>This is a comment. + <p>Hello world!<br> + This is a comment.</p> EOD cat <<-EOD | sqlite3 aur.db >actual && SELECT RenderedComment FROM PackageComments WHERE ID = 1; -- 2.12.2
Before switching to the new comment rendering script and Markdown, no special syntax was needed to make URLs clickable. Reintroduce this feature and automatically detect links in addition to the hyperlink syntax already supported by Markdown. Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org> --- aurweb/scripts/rendercomment.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/aurweb/scripts/rendercomment.py b/aurweb/scripts/rendercomment.py index 7e8a16b..c8921f8 100755 --- a/aurweb/scripts/rendercomment.py +++ b/aurweb/scripts/rendercomment.py @@ -1,5 +1,6 @@ #!/usr/bin/python3 +import re import sys import bleach import markdown @@ -7,6 +8,19 @@ import markdown import aurweb.db +class LinkifyPreprocessor(markdown.preprocessors.Preprocessor): + _urlre = re.compile(r'(\b(?:https?|ftp):\/\/[\w\/\#~:.?+=&%@!\-;,]+?' + r'(?=[.:?\-;,]*(?:[^\w\/\#~:.?+=&%@!\-;,]|$)))') + + def run(self, lines): + return [self._urlre.sub(r'<\1>', line) for line in lines] + + +class LinkifyExtension(markdown.extensions.Extension): + def extendMarkdown(self, md, md_globals): + md.preprocessors.add('linkify', LinkifyPreprocessor(md), '_end') + + def get_comment(conn, commentid): cur = conn.execute('SELECT Comments FROM PackageComments WHERE ID = ?', [commentid]) @@ -24,7 +38,7 @@ def main(): conn = aurweb.db.Connection() text = get_comment(conn, commentid) - html = markdown.markdown(text, extensions=['nl2br']) + html = markdown.markdown(text, extensions=['nl2br', LinkifyExtension()]) allowed_tags = bleach.sanitizer.ALLOWED_TAGS + ['p', 'br'] html = bleach.clean(html, tags=allowed_tags) save_rendered_comment(conn, commentid, html) -- 2.12.2
Automatically detect Git commit identifiers, shorten them, and make them link to the cgit interface. Implements FS#43290. Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org> --- aurweb/scripts/rendercomment.py | 59 +++++++++++++++++++++++++++++++++++++---- conf/config.proto | 1 + 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/aurweb/scripts/rendercomment.py b/aurweb/scripts/rendercomment.py index c8921f8..659e18a 100755 --- a/aurweb/scripts/rendercomment.py +++ b/aurweb/scripts/rendercomment.py @@ -1,12 +1,17 @@ #!/usr/bin/python3 import re +import pygit2 import sys import bleach import markdown +import aurweb.config import aurweb.db +repo_path = aurweb.config.get('serve', 'repo-path') +commit_uri = aurweb.config.get('options', 'commit_uri') + class LinkifyPreprocessor(markdown.preprocessors.Preprocessor): _urlre = re.compile(r'(\b(?:https?|ftp):\/\/[\w\/\#~:.?+=&%@!\-;,]+?' @@ -21,10 +26,53 @@ class LinkifyExtension(markdown.extensions.Extension): md.preprocessors.add('linkify', LinkifyPreprocessor(md), '_end') +class GitCommitsPreprocessor(markdown.preprocessors.Preprocessor): + _oidre = re.compile(r'(\b)([0-9a-f]{7,40})(\b)') + _repo = pygit2.Repository(repo_path) + _head = None + + def __init__(self, md, head): + self._head = head + super(markdown.preprocessors.Preprocessor, self).__init__(md) + + def handleMatch(self, m): + oid = m.group(2) + if oid not in self._repo: + return oid + + prefixlen = 12 + while prefixlen < 40: + if oid[:prefixlen] in self._repo: + break + prefixlen += 1 + + html = '[`' + oid[:prefixlen] + '`]' + html += '(' + commit_uri % (self._head, oid[:prefixlen]) + ')' + + return html + + def run(self, lines): + return [self._oidre.sub(self.handleMatch, line) for line in lines] + + +class GitCommitsExtension(markdown.extensions.Extension): + _head = None + + def __init__(self, head): + self._head = head + super(markdown.extensions.Extension, self).__init__() + + def extendMarkdown(self, md, md_globals): + preprocessor = GitCommitsPreprocessor(md, self._head) + md.preprocessors.add('git-commits', preprocessor, '_end') + + def get_comment(conn, commentid): - cur = conn.execute('SELECT Comments FROM PackageComments WHERE ID = ?', - [commentid]) - return cur.fetchone()[0] + cur = conn.execute('SELECT PackageComments.Comments, PackageBases.Name ' + 'FROM PackageComments INNER JOIN PackageBases ' + 'ON PackageBases.ID = PackageComments.PackageBaseID ' + 'WHERE PackageComments.ID = ?', [commentid]) + return cur.fetchone() def save_rendered_comment(conn, commentid, html): @@ -37,8 +85,9 @@ def main(): conn = aurweb.db.Connection() - text = get_comment(conn, commentid) - html = markdown.markdown(text, extensions=['nl2br', LinkifyExtension()]) + text, pkgbase = get_comment(conn, commentid) + html = markdown.markdown(text, extensions=['nl2br', LinkifyExtension(), + GitCommitsExtension(pkgbase)]) allowed_tags = bleach.sanitizer.ALLOWED_TAGS + ['p', 'br'] html = bleach.clean(html, tags=allowed_tags) save_rendered_comment(conn, commentid, html) diff --git a/conf/config.proto b/conf/config.proto index 094d821..84351d9 100644 --- a/conf/config.proto +++ b/conf/config.proto @@ -30,6 +30,7 @@ auto_orphan_age = 15552000 auto_delete_age = 86400 source_file_uri = https://aur.archlinux.org/cgit/aur.git/tree/%s?h=%s log_uri = https://aur.archlinux.org/cgit/aur.git/log/?h=%s +commit_uri = https://aur.archlinux.org/cgit/aur.git/log/?h=%s&id=%s snapshot_uri = /cgit/aur.git/snapshot/%s.tar.gz enable-maintenance = 1 maintenance-exceptions = 127.0.0.1 -- 2.12.2
On Sun, 23 Apr 2017 at 18:53:02, Lukas Fleischer wrote:
Automatically detect Git commit identifiers, shorten them, and make them link to the cgit interface.
Implements FS#43290.
Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org> --- aurweb/scripts/rendercomment.py | 59 +++++++++++++++++++++++++++++++++++++---- conf/config.proto | 1 + 2 files changed, 55 insertions(+), 5 deletions(-) [...] diff --git a/conf/config.proto b/conf/config.proto index 094d821..84351d9 100644 --- a/conf/config.proto +++ b/conf/config.proto @@ -30,6 +30,7 @@ auto_orphan_age = 15552000 auto_delete_age = 86400 source_file_uri = https://aur.archlinux.org/cgit/aur.git/tree/%s?h=%s log_uri = https://aur.archlinux.org/cgit/aur.git/log/?h=%s +commit_uri = https://aur.archlinux.org/cgit/aur.git/log/?h=%s&id=%s
Small typo here, this should link to the commit page instead of the log page. I queued a fixed version on pu which also includes some test suite changes.
snapshot_uri = /cgit/aur.git/snapshot/%s.tar.gz enable-maintenance = 1 maintenance-exceptions = 127.0.0.1 -- 2.12.2
participants (1)
-
Lukas Fleischer