Move the main program logic of all scripts to main() methods such that they can be used as modules and easily be invoked by setuptools wrapper scripts. Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org> --- scripts/aurblup.py | 86 +++++++++++++++++++++++++---------------------- scripts/mkpkglists.py | 52 ++++++++++++++++------------ scripts/notify.py | 6 +++- scripts/pkgmaint.py | 23 ++++++++----- scripts/popupdate.py | 31 ++++++++++------- scripts/tuvotereminder.py | 28 +++++++++------ 6 files changed, 132 insertions(+), 94 deletions(-) diff --git a/scripts/aurblup.py b/scripts/aurblup.py index 6733b45..07119b5 100755 --- a/scripts/aurblup.py +++ b/scripts/aurblup.py @@ -18,43 +18,49 @@ db_path = config.get('aurblup', 'db-path') sync_dbs = config.get('aurblup', 'sync-dbs').split(' ') servers = config.get('aurblup', 'servers').split(' ') -blacklist = set() -providers = set() -repomap = dict() - -h = pyalpm.Handle("/", db_path) -for sync_db in sync_dbs: - repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL) - repo.servers = [server.replace("%s", sync_db) for server in servers] - t = h.init_transaction() - repo.update(False) - t.release() - - for pkg in repo.pkgcache: - blacklist.add(pkg.name) - [blacklist.add(x) for x in pkg.replaces] - providers.add((pkg.name, pkg.name)) - repomap[(pkg.name, pkg.name)] = repo.name - for provision in pkg.provides: - provisionname = re.sub(r'(<|=|>).*', '', provision) - providers.add((pkg.name, provisionname)) - repomap[(pkg.name, provisionname)] = repo.name - -db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) -cur = db.cursor() - -cur.execute("SELECT Name, Provides FROM OfficialProviders") -oldproviders = set(cur.fetchall()) - -for pkg, provides in providers.difference(oldproviders): - repo = repomap[(pkg, provides)] - cur.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) " - "VALUES (%s, %s, %s)", [pkg, repo, provides]) -for pkg, provides in oldproviders.difference(providers): - cur.execute("DELETE FROM OfficialProviders " - "WHERE Name = %s AND Provides = %s", [pkg, provides]) - -db.commit() -db.close() + +def main(): + blacklist = set() + providers = set() + repomap = dict() + + h = pyalpm.Handle("/", db_path) + for sync_db in sync_dbs: + repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL) + repo.servers = [server.replace("%s", sync_db) for server in servers] + t = h.init_transaction() + repo.update(False) + t.release() + + for pkg in repo.pkgcache: + blacklist.add(pkg.name) + [blacklist.add(x) for x in pkg.replaces] + providers.add((pkg.name, pkg.name)) + repomap[(pkg.name, pkg.name)] = repo.name + for provision in pkg.provides: + provisionname = re.sub(r'(<|=|>).*', '', provision) + providers.add((pkg.name, provisionname)) + repomap[(pkg.name, provisionname)] = repo.name + + db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket, buffered=True) + cur = db.cursor() + + cur.execute("SELECT Name, Provides FROM OfficialProviders") + oldproviders = set(cur.fetchall()) + + for pkg, provides in providers.difference(oldproviders): + repo = repomap[(pkg, provides)] + cur.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) " + "VALUES (%s, %s, %s)", [pkg, repo, provides]) + for pkg, provides in oldproviders.difference(providers): + cur.execute("DELETE FROM OfficialProviders " + "WHERE Name = %s AND Provides = %s", [pkg, provides]) + + db.commit() + db.close() + + +if __name__ == '__main__': + main() diff --git a/scripts/mkpkglists.py b/scripts/mkpkglists.py index a6f8a19..8cab0a9 100755 --- a/scripts/mkpkglists.py +++ b/scripts/mkpkglists.py @@ -17,25 +17,33 @@ aur_db_user = config.get('database', 'user') aur_db_pass = config.get('database', 'password') aur_db_socket = config.get('database', 'socket') -db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) -cur = db.cursor() - -datestr = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") -pkglist_header = "# AUR package list, generated on " + datestr -pkgbaselist_header = "# AUR package base list, generated on " + datestr - -with gzip.open(docroot + "packages.gz", "w") as f: - f.write(bytes(pkglist_header + "\n", "UTF-8")) - cur.execute("SELECT Packages.Name FROM Packages INNER JOIN PackageBases " + - "ON PackageBases.ID = Packages.PackageBaseID " + - "WHERE PackageBases.PackagerUID IS NOT NULL") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - -with gzip.open(docroot + "pkgbase.gz", "w") as f: - f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) - cur.execute("SELECT Name FROM PackageBases WHERE PackagerUID IS NOT NULL") - f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - -db.close() + +def main(): + db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket, buffered=True) + cur = db.cursor() + + datestr = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") + pkglist_header = "# AUR package list, generated on " + datestr + pkgbaselist_header = "# AUR package base list, generated on " + datestr + + with gzip.open(docroot + "packages.gz", "w") as f: + f.write(bytes(pkglist_header + "\n", "UTF-8")) + cur.execute("SELECT Packages.Name FROM Packages " + + "INNER JOIN PackageBases " + + "ON PackageBases.ID = Packages.PackageBaseID " + + "WHERE PackageBases.PackagerUID IS NOT NULL") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + + with gzip.open(docroot + "pkgbase.gz", "w") as f: + f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) + cur.execute("SELECT Name FROM PackageBases " + + "WHERE PackagerUID IS NOT NULL") + f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) + + db.close() + + +if __name__ == '__main__': + main() diff --git a/scripts/notify.py b/scripts/notify.py index aeb59c3..cc99747 100755 --- a/scripts/notify.py +++ b/scripts/notify.py @@ -427,7 +427,7 @@ def tu_vote_reminder(cur, vote_id): send_notification(to, subject, body, refs) -if __name__ == '__main__': +def main(): action = sys.argv[1] action_map = { 'send-resetkey': send_resetkey, @@ -454,3 +454,7 @@ if __name__ == '__main__': db.commit() db.close() + + +if __name__ == '__main__': + main() diff --git a/scripts/pkgmaint.py b/scripts/pkgmaint.py index 0eb9422..346b046 100755 --- a/scripts/pkgmaint.py +++ b/scripts/pkgmaint.py @@ -13,13 +13,20 @@ aur_db_user = config.get('database', 'user') aur_db_pass = config.get('database', 'password') aur_db_socket = config.get('database', 'socket') -db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) -cur = db.cursor() -cur.execute("DELETE FROM PackageBases WHERE " + - "UNIX_TIMESTAMP() - SubmittedTS > 86400 AND PackagerUID IS NULL") +def main(): + db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket, buffered=True) + cur = db.cursor() -db.commit() -db.close() + cur.execute("DELETE FROM PackageBases WHERE " + + "UNIX_TIMESTAMP() - SubmittedTS > 86400 " + + "AND PackagerUID IS NULL") + + db.commit() + db.close() + + +if __name__ == '__main__': + main() diff --git a/scripts/popupdate.py b/scripts/popupdate.py index f3ba513..26d8379 100755 --- a/scripts/popupdate.py +++ b/scripts/popupdate.py @@ -13,18 +13,25 @@ aur_db_user = config.get('database', 'user') aur_db_pass = config.get('database', 'password') aur_db_socket = config.get('database', 'socket') -db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) -cur = db.cursor() -cur.execute("UPDATE PackageBases SET NumVotes = (SELECT COUNT(*) FROM " + - "PackageVotes WHERE PackageVotes.PackageBaseID = PackageBases.ID)") +def main(): + db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket, buffered=True) + cur = db.cursor() -cur.execute("UPDATE PackageBases SET Popularity = (" + - "SELECT COALESCE(SUM(POWER(0.98, (UNIX_TIMESTAMP() - VoteTS) / 86400)), 0.0) " + - "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + - "PackageBases.ID AND NOT VoteTS IS NULL)") + cur.execute("UPDATE PackageBases SET NumVotes = (" + + "SELECT COUNT(*) FROM PackageVotes " + + "WHERE PackageVotes.PackageBaseID = PackageBases.ID)") -db.commit() -db.close() + cur.execute("UPDATE PackageBases SET Popularity = (" + + "SELECT COALESCE(SUM(POWER(0.98, (UNIX_TIMESTAMP() - VoteTS) / 86400)), 0.0) " + + "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + + "PackageBases.ID AND NOT VoteTS IS NULL)") + + db.commit() + db.close() + + +if __name__ == '__main__': + main() diff --git a/scripts/tuvotereminder.py b/scripts/tuvotereminder.py index 0992623..dc16397 100755 --- a/scripts/tuvotereminder.py +++ b/scripts/tuvotereminder.py @@ -16,17 +16,23 @@ aur_db_pass = config.get('database', 'password') aur_db_socket = config.get('database', 'socket') notify_cmd = config.get('notifications', 'notify-cmd') -db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) -cur = db.cursor() -now = int(time.time()) -filter_from = now + 500 -filter_to = now + 172800 +def main(): + db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket, buffered=True) + cur = db.cursor() -cur.execute("SELECT ID FROM TU_VoteInfo WHERE End >= %s AND End <= %s", - [filter_from, filter_to]) + now = int(time.time()) + filter_from = now + 500 + filter_to = now + 172800 -for vote_id in [row[0] for row in cur.fetchall()]: - subprocess.Popen((notify_cmd, 'tu-vote-reminder', str(vote_id))) + cur.execute("SELECT ID FROM TU_VoteInfo WHERE End >= %s AND End <= %s", + [filter_from, filter_to]) + + for vote_id in [row[0] for row in cur.fetchall()]: + subprocess.Popen((notify_cmd, 'tu-vote-reminder', str(vote_id))) + + +if __name__ == '__main__': + main() -- 2.10.0

Move the main program logic of git-server to a main() method such that it can be used as a module and easily be invoked by setuptools wrapper scripts. Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org> --- git-interface/git-serve.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/git-interface/git-serve.py b/git-interface/git-serve.py index 47e7df9..ddec144 100755 --- a/git-interface/git-serve.py +++ b/git-interface/git-serve.py @@ -76,7 +76,7 @@ def create_pkgbase(pkgbase, user): conn.close() -def pkgbase_adopt(pkgbase): +def pkgbase_adopt(pkgbase, user, privileged): pkgbase_id = pkgbase_from_name(pkgbase) if not pkgbase_id: die('{:s}: package base not found: {:s}'.format(action, pkgbase)) @@ -124,7 +124,7 @@ def pkgbase_get_comaintainers(pkgbase): return [row[0] for row in cur.fetchall()] -def pkgbase_set_comaintainers(pkgbase, userlist): +def pkgbase_set_comaintainers(pkgbase, userlist, user, privileged): pkgbase_id = pkgbase_from_name(pkgbase) if not pkgbase_id: die('{:s}: package base not found: {:s}'.format(action, pkgbase)) @@ -183,7 +183,7 @@ def pkgbase_set_comaintainers(pkgbase, userlist): conn.close() -def pkgbase_disown(pkgbase): +def pkgbase_disown(pkgbase, user, privileged): pkgbase_id = pkgbase_from_name(pkgbase) if not pkgbase_id: die('{:s}: package base not found: {:s}'.format(action, pkgbase)) @@ -211,7 +211,7 @@ def pkgbase_disown(pkgbase): new_maintainer_userid = cur.fetchone()[0] comaintainers.remove(new_maintainer) - pkgbase_set_comaintainers(pkgbase, comaintainers) + pkgbase_set_comaintainers(pkgbase, comaintainers, user, privileged) cur = conn.execute("UPDATE PackageBases SET MaintainerUID = ? " + "WHERE ID = ?", [new_maintainer_userid, pkgbase_id]) @@ -365,7 +365,7 @@ elif action == 'adopt': die_with_help("{:s}: too many arguments".format(action)) pkgbase = cmdargv[1] - pkgbase_adopt(pkgbase) + pkgbase_adopt(pkgbase, user, privileged) elif action == 'disown': if len(cmdargv) < 2: die_with_help("{:s}: missing repository name".format(action)) @@ -373,14 +373,14 @@ elif action == 'disown': die_with_help("{:s}: too many arguments".format(action)) pkgbase = cmdargv[1] - pkgbase_disown(pkgbase) + pkgbase_disown(pkgbase, user, privileged) elif action == 'set-comaintainers': if len(cmdargv) < 2: die_with_help("{:s}: missing repository name".format(action)) pkgbase = cmdargv[1] userlist = cmdargv[2:] - pkgbase_set_comaintainers(pkgbase, userlist) + pkgbase_set_comaintainers(pkgbase, userlist, user, privileged) elif action == 'help': cmds = { "adopt <name>": "Adopt a package base.", -- 2.10.0

Move the main program logic of git-update to a main() method such that it can be used as a module and easily be invoked by setuptools wrapper scripts. Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org> --- git-interface/git-update.py | 294 +++++++++++++++++++++++--------------------- 1 file changed, 154 insertions(+), 140 deletions(-) diff --git a/git-interface/git-update.py b/git-interface/git-update.py index be4c9be..36c38ae 100755 --- a/git-interface/git-update.py +++ b/git-interface/git-update.py @@ -233,173 +233,187 @@ def die_commit(msg, commit): exit(1) -repo = pygit2.Repository(repo_path) - -user = os.environ.get("AUR_USER") -pkgbase = os.environ.get("AUR_PKGBASE") -privileged = (os.environ.get("AUR_PRIVILEGED", '0') == '1') -warn_or_die = warn if privileged else die - -if len(sys.argv) == 2 and sys.argv[1] == "restore": - if 'refs/heads/' + pkgbase not in repo.listall_references(): - die('{:s}: repository not found: {:s}'.format(sys.argv[1], pkgbase)) - refname = "refs/heads/master" - sha1_old = sha1_new = repo.lookup_reference('refs/heads/' + pkgbase).target -elif len(sys.argv) == 4: - refname, sha1_old, sha1_new = sys.argv[1:4] -else: - die("invalid arguments") - -if refname != "refs/heads/master": - die("pushing to a branch other than master is restricted") - -conn = db.Connection() - -# Detect and deny non-fast-forwards. -if sha1_old != "0000000000000000000000000000000000000000" and not privileged: - walker = repo.walk(sha1_old, pygit2.GIT_SORT_TOPOLOGICAL) - walker.hide(sha1_new) - if next(walker, None) is not None: - die("denying non-fast-forward (you should pull first)") - -# Prepare the walker that validates new commits. -walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL) -if sha1_old != "0000000000000000000000000000000000000000": - walker.hide(sha1_old) - -# Validate all new commits. -for commit in walker: - for fname in ('.SRCINFO', 'PKGBUILD'): - if fname not in commit.tree: - die_commit("missing {:s}".format(fname), str(commit.id)) - - for treeobj in commit.tree: - blob = repo[treeobj.id] - - if isinstance(blob, pygit2.Tree): - die_commit("the repository must not contain subdirectories", - str(commit.id)) +def main(): + repo = pygit2.Repository(repo_path) + + user = os.environ.get("AUR_USER") + pkgbase = os.environ.get("AUR_PKGBASE") + privileged = (os.environ.get("AUR_PRIVILEGED", '0') == '1') + warn_or_die = warn if privileged else die + + if len(sys.argv) == 2 and sys.argv[1] == "restore": + if 'refs/heads/' + pkgbase not in repo.listall_references(): + die('{:s}: repository not found: {:s}'.format(sys.argv[1], + pkgbase)) + refname = "refs/heads/master" + branchref = 'refs/heads/' + pkgbase + sha1_old = sha1_new = repo.lookup_reference(branchref).target + elif len(sys.argv) == 4: + refname, sha1_old, sha1_new = sys.argv[1:4] + else: + die("invalid arguments") - if not isinstance(blob, pygit2.Blob): - die_commit("not a blob object: {:s}".format(treeobj), - str(commit.id)) + if refname != "refs/heads/master": + die("pushing to a branch other than master is restricted") - if blob.size > max_blob_size: - die_commit("maximum blob size ({:s}) exceeded".format(size_humanize(max_blob_size)), str(commit.id)) + conn = db.Connection() - metadata_raw = repo[commit.tree['.SRCINFO'].id].data.decode() - (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) - if errors: - sys.stderr.write("error: The following errors occurred " - "when parsing .SRCINFO in commit\n") - sys.stderr.write("error: {:s}:\n".format(str(commit.id))) - for error in errors: - for err in error['error']: - sys.stderr.write("error: line {:d}: {:s}\n".format(error['line'], err)) - exit(1) + # Detect and deny non-fast-forwards. + if sha1_old != "0" * 40 and not privileged: + walker = repo.walk(sha1_old, pygit2.GIT_SORT_TOPOLOGICAL) + walker.hide(sha1_new) + if next(walker, None) is not None: + die("denying non-fast-forward (you should pull first)") - metadata_pkgbase = metadata['pkgbase'] - if not re.match(repo_regex, metadata_pkgbase): - die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase), - str(commit.id)) + # Prepare the walker that validates new commits. + walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL) + if sha1_old != "0" * 40: + walker.hide(sha1_old) - for pkgname in set(metadata['packages'].keys()): - pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) + # Validate all new commits. + for commit in walker: + for fname in ('.SRCINFO', 'PKGBUILD'): + if fname not in commit.tree: + die_commit("missing {:s}".format(fname), str(commit.id)) - for field in ('pkgver', 'pkgrel', 'pkgname'): - if field not in pkginfo: - die_commit('missing mandatory field: {:s}'.format(field), + for treeobj in commit.tree: + blob = repo[treeobj.id] + + if isinstance(blob, pygit2.Tree): + die_commit("the repository must not contain subdirectories", str(commit.id)) - if 'epoch' in pkginfo and not pkginfo['epoch'].isdigit(): - die_commit('invalid epoch: {:s}'.format(pkginfo['epoch']), - str(commit.id)) + if not isinstance(blob, pygit2.Blob): + die_commit("not a blob object: {:s}".format(treeobj), + str(commit.id)) - if not re.match(r'[a-z0-9][a-z0-9\.+_-]*$', pkginfo['pkgname']): - die_commit('invalid package name: {:s}'.format(pkginfo['pkgname']), + if blob.size > max_blob_size: + die_commit("maximum blob size ({:s}) exceeded".format( + size_humanize(max_blob_size)), str(commit.id)) + + metadata_raw = repo[commit.tree['.SRCINFO'].id].data.decode() + (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) + if errors: + sys.stderr.write("error: The following errors occurred " + "when parsing .SRCINFO in commit\n") + sys.stderr.write("error: {:s}:\n".format(str(commit.id))) + for error in errors: + for err in error['error']: + sys.stderr.write("error: line {:d}: {:s}\n".format( + error['line'], err)) + exit(1) + + metadata_pkgbase = metadata['pkgbase'] + if not re.match(repo_regex, metadata_pkgbase): + die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase), str(commit.id)) - for field in ('pkgname', 'pkgdesc', 'url'): - if field in pkginfo and len(pkginfo[field]) > 255: - die_commit('{:s} field too long: {:s}'.format(field, pkginfo[field]), - str(commit.id)) + for pkgname in set(metadata['packages'].keys()): + pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) - for field in ('install', 'changelog'): - if field in pkginfo and not pkginfo[field] in commit.tree: - die_commit('missing {:s} file: {:s}'.format(field, pkginfo[field]), - str(commit.id)) + for field in ('pkgver', 'pkgrel', 'pkgname'): + if field not in pkginfo: + die_commit('missing mandatory field: {:s}'.format(field), + str(commit.id)) - for field in extract_arch_fields(pkginfo, 'source'): - fname = field['value'] - if "://" in fname or "lp:" in fname: - continue - if fname not in commit.tree: - die_commit('missing source file: {:s}'.format(fname), + if 'epoch' in pkginfo and not pkginfo['epoch'].isdigit(): + die_commit('invalid epoch: {:s}'.format(pkginfo['epoch']), str(commit.id)) + if not re.match(r'[a-z0-9][a-z0-9\.+_-]*$', pkginfo['pkgname']): + die_commit('invalid package name: {:s}'.format( + pkginfo['pkgname']), str(commit.id)) + + for field in ('pkgname', 'pkgdesc', 'url'): + if field in pkginfo and len(pkginfo[field]) > 255: + die_commit('{:s} field too long: {:s}'.format(field, + pkginfo[field]), str(commit.id)) + + for field in ('install', 'changelog'): + if field in pkginfo and not pkginfo[field] in commit.tree: + die_commit('missing {:s} file: {:s}'.format(field, + pkginfo[field]), str(commit.id)) + + for field in extract_arch_fields(pkginfo, 'source'): + fname = field['value'] + if "://" in fname or "lp:" in fname: + continue + if fname not in commit.tree: + die_commit('missing source file: {:s}'.format(fname), + str(commit.id)) + + # Display a warning if .SRCINFO is unchanged. + if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new): + srcinfo_id_old = repo[sha1_old].tree['.SRCINFO'].id + srcinfo_id_new = repo[sha1_new].tree['.SRCINFO'].id + if srcinfo_id_old == srcinfo_id_new: + warn(".SRCINFO unchanged. " + "The package database will not be updated!") + + # Read .SRCINFO from the HEAD commit. + metadata_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode() + (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) -# Display a warning if .SRCINFO is unchanged. -if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new): - srcinfo_id_old = repo[sha1_old].tree['.SRCINFO'].id - srcinfo_id_new = repo[sha1_new].tree['.SRCINFO'].id - if srcinfo_id_old == srcinfo_id_new: - warn(".SRCINFO unchanged. The package database will not be updated!") + # Ensure that the package base name matches the repository name. + metadata_pkgbase = metadata['pkgbase'] + if metadata_pkgbase != pkgbase: + die('invalid pkgbase: {:s}, expected {:s}'.format(metadata_pkgbase, + pkgbase)) -# Read .SRCINFO from the HEAD commit. -metadata_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode() -(metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) + # Ensure that packages are neither blacklisted nor overwritten. + pkgbase = metadata['pkgbase'] + cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase]) + row = cur.fetchone() + pkgbase_id = row[0] if row else 0 -# Ensure that the package base name matches the repository name. -metadata_pkgbase = metadata['pkgbase'] -if metadata_pkgbase != pkgbase: - die('invalid pkgbase: {:s}, expected {:s}'.format(metadata_pkgbase, pkgbase)) + cur = conn.execute("SELECT Name FROM PackageBlacklist") + blacklist = [row[0] for row in cur.fetchall()] -# Ensure that packages are neither blacklisted nor overwritten. -pkgbase = metadata['pkgbase'] -cur = conn.execute("SELECT ID FROM PackageBases WHERE Name = ?", [pkgbase]) -row = cur.fetchone() -pkgbase_id = row[0] if row else 0 + cur = conn.execute("SELECT Name, Repo FROM OfficialProviders") + providers = dict(cur.fetchall()) -cur = conn.execute("SELECT Name FROM PackageBlacklist") -blacklist = [row[0] for row in cur.fetchall()] + for pkgname in srcinfo.utils.get_package_names(metadata): + pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) + pkgname = pkginfo['pkgname'] -cur = conn.execute("SELECT Name, Repo FROM OfficialProviders") -providers = dict(cur.fetchall()) + if pkgname in blacklist: + warn_or_die('package is blacklisted: {:s}'.format(pkgname)) + if pkgname in providers: + warn_or_die('package already provided by [{:s}]: {:s}'.format( + providers[pkgname], pkgname)) -for pkgname in srcinfo.utils.get_package_names(metadata): - pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) - pkgname = pkginfo['pkgname'] + cur = conn.execute("SELECT COUNT(*) FROM Packages WHERE Name = ? " + + "AND PackageBaseID <> ?", [pkgname, pkgbase_id]) + if cur.fetchone()[0] > 0: + die('cannot overwrite package: {:s}'.format(pkgname)) - if pkgname in blacklist: - warn_or_die('package is blacklisted: {:s}'.format(pkgname)) - if pkgname in providers: - warn_or_die('package already provided by [{:s}]: {:s}'.format(providers[pkgname], pkgname)) + # Create a new package base if it does not exist yet. + if pkgbase_id == 0: + pkgbase_id = create_pkgbase(conn, pkgbase, user) - cur = conn.execute("SELECT COUNT(*) FROM Packages WHERE Name = ? AND " + - "PackageBaseID <> ?", [pkgname, pkgbase_id]) - if cur.fetchone()[0] > 0: - die('cannot overwrite package: {:s}'.format(pkgname)) + # Store package base details in the database. + save_metadata(metadata, conn, user) -# Create a new package base if it does not exist yet. -if pkgbase_id == 0: - pkgbase_id = create_pkgbase(conn, pkgbase, user) + # Create (or update) a branch with the name of the package base for better + # accessibility. + branchref = 'refs/heads/' + pkgbase + repo.create_reference(branchref, sha1_new, True) -# Store package base details in the database. -save_metadata(metadata, conn, user) + # Work around a Git bug: The HEAD ref is not updated when using + # gitnamespaces. This can be removed once the bug fix is included in Git + # mainline. See + # http://git.661346.n2.nabble.com/PATCH-receive-pack-Create-a-HEAD-ref-for-ref... + # for details. + headref = 'refs/namespaces/' + pkgbase + '/HEAD' + repo.create_reference(headref, sha1_new, True) -# Create (or update) a branch with the name of the package base for better -# accessibility. -repo.create_reference('refs/heads/' + pkgbase, sha1_new, True) + # Send package update notifications. + update_notify(conn, user, pkgbase_id) -# Work around a Git bug: The HEAD ref is not updated when using gitnamespaces. -# This can be removed once the bug fix is included in Git mainline. See -# http://git.661346.n2.nabble.com/PATCH-receive-pack-Create-a-HEAD-ref-for-ref... -# for details. -repo.create_reference('refs/namespaces/' + pkgbase + '/HEAD', sha1_new, True) + # Close the database. + cur.close() + conn.close() -# Send package update notifications. -update_notify(conn, user, pkgbase_id) -# Close the database. -cur.close() -conn.close() +if __name__ == '__main__': + main() -- 2.10.0

Instead of using configparser and mysql.connector directly, change all Python scripts to use the config and db Python modules which are now accessible from a common location. Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org> --- scripts/aurblup.py | 37 +++---- scripts/mkpkglists.py | 32 ++---- scripts/notify.py | 272 +++++++++++++++++++++++----------------------- scripts/pkgmaint.py | 28 ++--- scripts/popupdate.py | 40 +++---- scripts/tuvotereminder.py | 24 ++-- 6 files changed, 183 insertions(+), 250 deletions(-) diff --git a/scripts/aurblup.py b/scripts/aurblup.py index 07119b5..0009715 100755 --- a/scripts/aurblup.py +++ b/scripts/aurblup.py @@ -1,22 +1,14 @@ #!/usr/bin/python3 -import configparser -import mysql.connector -import os import pyalpm import re -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") +import aurweb.config +import aurweb.db -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') -db_path = config.get('aurblup', 'db-path') -sync_dbs = config.get('aurblup', 'sync-dbs').split(' ') -servers = config.get('aurblup', 'servers').split(' ') +db_path = aurweb.config.get('aurblup', 'db-path') +sync_dbs = aurweb.config.get('aurblup', 'sync-dbs').split(' ') +servers = aurweb.config.get('aurblup', 'servers').split(' ') def main(): @@ -42,24 +34,21 @@ def main(): providers.add((pkg.name, provisionname)) repomap[(pkg.name, provisionname)] = repo.name - db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) - cur = db.cursor() + conn = aurweb.db.Connection() - cur.execute("SELECT Name, Provides FROM OfficialProviders") + cur = conn.execute("SELECT Name, Provides FROM OfficialProviders") oldproviders = set(cur.fetchall()) for pkg, provides in providers.difference(oldproviders): repo = repomap[(pkg, provides)] - cur.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) " - "VALUES (%s, %s, %s)", [pkg, repo, provides]) + conn.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) " + "VALUES (?, ?, ?)", [pkg, repo, provides]) for pkg, provides in oldproviders.difference(providers): - cur.execute("DELETE FROM OfficialProviders " - "WHERE Name = %s AND Provides = %s", [pkg, provides]) + conn.execute("DELETE FROM OfficialProviders " + "WHERE Name = ? AND Provides = ?", [pkg, provides]) - db.commit() - db.close() + conn.commit() + conn.close() if __name__ == '__main__': diff --git a/scripts/mkpkglists.py b/scripts/mkpkglists.py index 8cab0a9..70cbd13 100755 --- a/scripts/mkpkglists.py +++ b/scripts/mkpkglists.py @@ -1,28 +1,16 @@ #!/usr/bin/python3 -import configparser import datetime import gzip -import mysql.connector import os -docroot = os.path.dirname(os.path.realpath(__file__)) + "/../web/html/" - -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") +import aurweb.db -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') +docroot = os.path.dirname(os.path.realpath(__file__)) + "/../web/html/" def main(): - db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) - cur = db.cursor() + conn = aurweb.db.Connection() datestr = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") pkglist_header = "# AUR package list, generated on " + datestr @@ -30,19 +18,19 @@ def main(): with gzip.open(docroot + "packages.gz", "w") as f: f.write(bytes(pkglist_header + "\n", "UTF-8")) - cur.execute("SELECT Packages.Name FROM Packages " + - "INNER JOIN PackageBases " + - "ON PackageBases.ID = Packages.PackageBaseID " + - "WHERE PackageBases.PackagerUID IS NOT NULL") + cur = conn.execute("SELECT Packages.Name FROM Packages " + + "INNER JOIN PackageBases " + + "ON PackageBases.ID = Packages.PackageBaseID " + + "WHERE PackageBases.PackagerUID IS NOT NULL") f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) with gzip.open(docroot + "pkgbase.gz", "w") as f: f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) - cur.execute("SELECT Name FROM PackageBases " + - "WHERE PackagerUID IS NOT NULL") + cur = conn.execute("SELECT Name FROM PackageBases " + + "WHERE PackagerUID IS NOT NULL") f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - db.close() + conn.close() if __name__ == '__main__': diff --git a/scripts/notify.py b/scripts/notify.py index cc99747..0f1a073 100755 --- a/scripts/notify.py +++ b/scripts/notify.py @@ -1,28 +1,19 @@ #!/usr/bin/python3 -import configparser import email.mime.text -import mysql.connector -import os import subprocess import sys import textwrap -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + '/../conf/config') +import aurweb.config +import aurweb.db -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') +aur_location = aurweb.config.get('options', 'aur_location') +aur_request_ml = aurweb.config.get('options', 'aur_request_ml') -aur_location = config.get('options', 'aur_location') -aur_request_ml = config.get('options', 'aur_request_ml') - -sendmail = config.get('notifications', 'sendmail') -sender = config.get('notifications', 'sender') -reply_to = config.get('notifications', 'reply-to') +sendmail = aurweb.config.get('notifications', 'sendmail') +sender = aurweb.config.get('notifications', 'sender') +reply_to = aurweb.config.get('notifications', 'reply-to') def headers_cc(cclist): @@ -60,120 +51,126 @@ def send_notification(to, subject, body, refs, headers={}): p.communicate(msg.as_bytes()) -def username_from_id(cur, uid): - cur.execute('SELECT UserName FROM Users WHERE ID = %s', [uid]) +def username_from_id(conn, uid): + cur = conn.execute('SELECT UserName FROM Users WHERE ID = ?', [uid]) return cur.fetchone()[0] -def pkgbase_from_id(cur, pkgbase_id): - cur.execute('SELECT Name FROM PackageBases WHERE ID = %s', [pkgbase_id]) +def pkgbase_from_id(conn, pkgbase_id): + cur = conn.execute('SELECT Name FROM PackageBases WHERE ID = ?', + [pkgbase_id]) return cur.fetchone()[0] -def pkgbase_from_pkgreq(cur, reqid): - cur.execute('SELECT PackageBaseID FROM PackageRequests WHERE ID = %s', - [reqid]) +def pkgbase_from_pkgreq(conn, reqid): + cur = conn.execute('SELECT PackageBaseID FROM PackageRequests ' + + 'WHERE ID = ?', [reqid]) return cur.fetchone()[0] -def get_user_email(cur, uid): - cur.execute('SELECT Email FROM Users WHERE ID = %s', [uid]) +def get_user_email(conn, uid): + cur = conn.execute('SELECT Email FROM Users WHERE ID = ?', [uid]) return cur.fetchone()[0] -def get_maintainer_email(cur, pkgbase_id): - cur.execute('SELECT Users.Email FROM Users ' + - 'INNER JOIN PackageBases ' + - 'ON PackageBases.MaintainerUID = Users.ID WHERE ' + - 'PackageBases.ID = %s', [pkgbase_id]) +def get_maintainer_email(conn, pkgbase_id): + cur = conn.execute('SELECT Users.Email FROM Users ' + + 'INNER JOIN PackageBases ' + + 'ON PackageBases.MaintainerUID = Users.ID WHERE ' + + 'PackageBases.ID = ?', [pkgbase_id]) return cur.fetchone()[0] -def get_recipients(cur, pkgbase_id, uid): - cur.execute('SELECT DISTINCT Users.Email FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'PackageNotifications.UserID != %s AND ' + - 'PackageNotifications.PackageBaseID = %s', [uid, pkgbase_id]) +def get_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) return [row[0] for row in cur.fetchall()] -def get_comment_recipients(cur, pkgbase_id, uid): - cur.execute('SELECT DISTINCT Users.Email FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'Users.CommentNotify = 1 AND ' + - 'PackageNotifications.UserID != %s AND ' + - 'PackageNotifications.PackageBaseID = %s', [uid, pkgbase_id]) +def get_comment_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'Users.CommentNotify = 1 AND ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) return [row[0] for row in cur.fetchall()] -def get_update_recipients(cur, pkgbase_id, uid): - cur.execute('SELECT DISTINCT Users.Email FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'Users.UpdateNotify = 1 AND ' + - 'PackageNotifications.UserID != %s AND ' + - 'PackageNotifications.PackageBaseID = %s', [uid, pkgbase_id]) +def get_update_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'Users.UpdateNotify = 1 AND ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) return [row[0] for row in cur.fetchall()] -def get_ownership_recipients(cur, pkgbase_id, uid): - cur.execute('SELECT DISTINCT Users.Email FROM Users ' + - 'INNER JOIN PackageNotifications ' + - 'ON PackageNotifications.UserID = Users.ID WHERE ' + - 'Users.OwnershipNotify = 1 AND ' + - 'PackageNotifications.UserID != %s AND ' + - 'PackageNotifications.PackageBaseID = %s', [uid, pkgbase_id]) +def get_ownership_recipients(conn, pkgbase_id, uid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM Users ' + + 'INNER JOIN PackageNotifications ' + + 'ON PackageNotifications.UserID = Users.ID WHERE ' + + 'Users.OwnershipNotify = 1 AND ' + + 'PackageNotifications.UserID != ? AND ' + + 'PackageNotifications.PackageBaseID = ?', + [uid, pkgbase_id]) return [row[0] for row in cur.fetchall()] -def get_request_recipients(cur, reqid): - cur.execute('SELECT DISTINCT Users.Email FROM PackageRequests ' + - 'INNER JOIN PackageBases ' + - 'ON PackageBases.ID = PackageRequests.PackageBaseID ' + - 'INNER JOIN Users ' + - 'ON Users.ID = PackageRequests.UsersID ' + - 'OR Users.ID = PackageBases.MaintainerUID ' + - 'WHERE PackageRequests.ID = %s', [reqid]) +def get_request_recipients(conn, reqid): + cur = conn.execute('SELECT DISTINCT Users.Email FROM PackageRequests ' + + 'INNER JOIN PackageBases ' + + 'ON PackageBases.ID = PackageRequests.PackageBaseID ' + + 'INNER JOIN Users ' + + 'ON Users.ID = PackageRequests.UsersID ' + + 'OR Users.ID = PackageBases.MaintainerUID ' + + 'WHERE PackageRequests.ID = ?', [reqid]) return [row[0] for row in cur.fetchall()] -def get_tu_vote_reminder_recipients(cur, vote_id): - cur.execute('SELECT ID FROM Users WHERE AccountTypeID = 2 ' + - 'EXCEPT SELECT Users.ID FROM Users ' + - 'INNER JOIN TU_Votes ' + - 'ON TU_Votes.UserID = Users.ID ' + - 'WHERE TU_Votes.VoteID = %s', [vote_id]) +def get_tu_vote_reminder_recipients(conn, vote_id): + cur = conn.execute('SELECT ID FROM Users WHERE AccountTypeID = 2 ' + + 'EXCEPT SELECT Users.ID FROM Users ' + + 'INNER JOIN TU_Votes ' + + 'ON TU_Votes.UserID = Users.ID ' + + 'WHERE TU_Votes.VoteID = ?', [vote_id]) return [row[0] for row in cur.fetchall()] -def get_comment(cur, comment_id): - cur.execute('SELECT Comments FROM PackageComments WHERE ID = %s', - [comment_id]) +def get_comment(conn, comment_id): + cur = conn.execute('SELECT Comments FROM PackageComments WHERE ID = ?', + [comment_id]) return cur.fetchone()[0] -def get_flagger_comment(cur, pkgbase_id): - cur.execute('SELECT FlaggerComment FROM PackageBases WHERE ID = %s', - [pkgbase_id]) +def get_flagger_comment(conn, pkgbase_id): + cur = conn.execute('SELECT FlaggerComment FROM PackageBases WHERE ID = ?', + [pkgbase_id]) return cur.fetchone()[0] -def get_request_comment(cur, reqid): - cur.execute('SELECT Comments FROM PackageRequests WHERE ID = %s', [reqid]) +def get_request_comment(conn, reqid): + cur = conn.execute('SELECT Comments FROM PackageRequests WHERE ID = ?', + [reqid]) return cur.fetchone()[0] -def get_request_closure_comment(cur, reqid): - cur.execute('SELECT ClosureComment FROM PackageRequests WHERE ID = %s', - [reqid]) +def get_request_closure_comment(conn, reqid): + cur = conn.execute('SELECT ClosureComment FROM PackageRequests ' + + 'WHERE ID = ?', [reqid]) return cur.fetchone()[0] -def send_resetkey(cur, uid): - cur.execute('SELECT UserName, Email, ResetKey FROM Users WHERE ID = %s', - [uid]) +def send_resetkey(conn, uid): + cur = conn.execute('SELECT UserName, Email, ResetKey FROM Users ' + + 'WHERE ID = ?', [uid]) username, to, resetkey = cur.fetchone() subject = 'AUR Password Reset' @@ -186,9 +183,9 @@ def send_resetkey(cur, uid): send_notification([to], subject, body, refs) -def welcome(cur, uid): - cur.execute('SELECT UserName, Email, ResetKey FROM Users WHERE ID = %s', - [uid]) +def welcome(conn, uid): + cur = conn.execute('SELECT UserName, Email, ResetKey FROM Users ' + + 'WHERE ID = ?', [uid]) username, to, resetkey = cur.fetchone() subject = 'Welcome to the Arch User Repository' @@ -201,11 +198,11 @@ def welcome(cur, uid): send_notification([to], subject, body, refs) -def comment(cur, uid, pkgbase_id, comment_id): - user = username_from_id(cur, uid) - pkgbase = pkgbase_from_id(cur, pkgbase_id) - to = get_comment_recipients(cur, pkgbase_id, uid) - text = get_comment(cur, comment_id) +def comment(conn, uid, pkgbase_id, comment_id): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_comment_recipients(conn, pkgbase_id, uid) + text = get_comment(conn, comment_id) user_uri = aur_location + '/account/' + user + '/' pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' @@ -224,10 +221,10 @@ def comment(cur, uid, pkgbase_id, comment_id): send_notification(to, subject, body, refs, headers) -def update(cur, uid, pkgbase_id): - user = username_from_id(cur, uid) - pkgbase = pkgbase_from_id(cur, pkgbase_id) - to = get_update_recipients(cur, pkgbase_id, uid) +def update(conn, uid, pkgbase_id): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_update_recipients(conn, pkgbase_id, uid) user_uri = aur_location + '/account/' + user + '/' pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' @@ -246,11 +243,11 @@ def update(cur, uid, pkgbase_id): send_notification(to, subject, body, refs, headers) -def flag(cur, uid, pkgbase_id): - user = username_from_id(cur, uid) - pkgbase = pkgbase_from_id(cur, pkgbase_id) - to = [get_maintainer_email(cur, pkgbase_id)] - text = get_flagger_comment(cur, pkgbase_id) +def flag(conn, uid, pkgbase_id): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [get_maintainer_email(conn, pkgbase_id)] + text = get_flagger_comment(conn, pkgbase_id) user_uri = aur_location + '/account/' + user + '/' pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' @@ -265,10 +262,10 @@ def flag(cur, uid, pkgbase_id): send_notification(to, subject, body, refs) -def adopt(cur, pkgbase_id, uid): - user = username_from_id(cur, uid) - pkgbase = pkgbase_from_id(cur, pkgbase_id) - to = get_ownership_recipients(cur, pkgbase_id, uid) +def adopt(conn, pkgbase_id, uid): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_ownership_recipients(conn, pkgbase_id, uid) user_uri = aur_location + '/account/' + user + '/' pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' @@ -281,10 +278,10 @@ def adopt(cur, pkgbase_id, uid): send_notification(to, subject, body, refs) -def disown(cur, pkgbase_id, uid): - user = username_from_id(cur, uid) - pkgbase = pkgbase_from_id(cur, pkgbase_id) - to = get_ownership_recipients(cur, pkgbase_id, uid) +def disown(conn, pkgbase_id, uid): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = get_ownership_recipients(conn, pkgbase_id, uid) user_uri = aur_location + '/account/' + user + '/' pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' @@ -297,9 +294,9 @@ def disown(cur, pkgbase_id, uid): send_notification(to, subject, body, refs) -def comaintainer_add(cur, pkgbase_id, uid): - pkgbase = pkgbase_from_id(cur, pkgbase_id) - to = [get_user_email(cur, uid)] +def comaintainer_add(conn, pkgbase_id, uid): + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [get_user_email(conn, uid)] pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' @@ -310,9 +307,9 @@ def comaintainer_add(cur, pkgbase_id, uid): send_notification(to, subject, body, refs) -def comaintainer_remove(cur, pkgbase_id, uid): - pkgbase = pkgbase_from_id(cur, pkgbase_id) - to = [get_user_email(cur, uid)] +def comaintainer_remove(conn, pkgbase_id, uid): + pkgbase = pkgbase_from_id(conn, pkgbase_id) + to = [get_user_email(conn, uid)] pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' @@ -324,12 +321,12 @@ def comaintainer_remove(cur, pkgbase_id, uid): send_notification(to, subject, body, refs) -def delete(cur, uid, old_pkgbase_id, new_pkgbase_id=None): - user = username_from_id(cur, uid) - old_pkgbase = pkgbase_from_id(cur, old_pkgbase_id) +def delete(conn, uid, old_pkgbase_id, new_pkgbase_id=None): + user = username_from_id(conn, uid) + old_pkgbase = pkgbase_from_id(conn, old_pkgbase_id) if new_pkgbase_id: - new_pkgbase = pkgbase_from_id(cur, new_pkgbase_id) - to = get_recipients(cur, old_pkgbase_id, uid) + new_pkgbase = pkgbase_from_id(conn, new_pkgbase_id) + to = get_recipients(conn, old_pkgbase_id, uid) user_uri = aur_location + '/account/' + user + '/' pkgbase_uri = aur_location + '/pkgbase/' + old_pkgbase + '/' @@ -354,12 +351,12 @@ def delete(cur, uid, old_pkgbase_id, new_pkgbase_id=None): send_notification(to, subject, body, refs) -def request_open(cur, uid, reqid, reqtype, pkgbase_id, merge_into=None): - user = username_from_id(cur, uid) - pkgbase = pkgbase_from_id(cur, pkgbase_id) +def request_open(conn, uid, reqid, reqtype, pkgbase_id, merge_into=None): + user = username_from_id(conn, uid) + pkgbase = pkgbase_from_id(conn, pkgbase_id) to = [aur_request_ml] - cc = get_request_recipients(cur, reqid) - text = get_request_comment(cur, reqid) + cc = get_request_recipients(conn, reqid) + text = get_request_comment(conn, reqid) user_uri = aur_location + '/account/' + user + '/' pkgbase_uri = aur_location + '/pkgbase/' + pkgbase + '/' @@ -388,14 +385,14 @@ def request_open(cur, uid, reqid, reqtype, pkgbase_id, merge_into=None): send_notification(to, subject, body, refs, headers) -def request_close(cur, uid, reqid, reason): +def request_close(conn, uid, reqid, reason): to = [aur_request_ml] - cc = get_request_recipients(cur, reqid) - text = get_request_closure_comment(cur, reqid) + cc = get_request_recipients(conn, reqid) + text = get_request_closure_comment(conn, reqid) subject = '[PRQ#%d] Request %s' % (int(reqid), reason.title()) if int(uid): - user = username_from_id(cur, uid) + user = username_from_id(conn, uid) user_uri = aur_location + '/account/' + user + '/' body = 'Request #%d has been %s by %s [1]' % (int(reqid), reason, user) refs = '[1] ' + user_uri @@ -414,8 +411,8 @@ def request_close(cur, uid, reqid, reason): send_notification(to, subject, body, refs, headers) -def tu_vote_reminder(cur, vote_id): - to = get_tu_vote_reminder_recipients(cur, vote_id) +def tu_vote_reminder(conn, vote_id): + to = get_tu_vote_reminder_recipients(conn, vote_id) vote_uri = aur_location + '/tu/?id=' + vote_id @@ -445,15 +442,12 @@ def main(): 'tu-vote-reminder': tu_vote_reminder, } - db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) - cur = db.cursor() + conn = aurweb.db.Connection() - action_map[action](cur, *sys.argv[2:]) + action_map[action](conn, *sys.argv[2:]) - db.commit() - db.close() + conn.commit() + conn.close() if __name__ == '__main__': diff --git a/scripts/pkgmaint.py b/scripts/pkgmaint.py index 346b046..d0f8fe7 100755 --- a/scripts/pkgmaint.py +++ b/scripts/pkgmaint.py @@ -1,31 +1,17 @@ #!/usr/bin/python3 -import configparser -import mysql.connector -import os - -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") - -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') +import aurweb.db def main(): - db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) - cur = db.cursor() + conn = aurweb.db.Connection() - cur.execute("DELETE FROM PackageBases WHERE " + - "UNIX_TIMESTAMP() - SubmittedTS > 86400 " + - "AND PackagerUID IS NULL") + conn.execute("DELETE FROM PackageBases WHERE " + + "UNIX_TIMESTAMP() - SubmittedTS > 86400 " + + "AND PackagerUID IS NULL") - db.commit() - db.close() + conn.commit() + conn.close() if __name__ == '__main__': diff --git a/scripts/popupdate.py b/scripts/popupdate.py index 26d8379..f5e09d9 100755 --- a/scripts/popupdate.py +++ b/scripts/popupdate.py @@ -1,36 +1,22 @@ #!/usr/bin/python3 -import configparser -import mysql.connector -import os +import aurweb.db -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') +def main(): + conn = aurweb.db.Connection() + conn.execute("UPDATE PackageBases SET NumVotes = (" + + "SELECT COUNT(*) FROM PackageVotes " + + "WHERE PackageVotes.PackageBaseID = PackageBases.ID)") -def main(): - db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) - cur = db.cursor() - - cur.execute("UPDATE PackageBases SET NumVotes = (" + - "SELECT COUNT(*) FROM PackageVotes " + - "WHERE PackageVotes.PackageBaseID = PackageBases.ID)") - - cur.execute("UPDATE PackageBases SET Popularity = (" + - "SELECT COALESCE(SUM(POWER(0.98, (UNIX_TIMESTAMP() - VoteTS) / 86400)), 0.0) " + - "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + - "PackageBases.ID AND NOT VoteTS IS NULL)") - - db.commit() - db.close() + conn.execute("UPDATE PackageBases SET Popularity = (" + + "SELECT COALESCE(SUM(POWER(0.98, (UNIX_TIMESTAMP() - VoteTS) / 86400)), 0.0) " + + "FROM PackageVotes WHERE PackageVotes.PackageBaseID = " + + "PackageBases.ID AND NOT VoteTS IS NULL)") + + conn.commit() + conn.close() if __name__ == '__main__': diff --git a/scripts/tuvotereminder.py b/scripts/tuvotereminder.py index dc16397..a053608 100755 --- a/scripts/tuvotereminder.py +++ b/scripts/tuvotereminder.py @@ -1,34 +1,24 @@ #!/usr/bin/python3 -import configparser -import mysql.connector -import os import subprocess import time -config = configparser.RawConfigParser() -config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") +import aurweb.config +import aurweb.db -aur_db_host = config.get('database', 'host') -aur_db_name = config.get('database', 'name') -aur_db_user = config.get('database', 'user') -aur_db_pass = config.get('database', 'password') -aur_db_socket = config.get('database', 'socket') -notify_cmd = config.get('notifications', 'notify-cmd') +notify_cmd = aurweb.config.get('notifications', 'notify-cmd') def main(): - db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, - passwd=aur_db_pass, db=aur_db_name, - unix_socket=aur_db_socket, buffered=True) - cur = db.cursor() + conn = aurweb.db.Connection() now = int(time.time()) filter_from = now + 500 filter_to = now + 172800 - cur.execute("SELECT ID FROM TU_VoteInfo WHERE End >= %s AND End <= %s", - [filter_from, filter_to]) + cur = conn.execute("SELECT ID FROM TU_VoteInfo " + + "WHERE End >= ? AND End <= ?", + [filter_from, filter_to]) for vote_id in [row[0] for row in cur.fetchall()]: subprocess.Popen((notify_cmd, 'tu-vote-reminder', str(vote_id))) -- 2.10.0

Move and rename the existing git-interface tests such that tests for other scripts can be added easily. In particular, the following changes are made: * Move the existing tests from git-interface/test/ to test/. * Rename t0001-auth.sh to t1100-git-auth.sh. * Rename t0002-serve.sh to t1200-git-serve.sh. * Rename t0003-update.sh to t1300-git-update.sh. Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org> --- {git-interface/test => test}/Makefile | 0 {git-interface/test => test}/setup.sh | 2 +- {git-interface/test => test}/sharness.sh | 0 git-interface/test/t0001-auth.sh => test/t1100-git-auth.sh | 0 git-interface/test/t0002-serve.sh => test/t1200-git-serve.sh | 0 git-interface/test/t0003-update.sh => test/t1300-git-update.sh | 0 6 files changed, 1 insertion(+), 1 deletion(-) rename {git-interface/test => test}/Makefile (100%) rename {git-interface/test => test}/setup.sh (99%) rename {git-interface/test => test}/sharness.sh (100%) rename git-interface/test/t0001-auth.sh => test/t1100-git-auth.sh (100%) rename git-interface/test/t0002-serve.sh => test/t1200-git-serve.sh (100%) rename git-interface/test/t0003-update.sh => test/t1300-git-update.sh (100%) diff --git a/git-interface/test/Makefile b/test/Makefile similarity index 100% rename from git-interface/test/Makefile rename to test/Makefile diff --git a/git-interface/test/setup.sh b/test/setup.sh similarity index 99% rename from git-interface/test/setup.sh rename to test/setup.sh index 0995f05..232b14d 100644 --- a/git-interface/test/setup.sh +++ b/test/setup.sh @@ -1,5 +1,5 @@ TEST_DIRECTORY="$(pwd)" -TOPLEVEL="$(cd ../.. && pwd)" +TOPLEVEL="$(cd .. && pwd)" . ./sharness.sh diff --git a/git-interface/test/sharness.sh b/test/sharness.sh similarity index 100% rename from git-interface/test/sharness.sh rename to test/sharness.sh diff --git a/git-interface/test/t0001-auth.sh b/test/t1100-git-auth.sh similarity index 100% rename from git-interface/test/t0001-auth.sh rename to test/t1100-git-auth.sh diff --git a/git-interface/test/t0002-serve.sh b/test/t1200-git-serve.sh similarity index 100% rename from git-interface/test/t0002-serve.sh rename to test/t1200-git-serve.sh diff --git a/git-interface/test/t0003-update.sh b/test/t1300-git-update.sh similarity index 100% rename from git-interface/test/t0003-update.sh rename to test/t1300-git-update.sh -- 2.10.0

Instead of writing the output to hardcoded files (relative to the document root), make the output paths of mkpkglists configurable. Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org> --- conf/config.proto | 4 ++++ scripts/mkpkglists.py | 9 +++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/conf/config.proto b/conf/config.proto index c56141c..e545977 100644 --- a/conf/config.proto +++ b/conf/config.proto @@ -63,3 +63,7 @@ max-blob-size = 256000 db-path = /srv/http/aurweb/aurblup/ sync-dbs = core extra community multilib testing community-testing servers = ftp://mirrors.kernel.org/archlinux/%s/os/x86_64 + +[mkpkglists] +packagesfile = /srv/http/aurweb/web/html/packages.gz +pkgbasefile = /srv/http/aurweb/web/html/pkgbase.gz diff --git a/scripts/mkpkglists.py b/scripts/mkpkglists.py index 70cbd13..8a0f2e9 100755 --- a/scripts/mkpkglists.py +++ b/scripts/mkpkglists.py @@ -2,11 +2,12 @@ import datetime import gzip -import os +import aurweb.config import aurweb.db -docroot = os.path.dirname(os.path.realpath(__file__)) + "/../web/html/" +packagesfile = aurweb.config.get('mkpkglists', 'packagesfile') +pkgbasefile = aurweb.config.get('mkpkglists', 'pkgbasefile') def main(): @@ -16,7 +17,7 @@ def main(): pkglist_header = "# AUR package list, generated on " + datestr pkgbaselist_header = "# AUR package base list, generated on " + datestr - with gzip.open(docroot + "packages.gz", "w") as f: + with gzip.open(packagesfile, "w") as f: f.write(bytes(pkglist_header + "\n", "UTF-8")) cur = conn.execute("SELECT Packages.Name FROM Packages " + "INNER JOIN PackageBases " + @@ -24,7 +25,7 @@ def main(): "WHERE PackageBases.PackagerUID IS NOT NULL") f.writelines([bytes(x[0] + "\n", "UTF-8") for x in cur.fetchall()]) - with gzip.open(docroot + "pkgbase.gz", "w") as f: + with gzip.open(pkgbasefile, "w") as f: f.write(bytes(pkgbaselist_header + "\n", "UTF-8")) cur = conn.execute("SELECT Name FROM PackageBases " + "WHERE PackagerUID IS NOT NULL") -- 2.10.0

Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org> --- test/setup.sh | 4 +++- test/t2200-tuvotereminder.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100755 test/t2200-tuvotereminder.sh diff --git a/test/setup.sh b/test/setup.sh index 85faba0..e6a9a16 100644 --- a/test/setup.sh +++ b/test/setup.sh @@ -12,6 +12,7 @@ GIT_AUTH="$TOPLEVEL/git-interface/git-auth.py" GIT_SERVE="$TOPLEVEL/git-interface/git-serve.py" GIT_UPDATE="$TOPLEVEL/git-interface/git-update.py" MKPKGLISTS="$TOPLEVEL/scripts/mkpkglists.py" +TUVOTEREMINDER="$TOPLEVEL/scripts/tuvotereminder.py" # Create the configuration file and a dummy notification script. cat >config <<-EOF @@ -47,8 +48,9 @@ packagesfile = packages.gz pkgbasefile = pkgbase.gz EOF -cat >notify.sh <<-EOF +cat >notify.sh <<-\EOF #!/bin/sh +echo $* >>notify.out EOF chmod +x notify.sh diff --git a/test/t2200-tuvotereminder.sh b/test/t2200-tuvotereminder.sh new file mode 100755 index 0000000..438c0c3 --- /dev/null +++ b/test/t2200-tuvotereminder.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +test_description='tuvotereminder tests' + +. ./setup.sh + +test_expect_success 'Test Trusted User vote reminders.' ' + now=$(date -d now +%s) && + tomorrow=$(date -d tomorrow +%s) && + threedays=$(date -d "3 days" +%s) && + cat <<-EOD | sqlite3 aur.db && + INSERT INTO TU_VoteInfo (ID, Agenda, User, Submitted, End, Quorum, SubmitterID) VALUES (1, "Lorem ipsum.", "user", 0, $now, 0.00, 2); + INSERT INTO TU_VoteInfo (ID, Agenda, User, Submitted, End, Quorum, SubmitterID) VALUES (2, "Lorem ipsum.", "user", 0, $tomorrow, 0.00, 2); + INSERT INTO TU_VoteInfo (ID, Agenda, User, Submitted, End, Quorum, SubmitterID) VALUES (3, "Lorem ipsum.", "user", 0, $tomorrow, 0.00, 2); + INSERT INTO TU_VoteInfo (ID, Agenda, User, Submitted, End, Quorum, SubmitterID) VALUES (4, "Lorem ipsum.", "user", 0, $threedays, 0.00, 2); + EOD + >notify.out && + "$TUVOTEREMINDER" && + cat <<-EOD >expected && + tu-vote-reminder 2 + tu-vote-reminder 3 + EOD + test_cmp notify.out expected +' + +test_done -- 2.10.0

Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org> --- test/setup.sh | 1 + test/t2300-pkgmaint.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100755 test/t2300-pkgmaint.sh diff --git a/test/setup.sh b/test/setup.sh index e6a9a16..622aab0 100644 --- a/test/setup.sh +++ b/test/setup.sh @@ -13,6 +13,7 @@ GIT_SERVE="$TOPLEVEL/git-interface/git-serve.py" GIT_UPDATE="$TOPLEVEL/git-interface/git-update.py" MKPKGLISTS="$TOPLEVEL/scripts/mkpkglists.py" TUVOTEREMINDER="$TOPLEVEL/scripts/tuvotereminder.py" +PKGMAINT="$TOPLEVEL/scripts/pkgmaint.py" # Create the configuration file and a dummy notification script. cat >config <<-EOF diff --git a/test/t2300-pkgmaint.sh b/test/t2300-pkgmaint.sh new file mode 100755 index 0000000..5c55aaf --- /dev/null +++ b/test/t2300-pkgmaint.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +test_description='pkgmaint tests' + +. ./setup.sh + +test_expect_success 'Test package base cleanup script.' ' + now=$(date -d now +%s) && + threedaysago=$(date -d "3 days ago" +%s) && + cat <<-EOD | sqlite3 aur.db && + INSERT INTO PackageBases (ID, Name, PackagerUID, SubmittedTS, ModifiedTS) VALUES (1, "foobar", 1, $now, 0); + INSERT INTO PackageBases (ID, Name, PackagerUID, SubmittedTS, ModifiedTS) VALUES (2, "foobar2", 2, $threedaysago, 0); + INSERT INTO PackageBases (ID, Name, PackagerUID, SubmittedTS, ModifiedTS) VALUES (3, "foobar3", NULL, $now, 0); + INSERT INTO PackageBases (ID, Name, PackagerUID, SubmittedTS, ModifiedTS) VALUES (4, "foobar4", NULL, $threedaysago, 0); + EOD + "$PKGMAINT" && + cat <<-EOD >expected && + foobar + foobar2 + foobar3 + EOD + echo "SELECT Name FROM PackageBases;" | sqlite3 aur.db >actual && + test_cmp actual expected +' + +test_done -- 2.10.0

Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org> --- test/setup.sh | 6 ++++++ test/t2400-aurblup.sh | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100755 test/t2400-aurblup.sh diff --git a/test/setup.sh b/test/setup.sh index 622aab0..e379ada 100644 --- a/test/setup.sh +++ b/test/setup.sh @@ -14,6 +14,7 @@ GIT_UPDATE="$TOPLEVEL/git-interface/git-update.py" MKPKGLISTS="$TOPLEVEL/scripts/mkpkglists.py" TUVOTEREMINDER="$TOPLEVEL/scripts/tuvotereminder.py" PKGMAINT="$TOPLEVEL/scripts/pkgmaint.py" +AURBLUP="$TOPLEVEL/scripts/aurblup.py" # Create the configuration file and a dummy notification script. cat >config <<-EOF @@ -44,6 +45,11 @@ ssh-cmdline = ssh aur@aur.archlinux.org [update] max-blob-size = 256000 +[aurblup] +db-path = $(pwd)/sync/ +sync-dbs = test +server = file://$(pwd)/remote/ + [mkpkglists] packagesfile = packages.gz pkgbasefile = pkgbase.gz diff --git a/test/t2400-aurblup.sh b/test/t2400-aurblup.sh new file mode 100755 index 0000000..708281c --- /dev/null +++ b/test/t2400-aurblup.sh @@ -0,0 +1,53 @@ +#!/bin/sh + +test_description='aurblup tests' + +. ./setup.sh + +test_expect_success 'Test official provider update script.' ' + mkdir -p remote/test/foobar-1.0-1 && + cat <<-EOD >remote/test/foobar-1.0-1/desc && + %FILENAME% + foobar-1.0-any.pkg.tar.xz + + %NAME% + foobar + + %VERSION% + 1.0-1 + + %ARCH% + any + EOD + mkdir -p remote/test/foobar2-1.0-1 && + cat <<-EOD >remote/test/foobar2-1.0-1/desc && + %FILENAME% + foobar2-1.0-any.pkg.tar.xz + + %NAME% + foobar2 + + %VERSION% + 1.0-1 + + %ARCH% + any + + %PROVIDES% + foobar3 + foobar4 + EOD + ( cd remote/test && bsdtar -czf ../test.db * ) && + mkdir sync && + "$AURBLUP" && + cat <<-EOD >expected && + foobar|test|foobar + foobar2|test|foobar2 + foobar2|test|foobar3 + foobar2|test|foobar4 + EOD + echo "SELECT Name, Repo, Provides FROM OfficialProviders ORDER BY Provides;" | sqlite3 aur.db >actual && + test_cmp actual expected +' + +test_done -- 2.10.0