From: Johannes Löthberg <johannes@kyriasis.com> python-srcinfo is a more transparent and simpler library for parsing SRCINFO files. Signed-off-by: Johannes Löthberg <johannes@kyriasis.com> Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org> --- INSTALL | 2 +- git-interface/aurinfo.py | 208 -------------------------------------------- git-interface/git-update.py | 51 ++++++----- 3 files changed, 26 insertions(+), 235 deletions(-) delete mode 100644 git-interface/aurinfo.py diff --git a/INSTALL b/INSTALL index d5664d2..be39052 100644 --- a/INSTALL +++ b/INSTALL @@ -53,7 +53,7 @@ Setup on Arch Linux 7) Install needed Python modules: - # pacman -S python-mysql-connector python-pygit2 + # pacman -S python-mysql-connector python-pygit2 python-srcinfo 8) Install the git-auth wrapper script: diff --git a/git-interface/aurinfo.py b/git-interface/aurinfo.py deleted file mode 100644 index b286316..0000000 --- a/git-interface/aurinfo.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python - -from copy import copy, deepcopy -import pprint -import sys - -class Attr(object): - def __init__(self, name, is_multivalued=False, allow_arch_extensions=False): - self.name = name - self.is_multivalued = is_multivalued - self.allow_arch_extensions = allow_arch_extensions - -PKGBUILD_ATTRIBUTES = { - 'arch': Attr('arch', True), - 'backup': Attr('backup', True), - 'changelog': Attr('changelog', False), - 'checkdepends': Attr('checkdepends', True), - 'conflicts': Attr('conflicts', True, True), - 'depends': Attr('depends', True, True), - 'epoch': Attr('epoch', False), - 'groups': Attr('groups', True), - 'install': Attr('install', False), - 'license': Attr('license', True), - 'makedepends': Attr('makedepends', True, True), - 'md5sums': Attr('md5sums', True, True), - 'noextract': Attr('noextract', True), - 'optdepends': Attr('optdepends', True, True), - 'options': Attr('options', True), - 'pkgname': Attr('pkgname', False), - 'pkgrel': Attr('pkgrel', False), - 'pkgver': Attr('pkgver', False), - 'provides': Attr('provides', True, True), - 'replaces': Attr('replaces', True, True), - 'sha1sums': Attr('sha1sums', True, True), - 'sha224sums': Attr('sha224sums', True, True), - 'sha256sums': Attr('sha256sums', True, True), - 'sha384sums': Attr('sha384sums', True, True), - 'sha512sums': Attr('sha512sums', True, True), - 'source': Attr('source', True, True), - 'url': Attr('url', False), - 'validpgpkeys': Attr('validpgpkeys', True), -} - -def find_attr(attrname): - # exact match - attr = PKGBUILD_ATTRIBUTES.get(attrname, None) - if attr: - return attr - - # prefix match - # XXX: this could break in the future if PKGBUILD(5) ever - # introduces a key which is a subset of another. - for k in PKGBUILD_ATTRIBUTES.keys(): - if attrname.startswith(k + '_'): - return PKGBUILD_ATTRIBUTES[k] - -def IsMultiValued(attrname): - attr = find_attr(attrname) - return attr and attr.is_multivalued - -class AurInfo(object): - def __init__(self): - self._pkgbase = {} - self._packages = {} - - def GetPackageNames(self): - return self._packages.keys() - - def GetMergedPackage(self, pkgname): - package = deepcopy(self._pkgbase) - package['pkgname'] = pkgname - for k, v in self._packages.get(pkgname).items(): - package[k] = deepcopy(v) - return package - - def AddPackage(self, pkgname): - self._packages[pkgname] = {} - return self._packages[pkgname] - - def SetPkgbase(self, pkgbasename): - self._pkgbase = {'pkgname' : pkgbasename} - return self._pkgbase - - -class StderrECatcher(object): - def Catch(self, lineno, error): - print('ERROR[{:d}]: {:s}'.format(lineno, error), file=sys.stderr) - - -class CollectionECatcher(object): - def __init__(self): - self._errors = [] - - def Catch(self, lineno, error): - self._errors.append((lineno, error)) - - def HasErrors(self): - return len(self._errors) > 0 - - def Errors(self): - return copy(self._errors) - - -def ParseAurinfoFromIterable(iterable, ecatcher=None): - aurinfo = AurInfo() - - if ecatcher is None: - ecatcher = StderrECatcher() - - current_package = None - lineno = 0 - - for line in iterable: - lineno += 1 - - if line.startswith('#'): - continue - - if not line.strip(): - # end of package - current_package = None - continue - - if not (line.startswith('\t') or line.startswith(' ')): - # start of new package - try: - key, value = map(str.strip, line.split('=', 1)) - except ValueError: - ecatcher.Catch(lineno, 'unexpected header format in section={:s}'.format( - current_package['pkgname'])) - continue - - if key == 'pkgbase': - current_package = aurinfo.SetPkgbase(value) - elif key == 'pkgname': - current_package = aurinfo.AddPackage(value) - else: - ecatcher.Catch(lineno, 'unexpected new section not starting ' - 'with \'pkgname\' found') - continue - else: - # package attribute - if current_package is None: - ecatcher.Catch(lineno, 'package attribute found outside of ' - 'a package section') - continue - - try: - key, value = map(str.strip, line.split('=', 1)) - except ValueError: - ecatcher.Catch(lineno, 'unexpected attribute format in ' - 'section={:s}'.format(current_package['pkgname'])) - - if IsMultiValued(key): - if not current_package.get(key): - current_package[key] = [] - if value: - current_package[key].append(value) - else: - if not current_package.get(key): - current_package[key] = value - else: - ecatcher.Catch(lineno, 'overwriting attribute ' - '{:s}: {:s} -> {:s}'.format(key, - current_package[key], value)) - - return aurinfo - - -def ParseAurinfo(filename='.AURINFO', ecatcher=None): - with open(filename) as f: - return ParseAurinfoFromIterable(f, ecatcher) - - -def ValidateAurinfo(filename='.AURINFO'): - ecatcher = CollectionECatcher() - ParseAurinfo(filename, ecatcher) - errors = ecatcher.Errors() - for error in errors: - print('error on line {:d}: {:s}'.format(error), file=sys.stderr) - return not errors - - -if __name__ == '__main__': - pp = pprint.PrettyPrinter(indent=4) - - if len(sys.argv) == 1: - print('error: not enough arguments') - sys.exit(1) - elif len(sys.argv) == 2: - action = sys.argv[1] - filename = '.AURINFO' - else: - action, filename = sys.argv[1:3] - - if action == 'parse': - aurinfo = ParseAurinfo(filename) - for pkgname in aurinfo.GetPackageNames(): - print(">>> merged package: {:s}".format(pkgname)) - pp.pprint(aurinfo.GetMergedPackage(pkgname)) - print() - elif action == 'validate': - sys.exit(not ValidateAurinfo(filename)) - else: - print('unknown action: {:s}'.format(action)) - sys.exit(1) - -# vim: set et ts=4 sw=4: diff --git a/git-interface/git-update.py b/git-interface/git-update.py index 5efc9b1..a4a8d8d 100755 --- a/git-interface/git-update.py +++ b/git-interface/git-update.py @@ -7,7 +7,8 @@ import pygit2 import re import sys -import aurinfo +import srcinfo.parse +import srcinfo.utils config = configparser.RawConfigParser() config.read(os.path.dirname(os.path.realpath(__file__)) + "/../conf/config") @@ -48,9 +49,9 @@ def parse_dep(depstring): return (depname, depcond) -def save_srcinfo(srcinfo, db, cur, user): +def save_metadata(metadata, db, cur, user): # Obtain package base ID and previous maintainer. - pkgbase = srcinfo._pkgbase['pkgname'] + pkgbase = metadata['pkgbase'] cur.execute("SELECT ID, MaintainerUID FROM PackageBases " "WHERE Name = %s", [pkgbase]) (pkgbase_id, maintainer_uid) = cur.fetchone() @@ -70,8 +71,8 @@ def save_srcinfo(srcinfo, db, cur, user): cur.execute("DELETE FROM Packages WHERE PackageBaseID = %s", [pkgbase_id]) - for pkgname in srcinfo.GetPackageNames(): - pkginfo = srcinfo.GetMergedPackage(pkgname) + for pkgname in srcinfo.utils.get_package_names(metadata): + pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) if 'epoch' in pkginfo and int(pkginfo['epoch']) > 0: ver = '{:d}:{:s}-{:s}'.format(int(pkginfo['epoch']), @@ -245,26 +246,24 @@ for commit in walker: if blob.size > 250000: die_commit("maximum blob size (250kB) exceeded", str(commit.id)) - srcinfo_raw = repo[commit.tree['.SRCINFO'].id].data.decode() - srcinfo_raw = srcinfo_raw.split('\n') - ecatcher = aurinfo.CollectionECatcher() - srcinfo = aurinfo.ParseAurinfoFromIterable(srcinfo_raw, ecatcher) - errors = ecatcher.Errors() + metadata_raw = repo[commit.tree['.SRCINFO'].id].data.decode() + (metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) if errors: sys.stderr.write("error: The following errors occurred " "when parsing .SRCINFO in commit\n") sys.stderr.write("error: {:s}:\n".format(str(commit.id))) for error in errors: - sys.stderr.write("error: line {:d}: {:s}\n".format(*error)) + for err in error['error']: + sys.stderr.write("error: line {:d}: {:s}\n".format(error['line'], err)) exit(1) - srcinfo_pkgbase = srcinfo._pkgbase['pkgname'] - if not re.match(repo_regex, srcinfo_pkgbase): - die_commit('invalid pkgbase: {:s}'.format(srcinfo_pkgbase), + metadata_pkgbase = metadata['pkgbase'] + if not re.match(repo_regex, metadata_pkgbase): + die_commit('invalid pkgbase: {:s}'.format(metadata_pkgbase), str(commit.id)) - for pkgname in srcinfo.GetPackageNames(): - pkginfo = srcinfo.GetMergedPackage(pkgname) + for pkgname in set(metadata['packages'].keys()): + pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) for field in ('pkgver', 'pkgrel', 'pkgname'): if field not in pkginfo: @@ -306,28 +305,28 @@ if sha1_old not in ("0000000000000000000000000000000000000000", sha1_new): warn(".SRCINFO unchanged. The package database will not be updated!") # Read .SRCINFO from the HEAD commit. -srcinfo_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode() -srcinfo_raw = srcinfo_raw.split('\n') -srcinfo = aurinfo.ParseAurinfoFromIterable(srcinfo_raw) +metadata_raw = repo[repo[sha1_new].tree['.SRCINFO'].id].data.decode() +(metadata, errors) = srcinfo.parse.parse_srcinfo(metadata_raw) # Ensure that the package base name matches the repository name. -srcinfo_pkgbase = srcinfo._pkgbase['pkgname'] -if srcinfo_pkgbase != pkgbase: - die('invalid pkgbase: {:s}, expected {:s}'.format(srcinfo_pkgbase, pkgbase)) +metadata_pkgbase = metadata['pkgbase'] +if metadata_pkgbase != pkgbase: + die('invalid pkgbase: {:s}, expected {:s}'.format(metadata_pkgbase, pkgbase)) # Ensure that packages are neither blacklisted nor overwritten. +pkgbase = metadata['pkgbase'] cur.execute("SELECT ID FROM PackageBases WHERE Name = %s", [pkgbase]) pkgbase_id = cur.fetchone()[0] if cur.rowcount == 1 else 0 cur.execute("SELECT Name FROM PackageBlacklist") blacklist = [row[0] for row in cur.fetchall()] -for pkgname in srcinfo.GetPackageNames(): - pkginfo = srcinfo.GetMergedPackage(pkgname) +for pkgname in srcinfo.utils.get_package_names(metadata): + pkginfo = srcinfo.utils.get_merged_package(pkgname, metadata) pkgname = pkginfo['pkgname'] if pkgname in blacklist and not privileged: - die('package is blacklisted: {:s}'.format(pkginfo['pkgname'])) + die('package is blacklisted: {:s}'.format(pkgname)) cur.execute("SELECT COUNT(*) FROM Packages WHERE Name = %s AND " + "PackageBaseID <> %s", [pkgname, pkgbase_id]) @@ -335,7 +334,7 @@ for pkgname in srcinfo.GetPackageNames(): die('cannot overwrite package: {:s}'.format(pkgname)) # Store package base details in the database. -save_srcinfo(srcinfo, db, cur, user) +save_metadata(metadata, db, cur, user) db.close() -- 2.7.0