[aur-dev] [PATCH/RFC 3/5] Add update hook template
Lukas Fleischer
archlinux at cryptocrack.de
Tue Jun 17 14:22:51 EDT 2014
This adds a script that can be used as an update hook to check all
commits for validity and to regenerate the package details page before
updating a named ref.
Signed-off-by: Lukas Fleischer <archlinux at cryptocrack.de>
---
scripts/git-integration/config.sample | 1 +
scripts/git-integration/git-serve.py | 2 +
scripts/git-integration/git-update.py | 334 ++++++++++++++++++++++++++++++++++
3 files changed, 337 insertions(+)
create mode 100755 scripts/git-integration/git-update.py
diff --git a/scripts/git-integration/config.sample b/scripts/git-integration/config.sample
index 7e53abd..39f68e4 100644
--- a/scripts/git-integration/config.sample
+++ b/scripts/git-integration/config.sample
@@ -13,4 +13,5 @@ ssh-options = no-port-forwarding,no-X11-forwarding,no-pty
[serve]
repo-base = /pub/git/
repo-regex = [a-z0-9][a-z0-9.+_-]*$
+git-update-hook = /srv/http/aur/scripts/git-integration/git-update.py
git-shell-cmd = /usr/bin/git-shell
diff --git a/scripts/git-integration/git-serve.py b/scripts/git-integration/git-serve.py
index 345c7ce..84d8dc6 100755
--- a/scripts/git-integration/git-serve.py
+++ b/scripts/git-integration/git-serve.py
@@ -18,6 +18,7 @@ aur_db_pass = config.get('database', 'password')
repo_base_path = config.get('serve', 'repo-base')
repo_regex = config.get('serve', 'repo-regex')
+git_update_hook = config.get('serve', 'git-update-hook')
git_shell_cmd = config.get('serve', 'git-shell-cmd')
def repo_path_validate(path):
@@ -60,6 +61,7 @@ def setup_repo(repo, user):
repo_path = repo_base_path + '/' + repo + '.git/'
pygit2.init_repository(repo_path, True)
+ os.symlink(git_update_hook, repo_path + 'hooks/update')
def check_permissions(pkgbase, user):
db = mysql.connector.connect(host=aur_db_host, user=aur_db_user,
diff --git a/scripts/git-integration/git-update.py b/scripts/git-integration/git-update.py
new file mode 100755
index 0000000..64606b8
--- /dev/null
+++ b/scripts/git-integration/git-update.py
@@ -0,0 +1,334 @@
+#!/usr/bin/python3
+
+from copy import copy, deepcopy
+import configparser
+import mysql.connector
+import os
+import pygit2
+import re
+import sys
+
+config = configparser.RawConfigParser()
+config.read(os.path.dirname(os.path.realpath(__file__)) + "/config")
+
+aur_db_host = config.get('database', 'host')
+aur_db_name = config.get('database', 'name')
+aur_db_user = config.get('database', 'user')
+aur_db_pass = config.get('database', 'password')
+
+MULTIVALUED_ATTRS = set([
+ 'arch',
+ 'groups',
+ 'makedepends',
+ 'checkdepends',
+ 'optdepends',
+ 'depends',
+ 'provides',
+ 'conflicts',
+ 'replaces',
+ 'options',
+ 'license',
+ 'source',
+ 'noextract',
+ 'backup',
+])
+
+def IsMultiValued(attr):
+ return attr in MULTIVALUED_ATTRS
+
+class AurInfo(object):
+ def __init__(self):
+ self._pkgbase = {}
+ self._packages = {}
+
+ def GetPackageNames(self):
+ return self._packages.keys()
+
+ def GetMergedPackage(self, pkgname):
+ package = deepcopy(self._pkgbase)
+ package['pkgname'] = pkgname
+ for k, v in self._packages.get(pkgname).items():
+ package[k] = deepcopy(v)
+ return package
+
+ def AddPackage(self, pkgname):
+ self._packages[pkgname] = {}
+ return self._packages[pkgname]
+
+ def GetPkgbase(self):
+ if 'pkgname' in self._pkgbase:
+ return self._pkgbase['pkgname']
+ else:
+ return None
+
+ def SetPkgbase(self, pkgbasename):
+ self._pkgbase = {'pkgname' : pkgbasename}
+ return self._pkgbase
+
+ def Save(self, db, cur, user):
+ # Obtain package base ID and previous maintainer.
+ cur.execute("SELECT ID, MaintainerUID FROM PackageBases "
+ "WHERE Name = %s", [self._pkgbase['pkgname']])
+ (pkgbase_id, maintainer_uid) = cur.fetchone()
+ was_orphan = not maintainer_uid
+
+ # Obtain the user ID of the new maintainer.
+ cur.execute("SELECT ID FROM Users WHERE Username = %s", [user])
+ user_id = int(cur.fetchone()[0])
+
+ # Update package base details and delete current packages.
+ cur.execute("UPDATE PackageBases SET ModifiedTS = UNIX_TIMESTAMP(), " +
+ "MaintainerUID = %s, PackagerUID = %s, " +
+ "OutOfDateTS = NULL WHERE ID = %s",
+ [user_id, user_id, pkgbase_id])
+ cur.execute("DELETE FROM Packages WHERE PackageBaseID = %s",
+ [pkgbase_id])
+
+ for pkgname in self._packages.keys():
+ pkginfo = self.GetMergedPackage(pkgname)
+
+ if 'epoch' in pkginfo and pkginfo['epoch'] > 0:
+ ver = '%d:%s-%s' % (pkginfo['epoch'], pkginfo['pkgver'],
+ pkginfo['pkgrel'])
+ else:
+ ver = '%s-%s' % (pkginfo['pkgver'], pkginfo['pkgrel'])
+
+ # Create a new package.
+ cur.execute("INSERT INTO Packages (PackageBaseID, Name, " +
+ "Version, Description, URL) " +
+ "VALUES (%s, %s, %s, %s, %s)",
+ [pkgbase_id, pkginfo['pkgname'], ver,
+ pkginfo['pkgdesc'], pkginfo['url']])
+ db.commit()
+ pkgid = cur.lastrowid
+
+ # Add package sources.
+ for source in pkginfo['source']:
+ cur.execute("INSERT INTO PackageSources (PackageID, Source) " +
+ "VALUES (%s, %s)", [pkgid, source])
+
+ # Add package dependencies.
+ for deptype in ('depends', 'makedepends',
+ 'checkdepends', 'optdepends'):
+ if not deptype in pkginfo:
+ continue
+ cur.execute("SELECT ID FROM DependencyTypes WHERE Name = %s",
+ [deptype])
+ deptypeid = cur.fetchone()[0]
+ for dep in pkginfo[deptype]:
+ depname = re.sub(r'(<|=|>).*', '', dep)
+ depcond = dep[len(depname):]
+ cur.execute("INSERT INTO PackageDepends (PackageID, " +
+ "DepTypeID, DepName, DepCondition) " +
+ "VALUES (%s, %s, %s, %s)", [pkgid, deptypeid,
+ depname, depcond])
+
+ # Add package relations (conflicts, provides, replaces).
+ for reltype in ('conflicts', 'provides', 'replaces'):
+ if not reltype in pkginfo:
+ continue
+ cur.execute("SELECT ID FROM RelationTypes WHERE Name = %s",
+ [reltype])
+ reltypeid = cur.fetchone()[0]
+ for rel in pkginfo[reltype]:
+ relname = re.sub(r'(<|=|>).*', '', rel)
+ relcond = rel[len(relname):]
+ cur.execute("INSERT INTO PackageRelations (PackageID, " +
+ "RelTypeID, RelName, RelCondition) " +
+ "VALUES (%s, %s, %s, %s)", [pkgid, reltypeid,
+ relname, relcond])
+
+ # Add package licenses.
+ if 'license' in pkginfo:
+ for license in pkginfo['license']:
+ cur.execute("SELECT ID FROM Licenses WHERE Name = %s",
+ [license])
+ if cur.rowcount == 1:
+ licenseid = cur.fetchone()[0]
+ else:
+ cur.execute("INSERT INTO Licenses (Name) VALUES (%s)",
+ [license])
+ db.commit()
+ licenseid = cur.lastrowid
+ cur.execute("INSERT INTO PackageLicenses (PackageID, " +
+ "LicenseID) VALUES (%s, %s)",
+ [pkgid, licenseid])
+
+ # Add package groups.
+ if 'groups' in pkginfo:
+ for group in pkginfo['groups']:
+ cur.execute("SELECT ID FROM Groups WHERE Name = %s",
+ [group])
+ if cur.rowcount == 1:
+ groupid = cur.fetchone()[0]
+ else:
+ cur.execute("INSERT INTO Groups (Name) VALUES (%s)",
+ [group])
+ db.commit()
+ groupid = cur.lastrowid
+ cur.execute("INSERT INTO PackageGroups (PackageID, "
+ "GroupID) VALUES (%s, %s)", [pkgid, groupid])
+
+ # Add user to notification list on adoption.
+ if was_orphan:
+ cur.execute("INSERT INTO CommentNotify (PackageBaseID, UserID) " +
+ "VALUES (%s, %s)", [pkgbase_id, user_id])
+
+ db.commit()
+
+class ECatcherInterface(object):
+ def Catch(self, lineno, error):
+ raise NotImplementedError
+
+
+class CollectionECatcher(ECatcherInterface):
+ def __init__(self):
+ self._errors = []
+
+ def Catch(self, lineno, error):
+ self._errors.append((lineno, error))
+
+ def HasErrors(self):
+ return len(self._errors) > 0
+
+ def Errors(self):
+ return copy(self._errors)
+
+
+def ParseAurinfoFromIterable(iterable, ecatcher=None):
+ aurinfo = AurInfo()
+
+ if ecatcher is None:
+ ecatcher = StderrECatcher()
+
+ current_package = None
+ lineno = 0
+
+ for line in iterable:
+ lineno += 1
+
+ if not line.strip():
+ # end of package
+ current_package = None
+ continue
+
+ if not line.startswith('\t'):
+ # start of new package
+ try:
+ key, value = map(lambda s: s.strip(), line.split('=', 1))
+ except ValueError:
+ ecatcher.Catch(lineno, 'unexpected header format in section=%s' %
+ current_package['pkgname'])
+ continue
+
+ if key == 'pkgbase':
+ current_package = aurinfo.SetPkgbase(value)
+ else:
+ current_package = aurinfo.AddPackage(value)
+ else:
+ # package attribute
+ if current_package is None:
+ ecatcher.Catch(lineno, 'package attribute found outside of '
+ 'a package section')
+ continue
+
+ try:
+ key, value = map(lambda s: s.strip(), line.split('=', 1))
+ except ValueError:
+ ecatcher.Catch(lineno, 'unexpected attribute format in '
+ 'section=%s' % current_package['pkgname'])
+
+ if IsMultiValued(key):
+ if not current_package.get(key):
+ current_package[key] = []
+ current_package[key].append(value)
+ else:
+ if not current_package.get(key):
+ current_package[key] = value
+ else:
+ ecatcher.Catch(lineno, 'overwriting attribute '
+ '%s: %s -> %s' % (key, current_package[key],
+ value))
+
+ return aurinfo
+
+def die(msg):
+ sys.stderr.write("error: %s\n" % (msg))
+ exit(1)
+
+def die_commit(msg, commit):
+ sys.stderr.write("error: The following error " +
+ "occurred when parsing commit\n")
+ sys.stderr.write("error: %s:\n" % (commit))
+ sys.stderr.write("error: %s\n" % (msg))
+ exit(1)
+
+if len(sys.argv) != 4:
+ die("invalid arguments")
+
+refname = sys.argv[1]
+sha1_old = sys.argv[2]
+sha1_new = sys.argv[3]
+
+user = os.environ.get("AUR_USER")
+pkgbase = os.environ.get("AUR_PKGBASE")
+git_dir = os.environ.get("AUR_GIT_DIR")
+
+if refname != "refs/heads/master":
+ die("pushing to a branch other than master is restricted")
+
+repo = pygit2.Repository(git_dir)
+walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL)
+if sha1_old != "0000000000000000000000000000000000000000":
+ walker.hide(sha1_old)
+
+for commit in walker:
+ if not '.AURINFO' in commit.tree:
+ die_commit("missing .AURINFO", commit.id)
+
+ for treeobj in commit.tree:
+ if repo[treeobj.id].size > 100000:
+ die_commit("maximum blob size (100kB) exceeded", commit.id)
+
+ aurinfo_raw = repo[commit.tree['.AURINFO'].id].data.decode()
+ ecatcher = CollectionECatcher()
+ aurinfo = ParseAurinfoFromIterable(aurinfo_raw.split('\n'), ecatcher)
+ errors = ecatcher.Errors()
+ if errors:
+ sys.stderr.write("error: The following errors occurred "
+ "when parsing .AURINFO in commit\n")
+ sys.stderr.write("error: %s:\n" % (commit.id))
+ for error in errors:
+ sys.stderr.write("error: line %d: %s\n" % error)
+ exit(1)
+
+ if aurinfo.GetPkgbase() != pkgbase:
+ die_commit('invalid pkgbase: %s' % (aurinfo.GetPkgbase()), commit.id)
+
+ for pkgname in aurinfo.GetPackageNames():
+ pkginfo = aurinfo.GetMergedPackage(pkgname)
+
+ if not re.match(r'[a-z0-9][a-z0-9\.+_-]*$', pkginfo['pkgname']):
+ die_commit('invalid package name: %s' % (pkginfo['pkgname']),
+ commit.id)
+
+ if not re.match(r'(?:http|ftp)s?://.*', pkginfo['url']):
+ die_commit('invalid URL: %s' % (pkginfo['url']), commit.id)
+
+ for field in ('pkgname', 'pkgdesc', 'url'):
+ if len(pkginfo[field]) > 255:
+ die_commit('%s field too long: %s' % (field, pkginfo[field]),
+ commit.id)
+
+aurinfo_raw = repo[repo[sha1_new].tree['.AURINFO'].id].data.decode()
+ecatcher = CollectionECatcher()
+aurinfo = ParseAurinfoFromIterable(aurinfo_raw.split('\n'), ecatcher)
+
+db = mysql.connector.connect(host=aur_db_host, user=aur_db_user,
+ passwd=aur_db_pass, db=aur_db_name,
+ buffered=True)
+cur = db.cursor()
+
+aurinfo.Save(db, cur, user)
+
+db.close()
--
2.0.0
More information about the aur-dev
mailing list