[aur-dev] [PATCH] Rewrite aurblup in Python

9 Jan 2015

The AUR backend already uses several Python scripts, rewrite the aurblup
helper as well. This has several advantages:

* We can easily use the main configuration file without using any shell
  script wrappers.

* aurblup does not need to be recompiled on libalpm soname bumps.

Signed-off-by: Lukas Fleischer <archlinux@cryptocrack.de>
---
 conf/config.proto               |   5 +
 scripts/aurblup/.gitignore      |   3 -
 scripts/aurblup/Makefile        |  26 ----
 scripts/aurblup/README          |  28 ----
 scripts/aurblup/aurblup-wrapper |  21 ---
 scripts/aurblup/aurblup.c       | 287 ----------------------------------------
 scripts/aurblup/aurblup.py      |  48 +++++++
 scripts/aurblup/config.h.proto  |  12 --
 scripts/aurblup/config.mk       |   6 -
 9 files changed, 53 insertions(+), 383 deletions(-)
 delete mode 100644 scripts/aurblup/.gitignore
 delete mode 100644 scripts/aurblup/Makefile
 delete mode 100644 scripts/aurblup/README
 delete mode 100755 scripts/aurblup/aurblup-wrapper
 delete mode 100644 scripts/aurblup/aurblup.c
 create mode 100755 scripts/aurblup/aurblup.py
 delete mode 100644 scripts/aurblup/config.h.proto
 delete mode 100644 scripts/aurblup/config.mk

diff --git a/conf/config.proto b/conf/config.proto
index f1d226c..703aeea 100644
--- a/conf/config.proto
+++ b/conf/config.proto
@@ -38,3 +38,8 @@ repo-regex = [a-z0-9][a-z0-9.+_-]*$
 template-path =  /srv/http/aur/scripts/git-integration/templates/
 git-update-hook = /srv/http/aur/scripts/git-integration/git-update.py
 git-shell-cmd = /usr/bin/git-shell
+
+[aurblup]
+db-path = /srv/http/aur/scripts/aurblup/
+sync-dbs = core extra community multilib testing community-testing
+servers = ftp://mirrors.kernel.org/archlinux/%s/os/x86_64
diff --git a/scripts/aurblup/.gitignore b/scripts/aurblup/.gitignore
deleted file mode 100644
index ae50736..0000000
--- a/scripts/aurblup/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-config.h
-*.o
-aurblup
diff --git a/scripts/aurblup/Makefile b/scripts/aurblup/Makefile
deleted file mode 100644
index 2d57470..0000000
--- a/scripts/aurblup/Makefile
+++ /dev/null
@@ -1,26 +0,0 @@
-include config.mk
-
-SRC = aurblup.c
-OBJ = ${SRC:.c=.o}
-
-all: aurblup
-
-config.h:
-	cp config.h.proto config.h
-
-${OBJ}: config.h
-
-aurblup: ${OBJ}
-
-install: aurblup
-	install -Dm0755 aurblup "${DESTDIR}${PREFIX}/bin/aurblup"
-	install -dm0755 "${DESTDIR}/var/lib/aurblup/"
-
-uninstall:
-	rm -f "${DESTDIR}${PREFIX}/bin/aurblup"
-	rm -f "${DESTDIR}/var/lib/aurblup/"
-
-clean:
-	rm -f aurblup ${OBJ}
-
-.PHONY: all install uninstall clean
diff --git a/scripts/aurblup/README b/scripts/aurblup/README
deleted file mode 100644
index 0cc811c..0000000
--- a/scripts/aurblup/README
+++ /dev/null
@@ -1,28 +0,0 @@
-aurblup
-=======
-
-aurblup is a small and lightweight tool that updates the package blacklist of
-an AUR MySQL database using one (or more) package databases. It does the
-following things:
-
-- Sync a bunch of local package databases with a remote server.
-
-- Get a list of packages in those databases.
-
-- Update the MySQL blacklist table to match those packages, including those
-  packages' provides and replaces.
-
-Requirements
-------------
-
-You need the libalpm and libmysqlclient header files to build aurblup.
-
-Installation
-------------
-
-Edit the "config.h" (copy from "config.h.proto" if doesn't exist) and
-"config.mk" configuration files to match your setup and enter the following
-command to build and install aurblup:
-
-	make install
-
diff --git a/scripts/aurblup/aurblup-wrapper b/scripts/aurblup/aurblup-wrapper
deleted file mode 100755
index c7b20af..0000000
--- a/scripts/aurblup/aurblup-wrapper
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/php
-<?php
-$dir = $argv[1];
-
-if (empty($dir)) {
-	echo "Please specify AUR directory.\n";
-	exit;
-}
-
-set_include_path(get_include_path() . PATH_SEPARATOR . "$dir/lib");
-include("confparser.inc.php");
-
-$user = config_get('database', 'user');
-$password = config_get('database', 'password');
-$name = config_get('database', 'name');
-
-exec($dir . "/../scripts/aurblup/aurblup " .
-	"-S /var/run/mysqld/mysqld.sock " .
-	"-u " . escapeshellarg($user) . " " .
-	"-p " . escapeshellarg($password) . " " .
-	"-D " . escapeshellarg($name));
diff --git a/scripts/aurblup/aurblup.c b/scripts/aurblup/aurblup.c
deleted file mode 100644
index 971dc06..0000000
--- a/scripts/aurblup/aurblup.c
+++ /dev/null
@@ -1,287 +0,0 @@
-/* aurblup - AUR blacklist updater
- *
- * Small utility to update the AUR package blacklist. Can be used in a cronjob.
- * Check the "README" file for details.
- */
-
-#include <alpm.h>
-#include <getopt.h>
-#include <mysql.h>
-#include <stdio.h>
-#include <string.h>
-
-#include "config.h"
-
-#define alpm_die(...) die(__VA_ARGS__, alpm_strerror(alpm_errno(handle)));
-#define mysql_die(...) die(__VA_ARGS__, mysql_error(c));
-
-static void die(const char *, ...);
-static alpm_list_t *pkglist_append(alpm_list_t *, const char *);
-static alpm_list_t *blacklist_get_pkglist();
-static void blacklist_add(const char *);
-static void blacklist_remove(const char *);
-static void blacklist_sync(alpm_list_t *, alpm_list_t *);
-static alpm_list_t *dblist_get_pkglist(alpm_list_t *);
-static alpm_list_t *dblist_create(void);
-static int parse_options(int, char **);
-static void init(void);
-static void cleanup(void);
-
-static char *mysql_host = "localhost";
-static char *mysql_socket = NULL;
-static char *mysql_user = "aur";
-static char *mysql_passwd = "aur";
-static char *mysql_db = "AUR";
-
-static MYSQL *c;
-
-static alpm_handle_t *handle;
-
-static void
-die(const char *format, ...)
-{
-  va_list arg;
-
-  va_start(arg, format);
-  fprintf(stderr, "aurblup: ");
-  vfprintf(stderr, format, arg);
-  va_end(arg);
-
-  cleanup();
-  exit(1);
-}
-
-static alpm_list_t *
-pkglist_append(alpm_list_t *pkglist, const char *pkgname)
-{
-  int len = strcspn(pkgname, "<=>");
-  if (!len)
-    len = strlen(pkgname);
-
-  char *s = malloc(len + 1);
-
-  strncpy(s, pkgname, len);
-  s[len] = '\0';
-
-  if (alpm_list_find_str(pkglist, s))
-    free(s);
-  else
-    pkglist = alpm_list_add(pkglist, s);
-
-  return pkglist;
-}
-
-static alpm_list_t *
-blacklist_get_pkglist()
-{
-  MYSQL_RES *res;
-  MYSQL_ROW row;
-  alpm_list_t *pkglist = NULL;
-
-  if (mysql_query(c, "SELECT Name FROM PackageBlacklist"))
-    mysql_die("failed to read blacklist from MySQL database: %s\n");
-
-  if (!(res = mysql_store_result(c)))
-    mysql_die("failed to store MySQL result: %s\n");
-
-  while ((row = mysql_fetch_row(res)))
-    pkglist = pkglist_append(pkglist, row[0]);
-
-  mysql_free_result(res);
-
-  return pkglist;
-}
-
-static void
-blacklist_add(const char *name)
-{
-  char *esc = malloc(strlen(name) * 2 + 1);
-  char query[1024];
-
-  mysql_real_escape_string(c, esc, name, strlen(name));
-  snprintf(query, 1024, "INSERT INTO PackageBlacklist (Name) "
-                        "VALUES ('%s')", esc);
-  free(esc);
-
-  if (mysql_query(c, query))
-    mysql_die("failed to query MySQL database (\"%s\"): %s\n", query);
-}
-
-static void
-blacklist_remove(const char *name)
-{
-  char *esc = malloc(strlen(name) * 2 + 1);
-  char query[1024];
-
-  mysql_real_escape_string(c, esc, name, strlen(name));
-  snprintf(query, 1024, "DELETE FROM PackageBlacklist WHERE Name = '%s'", esc);
-  free(esc);
-
-  if (mysql_query(c, query))
-    mysql_die("failed to query MySQL database (\"%s\"): %s\n", query);
-}
-
-static void
-blacklist_sync(alpm_list_t *pkgs_cur, alpm_list_t *pkgs_new)
-{
-  alpm_list_t *pkgs_add, *pkgs_rem, *p;
-
-  pkgs_add = alpm_list_diff(pkgs_new, pkgs_cur, (alpm_list_fn_cmp)strcmp);
-  pkgs_rem = alpm_list_diff(pkgs_cur, pkgs_new, (alpm_list_fn_cmp)strcmp);
-
-  if (mysql_query(c, "START TRANSACTION"))
-    mysql_die("failed to start MySQL transaction: %s\n");
-
-  for (p = pkgs_add; p; p = alpm_list_next(p))
-    blacklist_add(p->data);
-
-  for (p = pkgs_rem; p; p = alpm_list_next(p))
-    blacklist_remove(p->data);
-
-  if (mysql_query(c, "COMMIT"))
-    mysql_die("failed to commit MySQL transaction: %s\n");
-
-  alpm_list_free(pkgs_add);
-  alpm_list_free(pkgs_rem);
-}
-
-static alpm_list_t *
-dblist_get_pkglist(alpm_list_t *dblist)
-{
-  alpm_list_t *d, *p, *q;
-  alpm_list_t *pkglist = NULL;
-
-  for (d = dblist; d; d = alpm_list_next(d)) {
-    alpm_db_t *db = d->data;
-
-    if (alpm_trans_init(handle, 0))
-      alpm_die("failed to initialize ALPM transaction: %s\n");
-    if (alpm_db_update(0, db) < 0)
-      alpm_die("failed to update ALPM database: %s\n");
-    if (alpm_trans_release(handle))
-      alpm_die("failed to release ALPM transaction: %s\n");
-
-    for (p = alpm_db_get_pkgcache(db); p; p = alpm_list_next(p)) {
-      alpm_pkg_t *pkg = p->data;
-
-      pkglist = pkglist_append(pkglist, alpm_pkg_get_name(pkg));
-
-      for (q = alpm_pkg_get_replaces(pkg); q; q = alpm_list_next(q)) {
-        alpm_depend_t *replace = q->data;
-        pkglist = pkglist_append(pkglist, replace->name);
-      }
-    }
-  }
-
-  return pkglist;
-}
-
-static alpm_list_t *
-dblist_create(void)
-{
-  alpm_list_t *d;
-  alpm_list_t *dblist = NULL;
-  int i;
-
-  for (i = 0; i < sizeof(alpm_repos) / sizeof(char *); i++) {
-    if (!alpm_register_syncdb(handle, alpm_repos[i], 0))
-      alpm_die("failed to register sync db \"%s\": %s\n", alpm_repos[i]);
-  }
-
-  if (!(dblist = alpm_get_syncdbs(handle)))
-    alpm_die("failed to get sync DBs: %s\n");
-
-  for (d = dblist; d; d = alpm_list_next(d)) {
-    alpm_db_t *db = d->data;
-
-    char server[1024];
-    snprintf(server, 1024, ALPM_MIRROR, alpm_db_get_name(db));
-
-    if (alpm_db_add_server(db, server))
-      alpm_die("failed to set server \"%s\": %s\n", server);
-  }
-
-  return dblist;
-}
-
-static int parse_options(int argc, char **argv)
-{
-  int opt;
-
-  static const struct option opts[] = {
-    { "mysql-host",   required_argument, 0, 'h' },
-    { "mysql-socket", required_argument, 0, 'S' },
-    { "mysql-user",   required_argument, 0, 'u' },
-    { "mysql-passwd", required_argument, 0, 'p' },
-    { "mysql-db",     required_argument, 0, 'D' },
-    { 0, 0, 0, 0 }
-  };
-
-  while((opt = getopt_long(argc, argv, "h:S:u:p:D:", opts, NULL)) != -1) {
-    switch(opt) {
-      case 'h':
-        mysql_host = optarg;
-        break;;
-      case 'S':
-        mysql_socket = optarg;
-        break;;
-      case 'u':
-        mysql_user = optarg;
-        break;;
-      case 'p':
-        mysql_passwd = optarg;
-        break;;
-      case 'D':
-        mysql_db = optarg;
-        break;;
-      default:
-        return 0;
-    }
-  }
-
-  return 1;
-}
-
-static void
-init(void)
-{
-  enum _alpm_errno_t alpm_err;
-  if (mysql_library_init(0, NULL, NULL))
-    mysql_die("could not initialize MySQL library: %s\n");
-  if (!(c = mysql_init(NULL)))
-    mysql_die("failed to setup MySQL client: %s\n");
-  if (!mysql_real_connect(c, mysql_host, mysql_user, mysql_passwd,
-                          mysql_db, 0, mysql_socket, 0))
-    mysql_die("failed to initiate MySQL connection to %s: %s\n", mysql_host);
-
-  if ((handle = alpm_initialize("/", ALPM_DBPATH, &alpm_err)) == NULL)
-    die("failed to initialize ALPM: %s\n", alpm_strerror(alpm_err));
-}
-
-static void
-cleanup(void)
-{
-  alpm_release(handle);
-  mysql_close(c);
-  mysql_library_end();
-}
-
-int main(int argc, char *argv[])
-{
-  alpm_list_t *pkgs_cur, *pkgs_new;
-
-  if (!parse_options(argc, argv))
-    return 1;
-
-  init();
-
-  pkgs_cur = blacklist_get_pkglist();
-  pkgs_new = dblist_get_pkglist(dblist_create());
-  blacklist_sync(pkgs_cur, pkgs_new);
-  FREELIST(pkgs_new);
-  FREELIST(pkgs_cur);
-
-  cleanup();
-
-  return 0;
-}
diff --git a/scripts/aurblup/aurblup.py b/scripts/aurblup/aurblup.py
new file mode 100755
index 0000000..e678fee
--- /dev/null
+++ b/scripts/aurblup/aurblup.py
@@ -0,0 +1,48 @@
+#!/usr/bin/python3
+
+import configparser
+import mysql.connector
+import os
+import pyalpm
+
+config = configparser.RawConfigParser()
+config.read(os.path.dirname(os.path.realpath(__file__)) + "/../../conf/config")
+
+aur_db_host = config.get('database', 'host')
+aur_db_name = config.get('database', 'name')
+aur_db_user = config.get('database', 'user')
+aur_db_pass = config.get('database', 'password')
+aur_db_socket = config.get('database', 'socket')
+db_path = config.get('aurblup', 'db-path')
+sync_dbs = config.get('aurblup', 'sync-dbs').split(' ')
+servers = config.get('aurblup', 'servers').split(' ')
+
+blacklist = set()
+
+h = pyalpm.Handle("/", db_path)
+for sync_db in sync_dbs:
+    repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL)
+    repo.servers = [server.replace("%s", sync_db) for server in servers]
+    t = h.init_transaction()
+    repo.update(False)
+    t.release()
+
+    for pkg in repo.pkgcache:
+        blacklist.add(pkg.name)
+        [blacklist.add(x) for x in pkg.replaces]
+
+db = mysql.connector.connect(host=aur_db_host, user=aur_db_user,
+                             passwd=aur_db_pass, db=aur_db_name,
+                             unix_socket=aur_db_socket, buffered=True)
+cur = db.cursor()
+
+cur.execute("SELECT Name FROM PackageBlacklist")
+oldblacklist = set([row[0] for row in cur.fetchall()])
+
+for pkg in blacklist.difference(oldblacklist):
+    cur.execute("INSERT INTO PackageBlacklist (Name) VALUES (%s)", [pkg])
+for pkg in oldblacklist.difference(blacklist):
+    cur.execute("DELETE FROM PackageBlacklist WHERE Name = %s", [pkg])
+
+db.commit()
+db.close()
diff --git a/scripts/aurblup/config.h.proto b/scripts/aurblup/config.h.proto
deleted file mode 100644
index 45acb87..0000000
--- a/scripts/aurblup/config.h.proto
+++ /dev/null
@@ -1,12 +0,0 @@
-/* libalpm options */
-#define ALPM_DBPATH "/var/lib/aurblup/"
-#define ALPM_MIRROR "ftp://mirrors.kernel.org/archlinux/%s/os/x86_64"
-
-static const char *alpm_repos[] = {
-  "core",
-  "community",
-  "community-testing",
-  "extra",
-  "multilib",
-  "testing"
-};
diff --git a/scripts/aurblup/config.mk b/scripts/aurblup/config.mk
deleted file mode 100644
index f8e12a8..0000000
--- a/scripts/aurblup/config.mk
+++ /dev/null
@@ -1,6 +0,0 @@
-PREFIX = /usr/local
-
-CFLAGS = -g -O2 -std=c99 -pedantic -Wall -I/usr/include/mysql
-LDFLAGS = -g -lalpm -lmysqlclient
-
-CC = cc
-- 
2.2.1

    