The AUR backend already uses several Python scripts, rewrite the aurblup helper as well. This has several advantages: * We can easily use the main configuration file without using any shell script wrappers. * aurblup does not need to be recompiled on libalpm soname bumps. Signed-off-by: Lukas Fleischer <archlinux@cryptocrack.de> --- conf/config.proto | 5 + scripts/aurblup/.gitignore | 3 - scripts/aurblup/Makefile | 26 ---- scripts/aurblup/README | 28 ---- scripts/aurblup/aurblup-wrapper | 21 --- scripts/aurblup/aurblup.c | 287 ---------------------------------------- scripts/aurblup/aurblup.py | 48 +++++++ scripts/aurblup/config.h.proto | 12 -- scripts/aurblup/config.mk | 6 - 9 files changed, 53 insertions(+), 383 deletions(-) delete mode 100644 scripts/aurblup/.gitignore delete mode 100644 scripts/aurblup/Makefile delete mode 100644 scripts/aurblup/README delete mode 100755 scripts/aurblup/aurblup-wrapper delete mode 100644 scripts/aurblup/aurblup.c create mode 100755 scripts/aurblup/aurblup.py delete mode 100644 scripts/aurblup/config.h.proto delete mode 100644 scripts/aurblup/config.mk diff --git a/conf/config.proto b/conf/config.proto index f1d226c..703aeea 100644 --- a/conf/config.proto +++ b/conf/config.proto @@ -38,3 +38,8 @@ repo-regex = [a-z0-9][a-z0-9.+_-]*$ template-path = /srv/http/aur/scripts/git-integration/templates/ git-update-hook = /srv/http/aur/scripts/git-integration/git-update.py git-shell-cmd = /usr/bin/git-shell + +[aurblup] +db-path = /srv/http/aur/scripts/aurblup/ +sync-dbs = core extra community multilib testing community-testing +servers = ftp://mirrors.kernel.org/archlinux/%s/os/x86_64 diff --git a/scripts/aurblup/.gitignore b/scripts/aurblup/.gitignore deleted file mode 100644 index ae50736..0000000 --- a/scripts/aurblup/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -config.h -*.o -aurblup diff --git a/scripts/aurblup/Makefile b/scripts/aurblup/Makefile deleted file mode 100644 index 2d57470..0000000 --- a/scripts/aurblup/Makefile +++ /dev/null @@ -1,26 +0,0 @@ -include config.mk - -SRC = aurblup.c -OBJ = ${SRC:.c=.o} - -all: aurblup - -config.h: - cp config.h.proto config.h - -${OBJ}: config.h - -aurblup: ${OBJ} - -install: aurblup - install -Dm0755 aurblup "${DESTDIR}${PREFIX}/bin/aurblup" - install -dm0755 "${DESTDIR}/var/lib/aurblup/" - -uninstall: - rm -f "${DESTDIR}${PREFIX}/bin/aurblup" - rm -f "${DESTDIR}/var/lib/aurblup/" - -clean: - rm -f aurblup ${OBJ} - -.PHONY: all install uninstall clean diff --git a/scripts/aurblup/README b/scripts/aurblup/README deleted file mode 100644 index 0cc811c..0000000 --- a/scripts/aurblup/README +++ /dev/null @@ -1,28 +0,0 @@ -aurblup -======= - -aurblup is a small and lightweight tool that updates the package blacklist of -an AUR MySQL database using one (or more) package databases. It does the -following things: - -- Sync a bunch of local package databases with a remote server. - -- Get a list of packages in those databases. - -- Update the MySQL blacklist table to match those packages, including those - packages' provides and replaces. - -Requirements ------------- - -You need the libalpm and libmysqlclient header files to build aurblup. - -Installation ------------- - -Edit the "config.h" (copy from "config.h.proto" if doesn't exist) and -"config.mk" configuration files to match your setup and enter the following -command to build and install aurblup: - - make install - diff --git a/scripts/aurblup/aurblup-wrapper b/scripts/aurblup/aurblup-wrapper deleted file mode 100755 index c7b20af..0000000 --- a/scripts/aurblup/aurblup-wrapper +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/php -<?php -$dir = $argv[1]; - -if (empty($dir)) { - echo "Please specify AUR directory.\n"; - exit; -} - -set_include_path(get_include_path() . PATH_SEPARATOR . "$dir/lib"); -include("confparser.inc.php"); - -$user = config_get('database', 'user'); -$password = config_get('database', 'password'); -$name = config_get('database', 'name'); - -exec($dir . "/../scripts/aurblup/aurblup " . - "-S /var/run/mysqld/mysqld.sock " . - "-u " . escapeshellarg($user) . " " . - "-p " . escapeshellarg($password) . " " . - "-D " . escapeshellarg($name)); diff --git a/scripts/aurblup/aurblup.c b/scripts/aurblup/aurblup.c deleted file mode 100644 index 971dc06..0000000 --- a/scripts/aurblup/aurblup.c +++ /dev/null @@ -1,287 +0,0 @@ -/* aurblup - AUR blacklist updater - * - * Small utility to update the AUR package blacklist. Can be used in a cronjob. - * Check the "README" file for details. - */ - -#include <alpm.h> -#include <getopt.h> -#include <mysql.h> -#include <stdio.h> -#include <string.h> - -#include "config.h" - -#define alpm_die(...) die(__VA_ARGS__, alpm_strerror(alpm_errno(handle))); -#define mysql_die(...) die(__VA_ARGS__, mysql_error(c)); - -static void die(const char *, ...); -static alpm_list_t *pkglist_append(alpm_list_t *, const char *); -static alpm_list_t *blacklist_get_pkglist(); -static void blacklist_add(const char *); -static void blacklist_remove(const char *); -static void blacklist_sync(alpm_list_t *, alpm_list_t *); -static alpm_list_t *dblist_get_pkglist(alpm_list_t *); -static alpm_list_t *dblist_create(void); -static int parse_options(int, char **); -static void init(void); -static void cleanup(void); - -static char *mysql_host = "localhost"; -static char *mysql_socket = NULL; -static char *mysql_user = "aur"; -static char *mysql_passwd = "aur"; -static char *mysql_db = "AUR"; - -static MYSQL *c; - -static alpm_handle_t *handle; - -static void -die(const char *format, ...) -{ - va_list arg; - - va_start(arg, format); - fprintf(stderr, "aurblup: "); - vfprintf(stderr, format, arg); - va_end(arg); - - cleanup(); - exit(1); -} - -static alpm_list_t * -pkglist_append(alpm_list_t *pkglist, const char *pkgname) -{ - int len = strcspn(pkgname, "<=>"); - if (!len) - len = strlen(pkgname); - - char *s = malloc(len + 1); - - strncpy(s, pkgname, len); - s[len] = '\0'; - - if (alpm_list_find_str(pkglist, s)) - free(s); - else - pkglist = alpm_list_add(pkglist, s); - - return pkglist; -} - -static alpm_list_t * -blacklist_get_pkglist() -{ - MYSQL_RES *res; - MYSQL_ROW row; - alpm_list_t *pkglist = NULL; - - if (mysql_query(c, "SELECT Name FROM PackageBlacklist")) - mysql_die("failed to read blacklist from MySQL database: %s\n"); - - if (!(res = mysql_store_result(c))) - mysql_die("failed to store MySQL result: %s\n"); - - while ((row = mysql_fetch_row(res))) - pkglist = pkglist_append(pkglist, row[0]); - - mysql_free_result(res); - - return pkglist; -} - -static void -blacklist_add(const char *name) -{ - char *esc = malloc(strlen(name) * 2 + 1); - char query[1024]; - - mysql_real_escape_string(c, esc, name, strlen(name)); - snprintf(query, 1024, "INSERT INTO PackageBlacklist (Name) " - "VALUES ('%s')", esc); - free(esc); - - if (mysql_query(c, query)) - mysql_die("failed to query MySQL database (\"%s\"): %s\n", query); -} - -static void -blacklist_remove(const char *name) -{ - char *esc = malloc(strlen(name) * 2 + 1); - char query[1024]; - - mysql_real_escape_string(c, esc, name, strlen(name)); - snprintf(query, 1024, "DELETE FROM PackageBlacklist WHERE Name = '%s'", esc); - free(esc); - - if (mysql_query(c, query)) - mysql_die("failed to query MySQL database (\"%s\"): %s\n", query); -} - -static void -blacklist_sync(alpm_list_t *pkgs_cur, alpm_list_t *pkgs_new) -{ - alpm_list_t *pkgs_add, *pkgs_rem, *p; - - pkgs_add = alpm_list_diff(pkgs_new, pkgs_cur, (alpm_list_fn_cmp)strcmp); - pkgs_rem = alpm_list_diff(pkgs_cur, pkgs_new, (alpm_list_fn_cmp)strcmp); - - if (mysql_query(c, "START TRANSACTION")) - mysql_die("failed to start MySQL transaction: %s\n"); - - for (p = pkgs_add; p; p = alpm_list_next(p)) - blacklist_add(p->data); - - for (p = pkgs_rem; p; p = alpm_list_next(p)) - blacklist_remove(p->data); - - if (mysql_query(c, "COMMIT")) - mysql_die("failed to commit MySQL transaction: %s\n"); - - alpm_list_free(pkgs_add); - alpm_list_free(pkgs_rem); -} - -static alpm_list_t * -dblist_get_pkglist(alpm_list_t *dblist) -{ - alpm_list_t *d, *p, *q; - alpm_list_t *pkglist = NULL; - - for (d = dblist; d; d = alpm_list_next(d)) { - alpm_db_t *db = d->data; - - if (alpm_trans_init(handle, 0)) - alpm_die("failed to initialize ALPM transaction: %s\n"); - if (alpm_db_update(0, db) < 0) - alpm_die("failed to update ALPM database: %s\n"); - if (alpm_trans_release(handle)) - alpm_die("failed to release ALPM transaction: %s\n"); - - for (p = alpm_db_get_pkgcache(db); p; p = alpm_list_next(p)) { - alpm_pkg_t *pkg = p->data; - - pkglist = pkglist_append(pkglist, alpm_pkg_get_name(pkg)); - - for (q = alpm_pkg_get_replaces(pkg); q; q = alpm_list_next(q)) { - alpm_depend_t *replace = q->data; - pkglist = pkglist_append(pkglist, replace->name); - } - } - } - - return pkglist; -} - -static alpm_list_t * -dblist_create(void) -{ - alpm_list_t *d; - alpm_list_t *dblist = NULL; - int i; - - for (i = 0; i < sizeof(alpm_repos) / sizeof(char *); i++) { - if (!alpm_register_syncdb(handle, alpm_repos[i], 0)) - alpm_die("failed to register sync db \"%s\": %s\n", alpm_repos[i]); - } - - if (!(dblist = alpm_get_syncdbs(handle))) - alpm_die("failed to get sync DBs: %s\n"); - - for (d = dblist; d; d = alpm_list_next(d)) { - alpm_db_t *db = d->data; - - char server[1024]; - snprintf(server, 1024, ALPM_MIRROR, alpm_db_get_name(db)); - - if (alpm_db_add_server(db, server)) - alpm_die("failed to set server \"%s\": %s\n", server); - } - - return dblist; -} - -static int parse_options(int argc, char **argv) -{ - int opt; - - static const struct option opts[] = { - { "mysql-host", required_argument, 0, 'h' }, - { "mysql-socket", required_argument, 0, 'S' }, - { "mysql-user", required_argument, 0, 'u' }, - { "mysql-passwd", required_argument, 0, 'p' }, - { "mysql-db", required_argument, 0, 'D' }, - { 0, 0, 0, 0 } - }; - - while((opt = getopt_long(argc, argv, "h:S:u:p:D:", opts, NULL)) != -1) { - switch(opt) { - case 'h': - mysql_host = optarg; - break;; - case 'S': - mysql_socket = optarg; - break;; - case 'u': - mysql_user = optarg; - break;; - case 'p': - mysql_passwd = optarg; - break;; - case 'D': - mysql_db = optarg; - break;; - default: - return 0; - } - } - - return 1; -} - -static void -init(void) -{ - enum _alpm_errno_t alpm_err; - if (mysql_library_init(0, NULL, NULL)) - mysql_die("could not initialize MySQL library: %s\n"); - if (!(c = mysql_init(NULL))) - mysql_die("failed to setup MySQL client: %s\n"); - if (!mysql_real_connect(c, mysql_host, mysql_user, mysql_passwd, - mysql_db, 0, mysql_socket, 0)) - mysql_die("failed to initiate MySQL connection to %s: %s\n", mysql_host); - - if ((handle = alpm_initialize("/", ALPM_DBPATH, &alpm_err)) == NULL) - die("failed to initialize ALPM: %s\n", alpm_strerror(alpm_err)); -} - -static void -cleanup(void) -{ - alpm_release(handle); - mysql_close(c); - mysql_library_end(); -} - -int main(int argc, char *argv[]) -{ - alpm_list_t *pkgs_cur, *pkgs_new; - - if (!parse_options(argc, argv)) - return 1; - - init(); - - pkgs_cur = blacklist_get_pkglist(); - pkgs_new = dblist_get_pkglist(dblist_create()); - blacklist_sync(pkgs_cur, pkgs_new); - FREELIST(pkgs_new); - FREELIST(pkgs_cur); - - cleanup(); - - return 0; -} diff --git a/scripts/aurblup/aurblup.py b/scripts/aurblup/aurblup.py new file mode 100755 index 0000000..e678fee --- /dev/null +++ b/scripts/aurblup/aurblup.py @@ -0,0 +1,48 @@ +#!/usr/bin/python3 + +import configparser +import mysql.connector +import os +import pyalpm + +config = configparser.RawConfigParser() +config.read(os.path.dirname(os.path.realpath(__file__)) + "/../../conf/config") + +aur_db_host = config.get('database', 'host') +aur_db_name = config.get('database', 'name') +aur_db_user = config.get('database', 'user') +aur_db_pass = config.get('database', 'password') +aur_db_socket = config.get('database', 'socket') +db_path = config.get('aurblup', 'db-path') +sync_dbs = config.get('aurblup', 'sync-dbs').split(' ') +servers = config.get('aurblup', 'servers').split(' ') + +blacklist = set() + +h = pyalpm.Handle("/", db_path) +for sync_db in sync_dbs: + repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL) + repo.servers = [server.replace("%s", sync_db) for server in servers] + t = h.init_transaction() + repo.update(False) + t.release() + + for pkg in repo.pkgcache: + blacklist.add(pkg.name) + [blacklist.add(x) for x in pkg.replaces] + +db = mysql.connector.connect(host=aur_db_host, user=aur_db_user, + passwd=aur_db_pass, db=aur_db_name, + unix_socket=aur_db_socket, buffered=True) +cur = db.cursor() + +cur.execute("SELECT Name FROM PackageBlacklist") +oldblacklist = set([row[0] for row in cur.fetchall()]) + +for pkg in blacklist.difference(oldblacklist): + cur.execute("INSERT INTO PackageBlacklist (Name) VALUES (%s)", [pkg]) +for pkg in oldblacklist.difference(blacklist): + cur.execute("DELETE FROM PackageBlacklist WHERE Name = %s", [pkg]) + +db.commit() +db.close() diff --git a/scripts/aurblup/config.h.proto b/scripts/aurblup/config.h.proto deleted file mode 100644 index 45acb87..0000000 --- a/scripts/aurblup/config.h.proto +++ /dev/null @@ -1,12 +0,0 @@ -/* libalpm options */ -#define ALPM_DBPATH "/var/lib/aurblup/" -#define ALPM_MIRROR "ftp://mirrors.kernel.org/archlinux/%s/os/x86_64" - -static const char *alpm_repos[] = { - "core", - "community", - "community-testing", - "extra", - "multilib", - "testing" -}; diff --git a/scripts/aurblup/config.mk b/scripts/aurblup/config.mk deleted file mode 100644 index f8e12a8..0000000 --- a/scripts/aurblup/config.mk +++ /dev/null @@ -1,6 +0,0 @@ -PREFIX = /usr/local - -CFLAGS = -g -O2 -std=c99 -pedantic -Wall -I/usr/include/mysql -LDFLAGS = -g -lalpm -lmysqlclient - -CC = cc -- 2.2.1