[aur-dev] [PATCH] Rewrite aurblup in Python
Lukas Fleischer
archlinux at cryptocrack.de
Fri Jan 9 09:59:43 UTC 2015
The AUR backend already uses several Python scripts, rewrite the aurblup
helper as well. This has several advantages:
* We can easily use the main configuration file without using any shell
script wrappers.
* aurblup does not need to be recompiled on libalpm soname bumps.
Signed-off-by: Lukas Fleischer <archlinux at cryptocrack.de>
---
conf/config.proto | 5 +
scripts/aurblup/.gitignore | 3 -
scripts/aurblup/Makefile | 26 ----
scripts/aurblup/README | 28 ----
scripts/aurblup/aurblup-wrapper | 21 ---
scripts/aurblup/aurblup.c | 287 ----------------------------------------
scripts/aurblup/aurblup.py | 48 +++++++
scripts/aurblup/config.h.proto | 12 --
scripts/aurblup/config.mk | 6 -
9 files changed, 53 insertions(+), 383 deletions(-)
delete mode 100644 scripts/aurblup/.gitignore
delete mode 100644 scripts/aurblup/Makefile
delete mode 100644 scripts/aurblup/README
delete mode 100755 scripts/aurblup/aurblup-wrapper
delete mode 100644 scripts/aurblup/aurblup.c
create mode 100755 scripts/aurblup/aurblup.py
delete mode 100644 scripts/aurblup/config.h.proto
delete mode 100644 scripts/aurblup/config.mk
diff --git a/conf/config.proto b/conf/config.proto
index f1d226c..703aeea 100644
--- a/conf/config.proto
+++ b/conf/config.proto
@@ -38,3 +38,8 @@ repo-regex = [a-z0-9][a-z0-9.+_-]*$
template-path = /srv/http/aur/scripts/git-integration/templates/
git-update-hook = /srv/http/aur/scripts/git-integration/git-update.py
git-shell-cmd = /usr/bin/git-shell
+
+[aurblup]
+db-path = /srv/http/aur/scripts/aurblup/
+sync-dbs = core extra community multilib testing community-testing
+servers = ftp://mirrors.kernel.org/archlinux/%s/os/x86_64
diff --git a/scripts/aurblup/.gitignore b/scripts/aurblup/.gitignore
deleted file mode 100644
index ae50736..0000000
--- a/scripts/aurblup/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-config.h
-*.o
-aurblup
diff --git a/scripts/aurblup/Makefile b/scripts/aurblup/Makefile
deleted file mode 100644
index 2d57470..0000000
--- a/scripts/aurblup/Makefile
+++ /dev/null
@@ -1,26 +0,0 @@
-include config.mk
-
-SRC = aurblup.c
-OBJ = ${SRC:.c=.o}
-
-all: aurblup
-
-config.h:
- cp config.h.proto config.h
-
-${OBJ}: config.h
-
-aurblup: ${OBJ}
-
-install: aurblup
- install -Dm0755 aurblup "${DESTDIR}${PREFIX}/bin/aurblup"
- install -dm0755 "${DESTDIR}/var/lib/aurblup/"
-
-uninstall:
- rm -f "${DESTDIR}${PREFIX}/bin/aurblup"
- rm -f "${DESTDIR}/var/lib/aurblup/"
-
-clean:
- rm -f aurblup ${OBJ}
-
-.PHONY: all install uninstall clean
diff --git a/scripts/aurblup/README b/scripts/aurblup/README
deleted file mode 100644
index 0cc811c..0000000
--- a/scripts/aurblup/README
+++ /dev/null
@@ -1,28 +0,0 @@
-aurblup
-=======
-
-aurblup is a small and lightweight tool that updates the package blacklist of
-an AUR MySQL database using one (or more) package databases. It does the
-following things:
-
-- Sync a bunch of local package databases with a remote server.
-
-- Get a list of packages in those databases.
-
-- Update the MySQL blacklist table to match those packages, including those
- packages' provides and replaces.
-
-Requirements
-------------
-
-You need the libalpm and libmysqlclient header files to build aurblup.
-
-Installation
-------------
-
-Edit the "config.h" (copy from "config.h.proto" if doesn't exist) and
-"config.mk" configuration files to match your setup and enter the following
-command to build and install aurblup:
-
- make install
-
diff --git a/scripts/aurblup/aurblup-wrapper b/scripts/aurblup/aurblup-wrapper
deleted file mode 100755
index c7b20af..0000000
--- a/scripts/aurblup/aurblup-wrapper
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/php
-<?php
-$dir = $argv[1];
-
-if (empty($dir)) {
- echo "Please specify AUR directory.\n";
- exit;
-}
-
-set_include_path(get_include_path() . PATH_SEPARATOR . "$dir/lib");
-include("confparser.inc.php");
-
-$user = config_get('database', 'user');
-$password = config_get('database', 'password');
-$name = config_get('database', 'name');
-
-exec($dir . "/../scripts/aurblup/aurblup " .
- "-S /var/run/mysqld/mysqld.sock " .
- "-u " . escapeshellarg($user) . " " .
- "-p " . escapeshellarg($password) . " " .
- "-D " . escapeshellarg($name));
diff --git a/scripts/aurblup/aurblup.c b/scripts/aurblup/aurblup.c
deleted file mode 100644
index 971dc06..0000000
--- a/scripts/aurblup/aurblup.c
+++ /dev/null
@@ -1,287 +0,0 @@
-/* aurblup - AUR blacklist updater
- *
- * Small utility to update the AUR package blacklist. Can be used in a cronjob.
- * Check the "README" file for details.
- */
-
-#include <alpm.h>
-#include <getopt.h>
-#include <mysql.h>
-#include <stdio.h>
-#include <string.h>
-
-#include "config.h"
-
-#define alpm_die(...) die(__VA_ARGS__, alpm_strerror(alpm_errno(handle)));
-#define mysql_die(...) die(__VA_ARGS__, mysql_error(c));
-
-static void die(const char *, ...);
-static alpm_list_t *pkglist_append(alpm_list_t *, const char *);
-static alpm_list_t *blacklist_get_pkglist();
-static void blacklist_add(const char *);
-static void blacklist_remove(const char *);
-static void blacklist_sync(alpm_list_t *, alpm_list_t *);
-static alpm_list_t *dblist_get_pkglist(alpm_list_t *);
-static alpm_list_t *dblist_create(void);
-static int parse_options(int, char **);
-static void init(void);
-static void cleanup(void);
-
-static char *mysql_host = "localhost";
-static char *mysql_socket = NULL;
-static char *mysql_user = "aur";
-static char *mysql_passwd = "aur";
-static char *mysql_db = "AUR";
-
-static MYSQL *c;
-
-static alpm_handle_t *handle;
-
-static void
-die(const char *format, ...)
-{
- va_list arg;
-
- va_start(arg, format);
- fprintf(stderr, "aurblup: ");
- vfprintf(stderr, format, arg);
- va_end(arg);
-
- cleanup();
- exit(1);
-}
-
-static alpm_list_t *
-pkglist_append(alpm_list_t *pkglist, const char *pkgname)
-{
- int len = strcspn(pkgname, "<=>");
- if (!len)
- len = strlen(pkgname);
-
- char *s = malloc(len + 1);
-
- strncpy(s, pkgname, len);
- s[len] = '\0';
-
- if (alpm_list_find_str(pkglist, s))
- free(s);
- else
- pkglist = alpm_list_add(pkglist, s);
-
- return pkglist;
-}
-
-static alpm_list_t *
-blacklist_get_pkglist()
-{
- MYSQL_RES *res;
- MYSQL_ROW row;
- alpm_list_t *pkglist = NULL;
-
- if (mysql_query(c, "SELECT Name FROM PackageBlacklist"))
- mysql_die("failed to read blacklist from MySQL database: %s\n");
-
- if (!(res = mysql_store_result(c)))
- mysql_die("failed to store MySQL result: %s\n");
-
- while ((row = mysql_fetch_row(res)))
- pkglist = pkglist_append(pkglist, row[0]);
-
- mysql_free_result(res);
-
- return pkglist;
-}
-
-static void
-blacklist_add(const char *name)
-{
- char *esc = malloc(strlen(name) * 2 + 1);
- char query[1024];
-
- mysql_real_escape_string(c, esc, name, strlen(name));
- snprintf(query, 1024, "INSERT INTO PackageBlacklist (Name) "
- "VALUES ('%s')", esc);
- free(esc);
-
- if (mysql_query(c, query))
- mysql_die("failed to query MySQL database (\"%s\"): %s\n", query);
-}
-
-static void
-blacklist_remove(const char *name)
-{
- char *esc = malloc(strlen(name) * 2 + 1);
- char query[1024];
-
- mysql_real_escape_string(c, esc, name, strlen(name));
- snprintf(query, 1024, "DELETE FROM PackageBlacklist WHERE Name = '%s'", esc);
- free(esc);
-
- if (mysql_query(c, query))
- mysql_die("failed to query MySQL database (\"%s\"): %s\n", query);
-}
-
-static void
-blacklist_sync(alpm_list_t *pkgs_cur, alpm_list_t *pkgs_new)
-{
- alpm_list_t *pkgs_add, *pkgs_rem, *p;
-
- pkgs_add = alpm_list_diff(pkgs_new, pkgs_cur, (alpm_list_fn_cmp)strcmp);
- pkgs_rem = alpm_list_diff(pkgs_cur, pkgs_new, (alpm_list_fn_cmp)strcmp);
-
- if (mysql_query(c, "START TRANSACTION"))
- mysql_die("failed to start MySQL transaction: %s\n");
-
- for (p = pkgs_add; p; p = alpm_list_next(p))
- blacklist_add(p->data);
-
- for (p = pkgs_rem; p; p = alpm_list_next(p))
- blacklist_remove(p->data);
-
- if (mysql_query(c, "COMMIT"))
- mysql_die("failed to commit MySQL transaction: %s\n");
-
- alpm_list_free(pkgs_add);
- alpm_list_free(pkgs_rem);
-}
-
-static alpm_list_t *
-dblist_get_pkglist(alpm_list_t *dblist)
-{
- alpm_list_t *d, *p, *q;
- alpm_list_t *pkglist = NULL;
-
- for (d = dblist; d; d = alpm_list_next(d)) {
- alpm_db_t *db = d->data;
-
- if (alpm_trans_init(handle, 0))
- alpm_die("failed to initialize ALPM transaction: %s\n");
- if (alpm_db_update(0, db) < 0)
- alpm_die("failed to update ALPM database: %s\n");
- if (alpm_trans_release(handle))
- alpm_die("failed to release ALPM transaction: %s\n");
-
- for (p = alpm_db_get_pkgcache(db); p; p = alpm_list_next(p)) {
- alpm_pkg_t *pkg = p->data;
-
- pkglist = pkglist_append(pkglist, alpm_pkg_get_name(pkg));
-
- for (q = alpm_pkg_get_replaces(pkg); q; q = alpm_list_next(q)) {
- alpm_depend_t *replace = q->data;
- pkglist = pkglist_append(pkglist, replace->name);
- }
- }
- }
-
- return pkglist;
-}
-
-static alpm_list_t *
-dblist_create(void)
-{
- alpm_list_t *d;
- alpm_list_t *dblist = NULL;
- int i;
-
- for (i = 0; i < sizeof(alpm_repos) / sizeof(char *); i++) {
- if (!alpm_register_syncdb(handle, alpm_repos[i], 0))
- alpm_die("failed to register sync db \"%s\": %s\n", alpm_repos[i]);
- }
-
- if (!(dblist = alpm_get_syncdbs(handle)))
- alpm_die("failed to get sync DBs: %s\n");
-
- for (d = dblist; d; d = alpm_list_next(d)) {
- alpm_db_t *db = d->data;
-
- char server[1024];
- snprintf(server, 1024, ALPM_MIRROR, alpm_db_get_name(db));
-
- if (alpm_db_add_server(db, server))
- alpm_die("failed to set server \"%s\": %s\n", server);
- }
-
- return dblist;
-}
-
-static int parse_options(int argc, char **argv)
-{
- int opt;
-
- static const struct option opts[] = {
- { "mysql-host", required_argument, 0, 'h' },
- { "mysql-socket", required_argument, 0, 'S' },
- { "mysql-user", required_argument, 0, 'u' },
- { "mysql-passwd", required_argument, 0, 'p' },
- { "mysql-db", required_argument, 0, 'D' },
- { 0, 0, 0, 0 }
- };
-
- while((opt = getopt_long(argc, argv, "h:S:u:p:D:", opts, NULL)) != -1) {
- switch(opt) {
- case 'h':
- mysql_host = optarg;
- break;;
- case 'S':
- mysql_socket = optarg;
- break;;
- case 'u':
- mysql_user = optarg;
- break;;
- case 'p':
- mysql_passwd = optarg;
- break;;
- case 'D':
- mysql_db = optarg;
- break;;
- default:
- return 0;
- }
- }
-
- return 1;
-}
-
-static void
-init(void)
-{
- enum _alpm_errno_t alpm_err;
- if (mysql_library_init(0, NULL, NULL))
- mysql_die("could not initialize MySQL library: %s\n");
- if (!(c = mysql_init(NULL)))
- mysql_die("failed to setup MySQL client: %s\n");
- if (!mysql_real_connect(c, mysql_host, mysql_user, mysql_passwd,
- mysql_db, 0, mysql_socket, 0))
- mysql_die("failed to initiate MySQL connection to %s: %s\n", mysql_host);
-
- if ((handle = alpm_initialize("/", ALPM_DBPATH, &alpm_err)) == NULL)
- die("failed to initialize ALPM: %s\n", alpm_strerror(alpm_err));
-}
-
-static void
-cleanup(void)
-{
- alpm_release(handle);
- mysql_close(c);
- mysql_library_end();
-}
-
-int main(int argc, char *argv[])
-{
- alpm_list_t *pkgs_cur, *pkgs_new;
-
- if (!parse_options(argc, argv))
- return 1;
-
- init();
-
- pkgs_cur = blacklist_get_pkglist();
- pkgs_new = dblist_get_pkglist(dblist_create());
- blacklist_sync(pkgs_cur, pkgs_new);
- FREELIST(pkgs_new);
- FREELIST(pkgs_cur);
-
- cleanup();
-
- return 0;
-}
diff --git a/scripts/aurblup/aurblup.py b/scripts/aurblup/aurblup.py
new file mode 100755
index 0000000..e678fee
--- /dev/null
+++ b/scripts/aurblup/aurblup.py
@@ -0,0 +1,48 @@
+#!/usr/bin/python3
+
+import configparser
+import mysql.connector
+import os
+import pyalpm
+
+config = configparser.RawConfigParser()
+config.read(os.path.dirname(os.path.realpath(__file__)) + "/../../conf/config")
+
+aur_db_host = config.get('database', 'host')
+aur_db_name = config.get('database', 'name')
+aur_db_user = config.get('database', 'user')
+aur_db_pass = config.get('database', 'password')
+aur_db_socket = config.get('database', 'socket')
+db_path = config.get('aurblup', 'db-path')
+sync_dbs = config.get('aurblup', 'sync-dbs').split(' ')
+servers = config.get('aurblup', 'servers').split(' ')
+
+blacklist = set()
+
+h = pyalpm.Handle("/", db_path)
+for sync_db in sync_dbs:
+ repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL)
+ repo.servers = [server.replace("%s", sync_db) for server in servers]
+ t = h.init_transaction()
+ repo.update(False)
+ t.release()
+
+ for pkg in repo.pkgcache:
+ blacklist.add(pkg.name)
+ [blacklist.add(x) for x in pkg.replaces]
+
+db = mysql.connector.connect(host=aur_db_host, user=aur_db_user,
+ passwd=aur_db_pass, db=aur_db_name,
+ unix_socket=aur_db_socket, buffered=True)
+cur = db.cursor()
+
+cur.execute("SELECT Name FROM PackageBlacklist")
+oldblacklist = set([row[0] for row in cur.fetchall()])
+
+for pkg in blacklist.difference(oldblacklist):
+ cur.execute("INSERT INTO PackageBlacklist (Name) VALUES (%s)", [pkg])
+for pkg in oldblacklist.difference(blacklist):
+ cur.execute("DELETE FROM PackageBlacklist WHERE Name = %s", [pkg])
+
+db.commit()
+db.close()
diff --git a/scripts/aurblup/config.h.proto b/scripts/aurblup/config.h.proto
deleted file mode 100644
index 45acb87..0000000
--- a/scripts/aurblup/config.h.proto
+++ /dev/null
@@ -1,12 +0,0 @@
-/* libalpm options */
-#define ALPM_DBPATH "/var/lib/aurblup/"
-#define ALPM_MIRROR "ftp://mirrors.kernel.org/archlinux/%s/os/x86_64"
-
-static const char *alpm_repos[] = {
- "core",
- "community",
- "community-testing",
- "extra",
- "multilib",
- "testing"
-};
diff --git a/scripts/aurblup/config.mk b/scripts/aurblup/config.mk
deleted file mode 100644
index f8e12a8..0000000
--- a/scripts/aurblup/config.mk
+++ /dev/null
@@ -1,6 +0,0 @@
-PREFIX = /usr/local
-
-CFLAGS = -g -O2 -std=c99 -pedantic -Wall -I/usr/include/mysql
-LDFLAGS = -g -lalpm -lmysqlclient
-
-CC = cc
--
2.2.1
More information about the aur-dev
mailing list