- Aur-dev - lists.archlinux.org

[PATCH] aurweb.spawn: Integrate FastAPI and nginx
by Frédéric Mangano-Tarumi 03 Jun '20

03 Jun '20

aurweb.spawn used to launch only PHP’s built-in server. Now it spawns a dummy FastAPI application too. Since both stacks spawn their own HTTP server, aurweb.spawn also spawns nginx as a reverse proxy to mount them under the same base URL, defined by aur_location in the configuration. --- TESTING | 3 +- aurweb/asgi.py | 8 +++++ aurweb/spawn.py | 80 +++++++++++++++++++++++++++++++++++++------- conf/config.defaults | 7 ++++ 4 files changed, 85 insertions(+), 13 deletions(-) create mode 100644 aurweb/asgi.py diff --git a/TESTING b/TESTING index a5e08cb8..31e3bcbd 100644 --- a/TESTING +++ b/TESTING @@ -12,7 +12,8 @@ INSTALL. 2) Install the necessary packages: # pacman -S --needed php php-sqlite sqlite words fortune-mod \ - python python-sqlalchemy python-alembic + python python-sqlalchemy python-alembic \ + python-fastapi uvicorn nginx Ensure to enable the pdo_sqlite extension in php.ini. diff --git a/aurweb/asgi.py b/aurweb/asgi.py new file mode 100644 index 00000000..8d3deedc --- /dev/null +++ b/aurweb/asgi.py @@ -0,0 +1,8 @@ +from fastapi import FastAPI + +app = FastAPI() + + +(a)app.get("/sso/") +async def hello(): + return {"message": "Hello from FastAPI!"} diff --git a/aurweb/spawn.py b/aurweb/spawn.py index 5fa646b5..df6a2ed5 100644 --- a/aurweb/spawn.py +++ b/aurweb/spawn.py @@ -10,8 +10,10 @@ configuration anyway. import atexit import argparse +import os import subprocess import sys +import tempfile import time import urllib @@ -20,6 +22,7 @@ import aurweb.schema children = [] +temporary_dir = None verbosity = 0 @@ -35,10 +38,42 @@ class ProcessExceptions(Exception): super().__init__("\n- ".join(messages)) +def generate_nginx_config(): + """ + Generate an nginx configuration based on aurweb's configuration. + The file is generated under `temporary_dir`. + Returns the path to the created configuration file. + """ + aur_location = aurweb.config.get("options", "aur_location") + aur_location_parts = urllib.parse.urlsplit(aur_location) + config_path = os.path.join(temporary_dir, "nginx.conf") + config = open(config_path, "w") + # We double nginx's braces because they conflict with Python's f-strings. + config.write(f""" + events {{}} + daemon off; + error_log /dev/stderr info; + pid {os.path.join(temporary_dir, "nginx.pid")}; + http {{ + access_log /dev/stdout; + server {{ + listen {aur_location_parts.netloc}; + location / {{ + proxy_pass http://{aurweb.config.get("php", "bind_address")}; + }} + location /sso {{ + proxy_pass http://{aurweb.config.get("fastapi", "bind_address")}; + }} + }} + }} + """) + return config_path + + def spawn_child(args): """Open a subprocess and add it to the global state.""" if verbosity >= 1: - print(f"Spawning {args}", file=sys.stderr) + print(f":: Spawning {args}", file=sys.stderr) children.append(subprocess.Popen(args)) @@ -52,10 +87,29 @@ def start(): if children: return atexit.register(stop) - aur_location = aurweb.config.get("options", "aur_location") - aur_location_parts = urllib.parse.urlsplit(aur_location) - htmldir = aurweb.config.get("options", "htmldir") - spawn_child(["php", "-S", aur_location_parts.netloc, "-t", htmldir]) + + # Friendly message + ruler = "-" * os.get_terminal_size().columns + print(ruler) + print("Spawing PHP and FastAPI, then nginx as a reverse proxy.") + print("Check out " + aurweb.config.get("options", "aur_location")) + print("Hit ^C to terminate everything.") + print(ruler) + + # PHP + php_address = aurweb.config.get("php", "bind_address") + htmldir = aurweb.config.get("php", "htmldir") + spawn_child(["php", "-S", php_address, "-t", htmldir]) + + # FastAPI + host, port = aurweb.config.get("fastapi", "bind_address").rsplit(":", 1) + spawn_child(["python", "-m", "uvicorn", + "--host", host, + "--port", port, + "aurweb.asgi:app"]) + + # nginx + spawn_child(["nginx", "-p", temporary_dir, "-c", generate_nginx_config()]) def stop(): @@ -73,7 +127,7 @@ def stop(): try: p.terminate() if verbosity >= 1: - print(f"Sent SIGTERM to {p.args}", file=sys.stderr) + print(f":: Sent SIGTERM to {p.args}", file=sys.stderr) except Exception as e: exceptions.append(e) for p in children: @@ -99,9 +153,11 @@ if __name__ == '__main__': help='increase verbosity') args = parser.parse_args() verbosity = args.verbose - start() - try: - while True: - time.sleep(60) - except KeyboardInterrupt: - stop() + with tempfile.TemporaryDirectory(prefix="aurweb-") as tmpdirname: + temporary_dir = tmpdirname + start() + try: + while True: + time.sleep(60) + except KeyboardInterrupt: + stop() diff --git a/conf/config.defaults b/conf/config.defaults index 86fe765c..ed495168 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -41,9 +41,16 @@ cache = none cache_pkginfo_ttl = 86400 memcache_servers = 127.0.0.1:11211 +[php] +; Address PHP should bind when spawned in development mode by aurweb.spawn. +bind_address = 127.0.0.1:8081 ; Directory containing aurweb's PHP code, required by aurweb.spawn. ;htmldir = /path/to/web/html +[fastapi] +; Address uvicorn should bind when spawned in development mode by aurweb.spawn. +bind_address = 127.0.0.1:8082 + [ratelimit] request_limit = 4000 window_length = 86400 -- 2.26.2

3 12

[PATCH v2 1/5] refactor code to comply with flake8 and isort
by Filipe Laíns 02 Jun '20

02 Jun '20

Signed-off-by: Filipe Laíns <lains(a)archlinux.org> --- aurweb/git/auth.py | 3 +- aurweb/git/serve.py | 14 +- aurweb/git/update.py | 6 +- aurweb/initdb.py | 7 +- aurweb/l10n.py | 2 +- aurweb/schema.py | 4 +- aurweb/scripts/aurblup.py | 3 +- aurweb/scripts/rendercomment.py | 6 +- migrations/env.py | 9 +- schema/gendummydata.py | 354 ++++++++++++++++---------------- setup.py | 3 +- 11 files changed, 210 insertions(+), 201 deletions(-) diff --git a/aurweb/git/auth.py b/aurweb/git/auth.py index 3b1e485f..abecd276 100755 --- a/aurweb/git/auth.py +++ b/aurweb/git/auth.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 -import os -import shlex import re +import shlex import sys import aurweb.config diff --git a/aurweb/git/serve.py b/aurweb/git/serve.py index 64d51b9e..b91f1a13 100755 --- a/aurweb/git/serve.py +++ b/aurweb/git/serve.py @@ -175,11 +175,11 @@ def pkgbase_set_comaintainers(pkgbase, userlist, user, privileged): i += 1 for userid in uids_rem: - cur = conn.execute("DELETE FROM PackageComaintainers " + - "WHERE PackageBaseID = ? AND UsersID = ?", - [pkgbase_id, userid]) - subprocess.Popen((notify_cmd, 'comaintainer-remove', - str(userid), str(pkgbase_id))) + cur = conn.execute("DELETE FROM PackageComaintainers " + + "WHERE PackageBaseID = ? AND UsersID = ?", + [pkgbase_id, userid]) + subprocess.Popen((notify_cmd, 'comaintainer-remove', + str(userid), str(pkgbase_id))) conn.commit() conn.close() @@ -268,7 +268,7 @@ def pkgbase_disown(pkgbase, user, privileged): cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) userid = cur.fetchone()[0] if userid == 0: - raise aurweb.exceptions.InvalidUserException(user) + raise aurweb.exceptions.InvalidUserException(user) subprocess.Popen((notify_cmd, 'disown', str(userid), str(pkgbase_id))) @@ -472,7 +472,7 @@ def checkarg(cmdargv, *argdesc): checkarg_atmost(cmdargv, *argdesc) -def serve(action, cmdargv, user, privileged, remote_addr): +def serve(action, cmdargv, user, privileged, remote_addr): # noqa: C901 if enable_maintenance: if remote_addr not in maintenance_exc: raise aurweb.exceptions.MaintenanceException diff --git a/aurweb/git/update.py b/aurweb/git/update.py index 39128f8b..929b254e 100755 --- a/aurweb/git/update.py +++ b/aurweb/git/update.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 import os -import pygit2 import re import subprocess import sys import time +import pygit2 import srcinfo.parse import srcinfo.utils @@ -75,7 +75,7 @@ def create_pkgbase(conn, pkgbase, user): return pkgbase_id -def save_metadata(metadata, conn, user): +def save_metadata(metadata, conn, user): # noqa: C901 # Obtain package base ID and previous maintainer. pkgbase = metadata['pkgbase'] cur = conn.execute("SELECT ID, MaintainerUID FROM PackageBases " @@ -232,7 +232,7 @@ def die_commit(msg, commit): exit(1) -def main(): +def main(): # noqa: C901 repo = pygit2.Repository(repo_path) user = os.environ.get("AUR_USER") diff --git a/aurweb/initdb.py b/aurweb/initdb.py index 91777f7e..c8d0b2ae 100644 --- a/aurweb/initdb.py +++ b/aurweb/initdb.py @@ -1,11 +1,12 @@ -import aurweb.db -import aurweb.schema +import argparse import alembic.command import alembic.config -import argparse import sqlalchemy +import aurweb.db +import aurweb.schema + def feed_initial_data(conn): conn.execute(aurweb.schema.AccountTypes.insert(), [ diff --git a/aurweb/l10n.py b/aurweb/l10n.py index a7c0103e..492200b3 100644 --- a/aurweb/l10n.py +++ b/aurweb/l10n.py @@ -16,4 +16,4 @@ class Translator: self._localedir, languages=[lang]) self._translator[lang].install() - return _(s) + return _(s) # _ is not defined, what is this? # noqa: F821 diff --git a/aurweb/schema.py b/aurweb/schema.py index 6792cf1d..20f3e5ce 100644 --- a/aurweb/schema.py +++ b/aurweb/schema.py @@ -6,7 +6,7 @@ usually be automatically generated. See `migrations/README` for details. """ -from sqlalchemy import CHAR, Column, ForeignKey, Index, MetaData, String, TIMESTAMP, Table, Text, text +from sqlalchemy import CHAR, TIMESTAMP, Column, ForeignKey, Index, MetaData, String, Table, Text, text from sqlalchemy.dialects.mysql import BIGINT, DECIMAL, INTEGER, TINYINT from sqlalchemy.ext.compiler import compiles @@ -24,7 +24,7 @@ def compile_bigint_sqlite(type_, compiler, **kw): to INTEGER. Aside from that, BIGINT is the same as INTEGER for SQLite. See https://docs.sqlalchemy.org/en/13/dialects/sqlite.html#allowing-autoincreme… - """ + """ # noqa: E501 return 'INTEGER' diff --git a/aurweb/scripts/aurblup.py b/aurweb/scripts/aurblup.py index a7d43f12..e32937ce 100755 --- a/aurweb/scripts/aurblup.py +++ b/aurweb/scripts/aurblup.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 -import pyalpm import re +import pyalpm + import aurweb.config import aurweb.db diff --git a/aurweb/scripts/rendercomment.py b/aurweb/scripts/rendercomment.py index 76865d27..422dd33b 100755 --- a/aurweb/scripts/rendercomment.py +++ b/aurweb/scripts/rendercomment.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -import re -import pygit2 import sys + import bleach import markdown +import pygit2 import aurweb.config import aurweb.db @@ -47,7 +47,7 @@ class FlysprayLinksInlineProcessor(markdown.inlinepatterns.InlineProcessor): class FlysprayLinksExtension(markdown.extensions.Extension): def extendMarkdown(self, md, md_globals): - processor = FlysprayLinksInlineProcessor(r'\bFS#(\d+)\b',md) + processor = FlysprayLinksInlineProcessor(r'\bFS#(\d+)\b', md) md.inlinePatterns.register(processor, 'flyspray-links', 118) diff --git a/migrations/env.py b/migrations/env.py index 1627e693..c2ff58c1 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -1,10 +1,11 @@ -import aurweb.db -import aurweb.schema - -from alembic import context import logging.config + import sqlalchemy +from alembic import context + +import aurweb.db +import aurweb.schema # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/schema/gendummydata.py b/schema/gendummydata.py index 1f3d0476..b3a73ef2 100755 --- a/schema/gendummydata.py +++ b/schema/gendummydata.py @@ -10,33 +10,32 @@ usage: gendummydata.py outputfilename.sql # insert these users/packages into the AUR database. # import hashlib -import random -import time +import logging import os +import random import sys -import io -import logging +import time -LOG_LEVEL = logging.DEBUG # logging level. set to logging.INFO to reduce output +LOG_LEVEL = logging.DEBUG # logging level. set to logging.INFO to reduce output SEED_FILE = "/usr/share/dict/words" -DB_HOST = os.getenv("DB_HOST", "localhost") -DB_NAME = os.getenv("DB_NAME", "AUR") -DB_USER = os.getenv("DB_USER", "aur") -DB_PASS = os.getenv("DB_PASS", "aur") -USER_ID = 5 # Users.ID of first bogus user -PKG_ID = 1 # Packages.ID of first package +DB_HOST = os.getenv("DB_HOST", "localhost") +DB_NAME = os.getenv("DB_NAME", "AUR") +DB_USER = os.getenv("DB_USER", "aur") +DB_PASS = os.getenv("DB_PASS", "aur") +USER_ID = 5 # Users.ID of first bogus user +PKG_ID = 1 # Packages.ID of first package MAX_USERS = 300 # how many users to 'register' -MAX_DEVS = .1 # what percentage of MAX_USERS are Developers -MAX_TUS = .2 # what percentage of MAX_USERS are Trusted Users -MAX_PKGS = 900 # how many packages to load -PKG_DEPS = (1, 15) # min/max depends a package has -PKG_RELS = (1, 5) # min/max relations a package has -PKG_SRC = (1, 3) # min/max sources a package has +MAX_DEVS = .1 # what percentage of MAX_USERS are Developers +MAX_TUS = .2 # what percentage of MAX_USERS are Trusted Users +MAX_PKGS = 900 # how many packages to load +PKG_DEPS = (1, 15) # min/max depends a package has +PKG_RELS = (1, 5) # min/max relations a package has +PKG_SRC = (1, 3) # min/max sources a package has PKG_CMNTS = (1, 5) # min/max number of comments a package has CATEGORIES_COUNT = 17 # the number of categories from aur-schema -VOTING = (0, .30) # percentage range for package voting -OPEN_PROPOSALS = 5 # number of open trusted user proposals -CLOSE_PROPOSALS = 15 # number of closed trusted user proposals +VOTING = (0, .30) # percentage range for package voting +OPEN_PROPOSALS = 5 # number of open trusted user proposals +CLOSE_PROPOSALS = 15 # number of closed trusted user proposals RANDOM_TLDS = ("edu", "com", "org", "net", "tw", "ru", "pl", "de", "es") RANDOM_URL = ("http://www.", "ftp://ftp.", "http://", "ftp://") RANDOM_LOCS = ("pub", "release", "files", "downloads", "src") @@ -48,20 +47,20 @@ logging.basicConfig(format=logformat, level=LOG_LEVEL) log = logging.getLogger() if len(sys.argv) != 2: - log.error("Missing output filename argument") - raise SystemExit + log.error("Missing output filename argument") + raise SystemExit # make sure the seed file exists # if not os.path.exists(SEED_FILE): - log.error("Please install the 'words' Arch package") - raise SystemExit + log.error("Please install the 'words' Arch package") + raise SystemExit # make sure comments can be created # if not os.path.exists(FORTUNE_FILE): - log.error("Please install the 'fortune-mod' Arch package") - raise SystemExit + log.error("Please install the 'fortune-mod' Arch package") + raise SystemExit # track what users/package names have been used # @@ -69,21 +68,28 @@ seen_users = {} seen_pkgs = {} user_keys = [] + # some functions to generate random data # def genVersion(): - ver = [] - ver.append("%d" % random.randrange(0,10)) - ver.append("%d" % random.randrange(0,20)) - if random.randrange(0,2) == 0: - ver.append("%d" % random.randrange(0,100)) - return ".".join(ver) + "-%d" % random.randrange(1,11) + ver = [] + ver.append("%d" % random.randrange(0, 10)) + ver.append("%d" % random.randrange(0, 20)) + if random.randrange(0, 2) == 0: + ver.append("%d" % random.randrange(0, 100)) + return ".".join(ver) + "-%d" % random.randrange(1, 11) + + def genCategory(): - return random.randrange(1,CATEGORIES_COUNT) + return random.randrange(1, CATEGORIES_COUNT) + + def genUID(): - return seen_users[user_keys[random.randrange(0,len(user_keys))]] + return seen_users[user_keys[random.randrange(0, len(user_keys))]] + + def genFortune(): - return fortunes[random.randrange(0,len(fortunes))].replace("'", "") + return fortunes[random.randrange(0, len(fortunes))].replace("'", "") # load the words, and make sure there are enough words for users/pkgs @@ -93,25 +99,25 @@ fp = open(SEED_FILE, "r", encoding="utf-8") contents = fp.readlines() fp.close() if MAX_USERS > len(contents): - MAX_USERS = len(contents) + MAX_USERS = len(contents) if MAX_PKGS > len(contents): - MAX_PKGS = len(contents) + MAX_PKGS = len(contents) if len(contents) - MAX_USERS > MAX_PKGS: - need_dupes = 0 + need_dupes = 0 else: - need_dupes = 1 + need_dupes = 1 # select random usernames # log.debug("Generating random user names...") user_id = USER_ID while len(seen_users) < MAX_USERS: - user = random.randrange(0, len(contents)) - word = contents[user].replace("'", "").replace(".","").replace(" ", "_") - word = word.strip().lower() - if word not in seen_users: - seen_users[word] = user_id - user_id += 1 + user = random.randrange(0, len(contents)) + word = contents[user].replace("'", "").replace(".", "").replace(" ", "_") + word = word.strip().lower() + if word not in seen_users: + seen_users[word] = user_id + user_id += 1 user_keys = list(seen_users.keys()) # select random package names @@ -119,17 +125,17 @@ user_keys = list(seen_users.keys()) log.debug("Generating random package names...") num_pkgs = PKG_ID while len(seen_pkgs) < MAX_PKGS: - pkg = random.randrange(0, len(contents)) - word = contents[pkg].replace("'", "").replace(".","").replace(" ", "_") - word = word.strip().lower() - if not need_dupes: - if word not in seen_pkgs and word not in seen_users: - seen_pkgs[word] = num_pkgs - num_pkgs += 1 - else: - if word not in seen_pkgs: - seen_pkgs[word] = num_pkgs - num_pkgs += 1 + pkg = random.randrange(0, len(contents)) + word = contents[pkg].replace("'", "").replace(".", "").replace(" ", "_") + word = word.strip().lower() + if not need_dupes: + if word not in seen_pkgs and word not in seen_users: + seen_pkgs[word] = num_pkgs + num_pkgs += 1 + else: + if word not in seen_pkgs: + seen_pkgs[word] = num_pkgs + num_pkgs += 1 # free up contents memory # @@ -151,32 +157,32 @@ out.write("BEGIN;\n") # log.debug("Creating SQL statements for users.") for u in user_keys: - account_type = 1 # default to normal user - if not has_devs or not has_tus: - account_type = random.randrange(1, 4) - if account_type == 3 and not has_devs: - # this will be a dev account - # - developers.append(seen_users[u]) - if len(developers) >= MAX_DEVS * MAX_USERS: - has_devs = 1 - elif account_type == 2 and not has_tus: - # this will be a trusted user account - # - trustedusers.append(seen_users[u]) - if len(trustedusers) >= MAX_TUS * MAX_USERS: - has_tus = 1 - else: - # a normal user account - # - pass - - h = hashlib.new('md5') - h.update(u.encode()); - s = ("INSERT INTO Users (ID, AccountTypeID, Username, Email, Passwd)" - " VALUES (%d, %d, '%s', '%s(a)example.com', '%s');\n") - s = s % (seen_users[u], account_type, u, u, h.hexdigest()) - out.write(s) + account_type = 1 # default to normal user + if not has_devs or not has_tus: + account_type = random.randrange(1, 4) + if account_type == 3 and not has_devs: + # this will be a dev account + # + developers.append(seen_users[u]) + if len(developers) >= MAX_DEVS * MAX_USERS: + has_devs = 1 + elif account_type == 2 and not has_tus: + # this will be a trusted user account + # + trustedusers.append(seen_users[u]) + if len(trustedusers) >= MAX_TUS * MAX_USERS: + has_tus = 1 + else: + # a normal user account + # + pass + + h = hashlib.new('md5') + h.update(u.encode()) + s = ("INSERT INTO Users (ID, AccountTypeID, Username, Email, Passwd)" + " VALUES (%d, %d, '%s', '%s(a)example.com', '%s');\n") + s = s % (seen_users[u], account_type, u, u, h.hexdigest()) + out.write(s) log.debug("Number of developers: %d" % len(developers)) log.debug("Number of trusted users: %d" % len(trustedusers)) @@ -193,123 +199,123 @@ fp.close() log.debug("Creating SQL statements for packages.") count = 0 for p in list(seen_pkgs.keys()): - NOW = int(time.time()) - if count % 2 == 0: - muid = developers[random.randrange(0,len(developers))] - puid = developers[random.randrange(0,len(developers))] - else: - muid = trustedusers[random.randrange(0,len(trustedusers))] - puid = trustedusers[random.randrange(0,len(trustedusers))] - if count % 20 == 0: # every so often, there are orphans... - muid = "NULL" - - uuid = genUID() # the submitter/user - - s = ("INSERT INTO PackageBases (ID, Name, FlaggerComment, SubmittedTS, ModifiedTS, " + NOW = int(time.time()) + if count % 2 == 0: + muid = developers[random.randrange(0, len(developers))] + puid = developers[random.randrange(0, len(developers))] + else: + muid = trustedusers[random.randrange(0, len(trustedusers))] + puid = trustedusers[random.randrange(0, len(trustedusers))] + if count % 20 == 0: # every so often, there are orphans... + muid = "NULL" + + uuid = genUID() # the submitter/user + + s = ("INSERT INTO PackageBases (ID, Name, FlaggerComment, SubmittedTS, ModifiedTS, " "SubmitterUID, MaintainerUID, PackagerUID) VALUES (%d, '%s', '', %d, %d, %d, %s, %s);\n") - s = s % (seen_pkgs[p], p, NOW, NOW, uuid, muid, puid) - out.write(s) + s = s % (seen_pkgs[p], p, NOW, NOW, uuid, muid, puid) + out.write(s) - s = ("INSERT INTO Packages (ID, PackageBaseID, Name, Version) VALUES " + s = ("INSERT INTO Packages (ID, PackageBaseID, Name, Version) VALUES " "(%d, %d, '%s', '%s');\n") - s = s % (seen_pkgs[p], seen_pkgs[p], p, genVersion()) - out.write(s) + s = s % (seen_pkgs[p], seen_pkgs[p], p, genVersion()) + out.write(s) - count += 1 + count += 1 - # create random comments for this package - # - num_comments = random.randrange(PKG_CMNTS[0], PKG_CMNTS[1]) - for i in range(0, num_comments): - now = NOW + random.randrange(400, 86400*3) - s = ("INSERT INTO PackageComments (PackageBaseID, UsersID," - " Comments, RenderedComment, CommentTS) VALUES (%d, %d, '%s', '', %d);\n") - s = s % (seen_pkgs[p], genUID(), genFortune(), now) - out.write(s) + # create random comments for this package + # + num_comments = random.randrange(PKG_CMNTS[0], PKG_CMNTS[1]) + for i in range(0, num_comments): + now = NOW + random.randrange(400, 86400*3) + s = ("INSERT INTO PackageComments (PackageBaseID, UsersID," + " Comments, RenderedComment, CommentTS) VALUES (%d, %d, '%s', '', %d);\n") + s = s % (seen_pkgs[p], genUID(), genFortune(), now) + out.write(s) # Cast votes # track_votes = {} log.debug("Casting votes for packages.") for u in user_keys: - num_votes = random.randrange(int(len(seen_pkgs)*VOTING[0]), - int(len(seen_pkgs)*VOTING[1])) - pkgvote = {} - for v in range(num_votes): - pkg = random.randrange(1, len(seen_pkgs) + 1) - if pkg not in pkgvote: - s = ("INSERT INTO PackageVotes (UsersID, PackageBaseID)" - " VALUES (%d, %d);\n") - s = s % (seen_users[u], pkg) - pkgvote[pkg] = 1 - if pkg not in track_votes: - track_votes[pkg] = 0 - track_votes[pkg] += 1 - out.write(s) + num_votes = random.randrange(int(len(seen_pkgs)*VOTING[0]), + int(len(seen_pkgs)*VOTING[1])) + pkgvote = {} + for v in range(num_votes): + pkg = random.randrange(1, len(seen_pkgs) + 1) + if pkg not in pkgvote: + s = ("INSERT INTO PackageVotes (UsersID, PackageBaseID)" + " VALUES (%d, %d);\n") + s = s % (seen_users[u], pkg) + pkgvote[pkg] = 1 + if pkg not in track_votes: + track_votes[pkg] = 0 + track_votes[pkg] += 1 + out.write(s) # Update statements for package votes # for p in list(track_votes.keys()): - s = "UPDATE PackageBases SET NumVotes = %d WHERE ID = %d;\n" - s = s % (track_votes[p], p) - out.write(s) + s = "UPDATE PackageBases SET NumVotes = %d WHERE ID = %d;\n" + s = s % (track_votes[p], p) + out.write(s) # Create package dependencies and sources # log.debug("Creating statements for package depends/sources.") for p in list(seen_pkgs.keys()): - num_deps = random.randrange(PKG_DEPS[0], PKG_DEPS[1]) - for i in range(0, num_deps): - dep = random.choice([k for k in seen_pkgs]) - deptype = random.randrange(1, 5) - if deptype == 4: - dep += ": for " + random.choice([k for k in seen_pkgs]) - s = "INSERT INTO PackageDepends(PackageID, DepTypeID, DepName) VALUES (%d, %d, '%s');\n" - s = s % (seen_pkgs[p], deptype, dep) - out.write(s) - - num_rels = random.randrange(PKG_RELS[0], PKG_RELS[1]) - for i in range(0, num_deps): - rel = random.choice([k for k in seen_pkgs]) - reltype = random.randrange(1, 4) - s = "INSERT INTO PackageRelations(PackageID, RelTypeID, RelName) VALUES (%d, %d, '%s');\n" - s = s % (seen_pkgs[p], reltype, rel) - out.write(s) - - num_sources = random.randrange(PKG_SRC[0], PKG_SRC[1]) - for i in range(num_sources): - src_file = user_keys[random.randrange(0, len(user_keys))] - src = "%s%s.%s/%s/%s-%s.tar.gz" % ( - RANDOM_URL[random.randrange(0,len(RANDOM_URL))], - p, RANDOM_TLDS[random.randrange(0,len(RANDOM_TLDS))], - RANDOM_LOCS[random.randrange(0,len(RANDOM_LOCS))], - src_file, genVersion()) - s = "INSERT INTO PackageSources(PackageID, Source) VALUES (%d, '%s');\n" - s = s % (seen_pkgs[p], src) - out.write(s) + num_deps = random.randrange(PKG_DEPS[0], PKG_DEPS[1]) + for i in range(0, num_deps): + dep = random.choice([k for k in seen_pkgs]) + deptype = random.randrange(1, 5) + if deptype == 4: + dep += ": for " + random.choice([k for k in seen_pkgs]) + s = "INSERT INTO PackageDepends(PackageID, DepTypeID, DepName) VALUES (%d, %d, '%s');\n" + s = s % (seen_pkgs[p], deptype, dep) + out.write(s) + + num_rels = random.randrange(PKG_RELS[0], PKG_RELS[1]) + for i in range(0, num_deps): + rel = random.choice([k for k in seen_pkgs]) + reltype = random.randrange(1, 4) + s = "INSERT INTO PackageRelations(PackageID, RelTypeID, RelName) VALUES (%d, %d, '%s');\n" + s = s % (seen_pkgs[p], reltype, rel) + out.write(s) + + num_sources = random.randrange(PKG_SRC[0], PKG_SRC[1]) + for i in range(num_sources): + src_file = user_keys[random.randrange(0, len(user_keys))] + src = "%s%s.%s/%s/%s-%s.tar.gz" % ( + RANDOM_URL[random.randrange(0, len(RANDOM_URL))], + p, RANDOM_TLDS[random.randrange(0, len(RANDOM_TLDS))], + RANDOM_LOCS[random.randrange(0, len(RANDOM_LOCS))], + src_file, genVersion()) + s = "INSERT INTO PackageSources(PackageID, Source) VALUES (%d, '%s');\n" + s = s % (seen_pkgs[p], src) + out.write(s) # Create trusted user proposals # log.debug("Creating SQL statements for trusted user proposals.") -count=0 +count = 0 for t in range(0, OPEN_PROPOSALS+CLOSE_PROPOSALS): - now = int(time.time()) - if count < CLOSE_PROPOSALS: - start = now - random.randrange(3600*24*7, 3600*24*21) - end = now - random.randrange(0, 3600*24*7) - else: - start = now - end = now + random.randrange(3600*24, 3600*24*7) - if count % 5 == 0: # Don't make the vote about anyone once in a while - user = "" - else: - user = user_keys[random.randrange(0,len(user_keys))] - suid = trustedusers[random.randrange(0,len(trustedusers))] - s = ("INSERT INTO TU_VoteInfo (Agenda, User, Submitted, End," - " Quorum, SubmitterID) VALUES ('%s', '%s', %d, %d, 0.0, %d);\n") - s = s % (genFortune(), user, start, end, suid) - out.write(s) - count += 1 + now = int(time.time()) + if count < CLOSE_PROPOSALS: + start = now - random.randrange(3600*24*7, 3600*24*21) + end = now - random.randrange(0, 3600*24*7) + else: + start = now + end = now + random.randrange(3600*24, 3600*24*7) + if count % 5 == 0: # Don't make the vote about anyone once in a while + user = "" + else: + user = user_keys[random.randrange(0, len(user_keys))] + suid = trustedusers[random.randrange(0, len(trustedusers))] + s = ("INSERT INTO TU_VoteInfo (Agenda, User, Submitted, End," + " Quorum, SubmitterID) VALUES ('%s', '%s', %d, %d, 0.0, %d);\n") + s = s % (genFortune(), user, start, end, suid) + out.write(s) + count += 1 # close output file # diff --git a/setup.py b/setup.py index ca26f0d8..cf88488c 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,8 @@ import re -from setuptools import setup, find_packages import sys +from setuptools import find_packages, setup + version = None with open('web/lib/version.inc.php', 'r') as f: for line in f.readlines(): -- 2.26.2

2 5

[PATCH 1/3] flake8: add initial config
by Filipe Laíns 01 Jun '20

01 Jun '20

Signed-off-by: Filipe Laíns <lains(a)archlinux.org> --- aurweb/git/auth.py | 1 - aurweb/git/serve.py | 14 +- aurweb/git/update.py | 4 +- aurweb/l10n.py | 2 +- aurweb/schema.py | 2 +- aurweb/scripts/rendercomment.py | 3 +- .../versions/f47cad5d6d03_initial_revision.py | 4 +- schema/gendummydata.py | 349 +++++++++--------- setup.cfg | 4 + 9 files changed, 195 insertions(+), 188 deletions(-) create mode 100644 setup.cfg diff --git a/aurweb/git/auth.py b/aurweb/git/auth.py index 3b1e485f..f0e47ad2 100755 --- a/aurweb/git/auth.py +++ b/aurweb/git/auth.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import os import shlex import re import sys diff --git a/aurweb/git/serve.py b/aurweb/git/serve.py index 64d51b9e..b91f1a13 100755 --- a/aurweb/git/serve.py +++ b/aurweb/git/serve.py @@ -175,11 +175,11 @@ def pkgbase_set_comaintainers(pkgbase, userlist, user, privileged): i += 1 for userid in uids_rem: - cur = conn.execute("DELETE FROM PackageComaintainers " + - "WHERE PackageBaseID = ? AND UsersID = ?", - [pkgbase_id, userid]) - subprocess.Popen((notify_cmd, 'comaintainer-remove', - str(userid), str(pkgbase_id))) + cur = conn.execute("DELETE FROM PackageComaintainers " + + "WHERE PackageBaseID = ? AND UsersID = ?", + [pkgbase_id, userid]) + subprocess.Popen((notify_cmd, 'comaintainer-remove', + str(userid), str(pkgbase_id))) conn.commit() conn.close() @@ -268,7 +268,7 @@ def pkgbase_disown(pkgbase, user, privileged): cur = conn.execute("SELECT ID FROM Users WHERE Username = ?", [user]) userid = cur.fetchone()[0] if userid == 0: - raise aurweb.exceptions.InvalidUserException(user) + raise aurweb.exceptions.InvalidUserException(user) subprocess.Popen((notify_cmd, 'disown', str(userid), str(pkgbase_id))) @@ -472,7 +472,7 @@ def checkarg(cmdargv, *argdesc): checkarg_atmost(cmdargv, *argdesc) -def serve(action, cmdargv, user, privileged, remote_addr): +def serve(action, cmdargv, user, privileged, remote_addr): # noqa: C901 if enable_maintenance: if remote_addr not in maintenance_exc: raise aurweb.exceptions.MaintenanceException diff --git a/aurweb/git/update.py b/aurweb/git/update.py index 39128f8b..9f3c7944 100755 --- a/aurweb/git/update.py +++ b/aurweb/git/update.py @@ -75,7 +75,7 @@ def create_pkgbase(conn, pkgbase, user): return pkgbase_id -def save_metadata(metadata, conn, user): +def save_metadata(metadata, conn, user): # noqa: C901 # Obtain package base ID and previous maintainer. pkgbase = metadata['pkgbase'] cur = conn.execute("SELECT ID, MaintainerUID FROM PackageBases " @@ -232,7 +232,7 @@ def die_commit(msg, commit): exit(1) -def main(): +def main(): # noqa: C901 repo = pygit2.Repository(repo_path) user = os.environ.get("AUR_USER") diff --git a/aurweb/l10n.py b/aurweb/l10n.py index a7c0103e..492200b3 100644 --- a/aurweb/l10n.py +++ b/aurweb/l10n.py @@ -16,4 +16,4 @@ class Translator: self._localedir, languages=[lang]) self._translator[lang].install() - return _(s) + return _(s) # _ is not defined, what is this? # noqa: F821 diff --git a/aurweb/schema.py b/aurweb/schema.py index 6792cf1d..96c38e57 100644 --- a/aurweb/schema.py +++ b/aurweb/schema.py @@ -24,7 +24,7 @@ def compile_bigint_sqlite(type_, compiler, **kw): to INTEGER. Aside from that, BIGINT is the same as INTEGER for SQLite. See https://docs.sqlalchemy.org/en/13/dialects/sqlite.html#allowing-autoincreme… - """ + """ # noqa: E501 return 'INTEGER' diff --git a/aurweb/scripts/rendercomment.py b/aurweb/scripts/rendercomment.py index 76865d27..d3453c54 100755 --- a/aurweb/scripts/rendercomment.py +++ b/aurweb/scripts/rendercomment.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import re import pygit2 import sys import bleach @@ -47,7 +46,7 @@ class FlysprayLinksInlineProcessor(markdown.inlinepatterns.InlineProcessor): class FlysprayLinksExtension(markdown.extensions.Extension): def extendMarkdown(self, md, md_globals): - processor = FlysprayLinksInlineProcessor(r'\bFS#(\d+)\b',md) + processor = FlysprayLinksInlineProcessor(r'\bFS#(\d+)\b', md) md.inlinePatterns.register(processor, 'flyspray-links', 118) diff --git a/migrations/versions/f47cad5d6d03_initial_revision.py b/migrations/versions/f47cad5d6d03_initial_revision.py index 9e99490f..ec5ecb67 100644 --- a/migrations/versions/f47cad5d6d03_initial_revision.py +++ b/migrations/versions/f47cad5d6d03_initial_revision.py @@ -1,12 +1,10 @@ """initial revision Revision ID: f47cad5d6d03 -Revises: +Revises: Create Date: 2020-02-23 13:23:32.331396 """ -from alembic import op -import sqlalchemy as sa # revision identifiers, used by Alembic. diff --git a/schema/gendummydata.py b/schema/gendummydata.py index 1f3d0476..ec67c609 100755 --- a/schema/gendummydata.py +++ b/schema/gendummydata.py @@ -14,29 +14,29 @@ import random import time import os import sys -import io import logging -LOG_LEVEL = logging.DEBUG # logging level. set to logging.INFO to reduce output + +LOG_LEVEL = logging.DEBUG # logging level. set to logging.INFO to reduce output SEED_FILE = "/usr/share/dict/words" -DB_HOST = os.getenv("DB_HOST", "localhost") -DB_NAME = os.getenv("DB_NAME", "AUR") -DB_USER = os.getenv("DB_USER", "aur") -DB_PASS = os.getenv("DB_PASS", "aur") -USER_ID = 5 # Users.ID of first bogus user -PKG_ID = 1 # Packages.ID of first package +DB_HOST = os.getenv("DB_HOST", "localhost") +DB_NAME = os.getenv("DB_NAME", "AUR") +DB_USER = os.getenv("DB_USER", "aur") +DB_PASS = os.getenv("DB_PASS", "aur") +USER_ID = 5 # Users.ID of first bogus user +PKG_ID = 1 # Packages.ID of first package MAX_USERS = 300 # how many users to 'register' -MAX_DEVS = .1 # what percentage of MAX_USERS are Developers -MAX_TUS = .2 # what percentage of MAX_USERS are Trusted Users -MAX_PKGS = 900 # how many packages to load -PKG_DEPS = (1, 15) # min/max depends a package has -PKG_RELS = (1, 5) # min/max relations a package has -PKG_SRC = (1, 3) # min/max sources a package has +MAX_DEVS = .1 # what percentage of MAX_USERS are Developers +MAX_TUS = .2 # what percentage of MAX_USERS are Trusted Users +MAX_PKGS = 900 # how many packages to load +PKG_DEPS = (1, 15) # min/max depends a package has +PKG_RELS = (1, 5) # min/max relations a package has +PKG_SRC = (1, 3) # min/max sources a package has PKG_CMNTS = (1, 5) # min/max number of comments a package has CATEGORIES_COUNT = 17 # the number of categories from aur-schema -VOTING = (0, .30) # percentage range for package voting -OPEN_PROPOSALS = 5 # number of open trusted user proposals -CLOSE_PROPOSALS = 15 # number of closed trusted user proposals +VOTING = (0, .30) # percentage range for package voting +OPEN_PROPOSALS = 5 # number of open trusted user proposals +CLOSE_PROPOSALS = 15 # number of closed trusted user proposals RANDOM_TLDS = ("edu", "com", "org", "net", "tw", "ru", "pl", "de", "es") RANDOM_URL = ("http://www.", "ftp://ftp.", "http://", "ftp://") RANDOM_LOCS = ("pub", "release", "files", "downloads", "src") @@ -48,20 +48,20 @@ logging.basicConfig(format=logformat, level=LOG_LEVEL) log = logging.getLogger() if len(sys.argv) != 2: - log.error("Missing output filename argument") - raise SystemExit + log.error("Missing output filename argument") + raise SystemExit # make sure the seed file exists # if not os.path.exists(SEED_FILE): - log.error("Please install the 'words' Arch package") - raise SystemExit + log.error("Please install the 'words' Arch package") + raise SystemExit # make sure comments can be created # if not os.path.exists(FORTUNE_FILE): - log.error("Please install the 'fortune-mod' Arch package") - raise SystemExit + log.error("Please install the 'fortune-mod' Arch package") + raise SystemExit # track what users/package names have been used # @@ -69,21 +69,28 @@ seen_users = {} seen_pkgs = {} user_keys = [] + # some functions to generate random data # def genVersion(): - ver = [] - ver.append("%d" % random.randrange(0,10)) - ver.append("%d" % random.randrange(0,20)) - if random.randrange(0,2) == 0: - ver.append("%d" % random.randrange(0,100)) - return ".".join(ver) + "-%d" % random.randrange(1,11) + ver = [] + ver.append("%d" % random.randrange(0, 10)) + ver.append("%d" % random.randrange(0, 20)) + if random.randrange(0, 2) == 0: + ver.append("%d" % random.randrange(0, 100)) + return ".".join(ver) + "-%d" % random.randrange(1, 11) + + def genCategory(): - return random.randrange(1,CATEGORIES_COUNT) + return random.randrange(1, CATEGORIES_COUNT) + + def genUID(): - return seen_users[user_keys[random.randrange(0,len(user_keys))]] + return seen_users[user_keys[random.randrange(0, len(user_keys))]] + + def genFortune(): - return fortunes[random.randrange(0,len(fortunes))].replace("'", "") + return fortunes[random.randrange(0, len(fortunes))].replace("'", "") # load the words, and make sure there are enough words for users/pkgs @@ -93,25 +100,25 @@ fp = open(SEED_FILE, "r", encoding="utf-8") contents = fp.readlines() fp.close() if MAX_USERS > len(contents): - MAX_USERS = len(contents) + MAX_USERS = len(contents) if MAX_PKGS > len(contents): - MAX_PKGS = len(contents) + MAX_PKGS = len(contents) if len(contents) - MAX_USERS > MAX_PKGS: - need_dupes = 0 + need_dupes = 0 else: - need_dupes = 1 + need_dupes = 1 # select random usernames # log.debug("Generating random user names...") user_id = USER_ID while len(seen_users) < MAX_USERS: - user = random.randrange(0, len(contents)) - word = contents[user].replace("'", "").replace(".","").replace(" ", "_") - word = word.strip().lower() - if word not in seen_users: - seen_users[word] = user_id - user_id += 1 + user = random.randrange(0, len(contents)) + word = contents[user].replace("'", "").replace(".", "").replace(" ", "_") + word = word.strip().lower() + if word not in seen_users: + seen_users[word] = user_id + user_id += 1 user_keys = list(seen_users.keys()) # select random package names @@ -119,17 +126,17 @@ user_keys = list(seen_users.keys()) log.debug("Generating random package names...") num_pkgs = PKG_ID while len(seen_pkgs) < MAX_PKGS: - pkg = random.randrange(0, len(contents)) - word = contents[pkg].replace("'", "").replace(".","").replace(" ", "_") - word = word.strip().lower() - if not need_dupes: - if word not in seen_pkgs and word not in seen_users: - seen_pkgs[word] = num_pkgs - num_pkgs += 1 - else: - if word not in seen_pkgs: - seen_pkgs[word] = num_pkgs - num_pkgs += 1 + pkg = random.randrange(0, len(contents)) + word = contents[pkg].replace("'", "").replace(".", "").replace(" ", "_") + word = word.strip().lower() + if not need_dupes: + if word not in seen_pkgs and word not in seen_users: + seen_pkgs[word] = num_pkgs + num_pkgs += 1 + else: + if word not in seen_pkgs: + seen_pkgs[word] = num_pkgs + num_pkgs += 1 # free up contents memory # @@ -151,32 +158,32 @@ out.write("BEGIN;\n") # log.debug("Creating SQL statements for users.") for u in user_keys: - account_type = 1 # default to normal user - if not has_devs or not has_tus: - account_type = random.randrange(1, 4) - if account_type == 3 and not has_devs: - # this will be a dev account - # - developers.append(seen_users[u]) - if len(developers) >= MAX_DEVS * MAX_USERS: - has_devs = 1 - elif account_type == 2 and not has_tus: - # this will be a trusted user account - # - trustedusers.append(seen_users[u]) - if len(trustedusers) >= MAX_TUS * MAX_USERS: - has_tus = 1 - else: - # a normal user account - # - pass - - h = hashlib.new('md5') - h.update(u.encode()); - s = ("INSERT INTO Users (ID, AccountTypeID, Username, Email, Passwd)" - " VALUES (%d, %d, '%s', '%s(a)example.com', '%s');\n") - s = s % (seen_users[u], account_type, u, u, h.hexdigest()) - out.write(s) + account_type = 1 # default to normal user + if not has_devs or not has_tus: + account_type = random.randrange(1, 4) + if account_type == 3 and not has_devs: + # this will be a dev account + # + developers.append(seen_users[u]) + if len(developers) >= MAX_DEVS * MAX_USERS: + has_devs = 1 + elif account_type == 2 and not has_tus: + # this will be a trusted user account + # + trustedusers.append(seen_users[u]) + if len(trustedusers) >= MAX_TUS * MAX_USERS: + has_tus = 1 + else: + # a normal user account + # + pass + + h = hashlib.new('md5') + h.update(u.encode()) + s = ("INSERT INTO Users (ID, AccountTypeID, Username, Email, Passwd)" + " VALUES (%d, %d, '%s', '%s(a)example.com', '%s');\n") + s = s % (seen_users[u], account_type, u, u, h.hexdigest()) + out.write(s) log.debug("Number of developers: %d" % len(developers)) log.debug("Number of trusted users: %d" % len(trustedusers)) @@ -193,123 +200,123 @@ fp.close() log.debug("Creating SQL statements for packages.") count = 0 for p in list(seen_pkgs.keys()): - NOW = int(time.time()) - if count % 2 == 0: - muid = developers[random.randrange(0,len(developers))] - puid = developers[random.randrange(0,len(developers))] - else: - muid = trustedusers[random.randrange(0,len(trustedusers))] - puid = trustedusers[random.randrange(0,len(trustedusers))] - if count % 20 == 0: # every so often, there are orphans... - muid = "NULL" - - uuid = genUID() # the submitter/user - - s = ("INSERT INTO PackageBases (ID, Name, FlaggerComment, SubmittedTS, ModifiedTS, " + NOW = int(time.time()) + if count % 2 == 0: + muid = developers[random.randrange(0, len(developers))] + puid = developers[random.randrange(0, len(developers))] + else: + muid = trustedusers[random.randrange(0, len(trustedusers))] + puid = trustedusers[random.randrange(0, len(trustedusers))] + if count % 20 == 0: # every so often, there are orphans... + muid = "NULL" + + uuid = genUID() # the submitter/user + + s = ("INSERT INTO PackageBases (ID, Name, FlaggerComment, SubmittedTS, ModifiedTS, " "SubmitterUID, MaintainerUID, PackagerUID) VALUES (%d, '%s', '', %d, %d, %d, %s, %s);\n") - s = s % (seen_pkgs[p], p, NOW, NOW, uuid, muid, puid) - out.write(s) + s = s % (seen_pkgs[p], p, NOW, NOW, uuid, muid, puid) + out.write(s) - s = ("INSERT INTO Packages (ID, PackageBaseID, Name, Version) VALUES " + s = ("INSERT INTO Packages (ID, PackageBaseID, Name, Version) VALUES " "(%d, %d, '%s', '%s');\n") - s = s % (seen_pkgs[p], seen_pkgs[p], p, genVersion()) - out.write(s) + s = s % (seen_pkgs[p], seen_pkgs[p], p, genVersion()) + out.write(s) - count += 1 + count += 1 - # create random comments for this package - # - num_comments = random.randrange(PKG_CMNTS[0], PKG_CMNTS[1]) - for i in range(0, num_comments): - now = NOW + random.randrange(400, 86400*3) - s = ("INSERT INTO PackageComments (PackageBaseID, UsersID," - " Comments, RenderedComment, CommentTS) VALUES (%d, %d, '%s', '', %d);\n") - s = s % (seen_pkgs[p], genUID(), genFortune(), now) - out.write(s) + # create random comments for this package + # + num_comments = random.randrange(PKG_CMNTS[0], PKG_CMNTS[1]) + for i in range(0, num_comments): + now = NOW + random.randrange(400, 86400*3) + s = ("INSERT INTO PackageComments (PackageBaseID, UsersID," + " Comments, RenderedComment, CommentTS) VALUES (%d, %d, '%s', '', %d);\n") + s = s % (seen_pkgs[p], genUID(), genFortune(), now) + out.write(s) # Cast votes # track_votes = {} log.debug("Casting votes for packages.") for u in user_keys: - num_votes = random.randrange(int(len(seen_pkgs)*VOTING[0]), - int(len(seen_pkgs)*VOTING[1])) - pkgvote = {} - for v in range(num_votes): - pkg = random.randrange(1, len(seen_pkgs) + 1) - if pkg not in pkgvote: - s = ("INSERT INTO PackageVotes (UsersID, PackageBaseID)" - " VALUES (%d, %d);\n") - s = s % (seen_users[u], pkg) - pkgvote[pkg] = 1 - if pkg not in track_votes: - track_votes[pkg] = 0 - track_votes[pkg] += 1 - out.write(s) + num_votes = random.randrange(int(len(seen_pkgs)*VOTING[0]), + int(len(seen_pkgs)*VOTING[1])) + pkgvote = {} + for v in range(num_votes): + pkg = random.randrange(1, len(seen_pkgs) + 1) + if pkg not in pkgvote: + s = ("INSERT INTO PackageVotes (UsersID, PackageBaseID)" + " VALUES (%d, %d);\n") + s = s % (seen_users[u], pkg) + pkgvote[pkg] = 1 + if pkg not in track_votes: + track_votes[pkg] = 0 + track_votes[pkg] += 1 + out.write(s) # Update statements for package votes # for p in list(track_votes.keys()): - s = "UPDATE PackageBases SET NumVotes = %d WHERE ID = %d;\n" - s = s % (track_votes[p], p) - out.write(s) + s = "UPDATE PackageBases SET NumVotes = %d WHERE ID = %d;\n" + s = s % (track_votes[p], p) + out.write(s) # Create package dependencies and sources # log.debug("Creating statements for package depends/sources.") for p in list(seen_pkgs.keys()): - num_deps = random.randrange(PKG_DEPS[0], PKG_DEPS[1]) - for i in range(0, num_deps): - dep = random.choice([k for k in seen_pkgs]) - deptype = random.randrange(1, 5) - if deptype == 4: - dep += ": for " + random.choice([k for k in seen_pkgs]) - s = "INSERT INTO PackageDepends(PackageID, DepTypeID, DepName) VALUES (%d, %d, '%s');\n" - s = s % (seen_pkgs[p], deptype, dep) - out.write(s) - - num_rels = random.randrange(PKG_RELS[0], PKG_RELS[1]) - for i in range(0, num_deps): - rel = random.choice([k for k in seen_pkgs]) - reltype = random.randrange(1, 4) - s = "INSERT INTO PackageRelations(PackageID, RelTypeID, RelName) VALUES (%d, %d, '%s');\n" - s = s % (seen_pkgs[p], reltype, rel) - out.write(s) - - num_sources = random.randrange(PKG_SRC[0], PKG_SRC[1]) - for i in range(num_sources): - src_file = user_keys[random.randrange(0, len(user_keys))] - src = "%s%s.%s/%s/%s-%s.tar.gz" % ( - RANDOM_URL[random.randrange(0,len(RANDOM_URL))], - p, RANDOM_TLDS[random.randrange(0,len(RANDOM_TLDS))], - RANDOM_LOCS[random.randrange(0,len(RANDOM_LOCS))], - src_file, genVersion()) - s = "INSERT INTO PackageSources(PackageID, Source) VALUES (%d, '%s');\n" - s = s % (seen_pkgs[p], src) - out.write(s) + num_deps = random.randrange(PKG_DEPS[0], PKG_DEPS[1]) + for i in range(0, num_deps): + dep = random.choice([k for k in seen_pkgs]) + deptype = random.randrange(1, 5) + if deptype == 4: + dep += ": for " + random.choice([k for k in seen_pkgs]) + s = "INSERT INTO PackageDepends(PackageID, DepTypeID, DepName) VALUES (%d, %d, '%s');\n" + s = s % (seen_pkgs[p], deptype, dep) + out.write(s) + + num_rels = random.randrange(PKG_RELS[0], PKG_RELS[1]) + for i in range(0, num_deps): + rel = random.choice([k for k in seen_pkgs]) + reltype = random.randrange(1, 4) + s = "INSERT INTO PackageRelations(PackageID, RelTypeID, RelName) VALUES (%d, %d, '%s');\n" + s = s % (seen_pkgs[p], reltype, rel) + out.write(s) + + num_sources = random.randrange(PKG_SRC[0], PKG_SRC[1]) + for i in range(num_sources): + src_file = user_keys[random.randrange(0, len(user_keys))] + src = "%s%s.%s/%s/%s-%s.tar.gz" % ( + RANDOM_URL[random.randrange(0, len(RANDOM_URL))], + p, RANDOM_TLDS[random.randrange(0, len(RANDOM_TLDS))], + RANDOM_LOCS[random.randrange(0, len(RANDOM_LOCS))], + src_file, genVersion()) + s = "INSERT INTO PackageSources(PackageID, Source) VALUES (%d, '%s');\n" + s = s % (seen_pkgs[p], src) + out.write(s) # Create trusted user proposals # log.debug("Creating SQL statements for trusted user proposals.") -count=0 +count = 0 for t in range(0, OPEN_PROPOSALS+CLOSE_PROPOSALS): - now = int(time.time()) - if count < CLOSE_PROPOSALS: - start = now - random.randrange(3600*24*7, 3600*24*21) - end = now - random.randrange(0, 3600*24*7) - else: - start = now - end = now + random.randrange(3600*24, 3600*24*7) - if count % 5 == 0: # Don't make the vote about anyone once in a while - user = "" - else: - user = user_keys[random.randrange(0,len(user_keys))] - suid = trustedusers[random.randrange(0,len(trustedusers))] - s = ("INSERT INTO TU_VoteInfo (Agenda, User, Submitted, End," - " Quorum, SubmitterID) VALUES ('%s', '%s', %d, %d, 0.0, %d);\n") - s = s % (genFortune(), user, start, end, suid) - out.write(s) - count += 1 + now = int(time.time()) + if count < CLOSE_PROPOSALS: + start = now - random.randrange(3600*24*7, 3600*24*21) + end = now - random.randrange(0, 3600*24*7) + else: + start = now + end = now + random.randrange(3600*24, 3600*24*7) + if count % 5 == 0: # Don't make the vote about anyone once in a while + user = "" + else: + user = user_keys[random.randrange(0, len(user_keys))] + suid = trustedusers[random.randrange(0, len(trustedusers))] + s = ("INSERT INTO TU_VoteInfo (Agenda, User, Submitted, End," + " Quorum, SubmitterID) VALUES ('%s', '%s', %d, %d, 0.0, %d);\n") + s = s % (genFortune(), user, start, end, suid) + out.write(s) + count += 1 # close output file # diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..04f5b8ba --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[flake8] +max-line-length = 127 +max-complexity = 10 + -- 2.26.2

2 5

aurweb Git repository moved to GitLab
by Lukas Fleischer 25 May '20

25 May '20

The official aurweb Git repository has been moved to our new GitLab instance. The new upstream URL is https://gitlab.archlinux.org/archlinux/aurweb.git For the time being, we will accept aurweb bug reports and requests via the mailing list, Flyspray and the GitLab issue tracker. Patches can be submitted to the mailing list or via GitLab. Features that require a GitLab account (such as creating tickets on GitLab or filing merge requests) are currently only available to members of the Arch Linux staff. I will send another email when this changes. In the meantime, please feel free to update any references to the old Git repository. Best regards, Lukas

3 2

Re: Tech stack for Python aurweb
by Sven-Hendrik Haase 24 May '20

24 May '20

Hi guys, This is just a drive-by suggestion but as I've had my fair share of contact with Python web development, both professional and as a hobby, I thought I'd write something. I've worked with Flask for a long time and it really got my career started so it'll always live in a special place for me but I wouldn't use it for modern projects anymore. Some of the patterns used by Flask and the technologies surrounding it are pretty dated by modern standards (magical globals anyone?). Let me instead suggest you consider FastAPI [0]. I've created a few projects using it and I love it. It has the feel of Flask while being entirely modern and yet doesn't feel bloated at all. Here's my feature top list: - Great documentation (even for outside-of-happy-path cases) - Very fast thanks to async (but you can mix sync and async routes if you want to) - Fully typed (with type-based input validation and marshaling using Pydantic by the same author) which makes it possible to run mypy to good effect - Automatic OpenAPI-based API documentation generation based on the aforementioned types - Testing is very nice - Dependency injection is nicer than using Flask's globals (but don't overuse it!) It doesn't ship with templating but you can easily add it. [1] I'm unlikely to be the one implementing this so it's always your call and I'll try to be helpful whichever framework you go with. Cheers, Sven [0] https://fastapi.tiangolo.com/ [1] https://fastapi.tiangolo.com/advanced/templates/

1 0

[PATCH] First HTTP functional test of the RPC interface
by Frédéric Mangano-Tarumi 23 May '20

23 May '20

Though barely useful in its current state, it shows how to integrate pytest with SQLAlchemy and Werkzeug, providing a test framework for the potential future Flask port, while actually testing the current PHP implementation. --- aurweb/test/__init__.py | 0 aurweb/test/conftest.py | 51 +++++++++++++++++++++++++++++++++++++++++ aurweb/test/test_rpc.py | 21 +++++++++++++++++ aurweb/test/wsgihttp.py | 38 ++++++++++++++++++++++++++++++ test/README.md | 3 +++ test/rpc.t | 2 ++ 6 files changed, 115 insertions(+) create mode 100644 aurweb/test/__init__.py create mode 100644 aurweb/test/conftest.py create mode 100644 aurweb/test/test_rpc.py create mode 100644 aurweb/test/wsgihttp.py create mode 100755 test/rpc.t diff --git a/aurweb/test/__init__.py b/aurweb/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/aurweb/test/conftest.py b/aurweb/test/conftest.py new file mode 100644 index 00000000..49cc2f6e --- /dev/null +++ b/aurweb/test/conftest.py @@ -0,0 +1,51 @@ +""" +Fixtures for pytest. + +This module is automatically loaded by pytest. +""" + +import pytest +import sqlalchemy +import werkzeug.test +import werkzeug.wrappers +import werkzeug.wrappers.json + +import aurweb.config +import aurweb.db +from aurweb.test.wsgihttp import WsgiHttpProxy + + +class Response(werkzeug.wrappers.CommonResponseDescriptorsMixin, + werkzeug.wrappers.json.JSONMixin, + werkzeug.wrappers.BaseResponse): + """ + Custom response object to be returned by the test client. More mixins could + be added if need be. + + See https://werkzeug.palletsprojects.com/en/1.0.x/wrappers/#mixin-classes + """ + pass + + +(a)pytest.fixture +def client(): + """ + Build a Werkzeug test client for making HTTP requests to AUR. It requires + that the AUR test website is already running at `[options] aur_location`, + specified in the configuration file. + + When aurweb becomes a pure Flask application, this should return Flask’s + test_client(), which is a Werkzeug test client too. + https://flask.palletsprojects.com/en/1.1.x/testing/#the-testing-skeleton + """ + base_uri = aurweb.config.get("options", "aur_location") + proxy = WsgiHttpProxy(base_uri) + return werkzeug.test.Client(proxy, Response) + + +(a)pytest.fixture(scope="session") +def db_engine(): + """ + Return an SQLAlchemy engine to the configured database. + """ + return sqlalchemy.create_engine(aurweb.db.get_sqlalchemy_url()) diff --git a/aurweb/test/test_rpc.py b/aurweb/test/test_rpc.py new file mode 100644 index 00000000..7079145c --- /dev/null +++ b/aurweb/test/test_rpc.py @@ -0,0 +1,21 @@ +""" +Test suite for the RPC interface. + +See also `doc/rpc.txt` for the RPC interface documentation. +""" + +import pytest +from sqlalchemy.sql import select + +from aurweb.schema import Packages + + +def test_search_by_name(client, db_engine): + """Take a package from the database, and find it through the RPC interface.""" + with db_engine.connect() as conn: + pkg = conn.execute(select([Packages]).limit(1)).fetchone() + if pkg is None: + pytest.skip("needs at least one package in the database") + resp = client.get("/rpc/", query_string={"v": "5", "type": "search", "arg": pkg["Name"]}) + result = resp.json + assert result["resultcount"] >= 1 diff --git a/aurweb/test/wsgihttp.py b/aurweb/test/wsgihttp.py new file mode 100644 index 00000000..5b9d8040 --- /dev/null +++ b/aurweb/test/wsgihttp.py @@ -0,0 +1,38 @@ +import http.client +import urllib.parse + + +class WsgiHttpProxy: + """ + WSGI-to-HTTP proxy, that is to say a WSGI application that forwards every + WSGI request to an HTTP server, then the HTTP response back to WSGI. + + The base URL the constructor takes is something like + `http://localhost:8080`. It must not contain a path, a query string or a + fragment, as the proxy wouldn’t now what to do with it. + + Only the HTTP scheme is supported, but HTTPS could probably be easily added. + """ + + def __init__(self, base_url): + parts = urllib.parse.urlsplit(base_url) + self.netloc = parts.netloc + # Limitations of this dumb proxy + assert parts.scheme == "http" + assert parts.path in ("", "/") + assert parts.query == "" + assert parts.fragment == "" + + def __call__(self, environ, start_response): + conn = http.client.HTTPConnection(self.netloc) + conn.request( + method=environ["REQUEST_METHOD"], + url=urllib.parse.urlunsplit(( + "http", self.netloc, + urllib.parse.quote(environ["PATH_INFO"]), + environ["QUERY_STRING"], "")), + body=environ["wsgi.input"].read(int(environ.get("CONTENT_LENGTH", 0))), + ) + resp = conn.getresponse() + start_response(f"{resp.status} {resp.reason}", resp.getheaders()) + return resp diff --git a/test/README.md b/test/README.md index de7eff18..cc8baf33 100644 --- a/test/README.md +++ b/test/README.md @@ -20,8 +20,11 @@ For all the test to run, the following Arch packages should be installed: - python-bleach - python-markdown - python-pygit2 +- python-pytest +- python-pytest-tap - python-sqlalchemy - python-srcinfo +- python-werkzeug Writing tests ------------- diff --git a/test/rpc.t b/test/rpc.t new file mode 100755 index 00000000..f950f7df --- /dev/null +++ b/test/rpc.t @@ -0,0 +1,2 @@ +#!/bin/sh +pytest --tap-stream "$(dirname "$0")/../aurweb/test/test_rpc.py" -- 2.26.0

3 15

[PATCH] ci: remove Travis CI
by Filipe Laíns 23 May '20

23 May '20

We are are moving to Gitlab CI. Signed-off-by: Filipe Laíns <lains(a)archlinux.org> --- .travis.yml | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 5bbfda1f..00000000 --- a/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -language: python - -python: 3.6 - -addons: - apt: - packages: - - bsdtar - - libarchive-dev - - libgpgme11-dev - - libprotobuf-dev - -install: - - curl https://codeload.github.com/libgit2/libgit2/tar.gz/v0.26.0 | tar -xz - - curl https://sources.archlinux.org/other/pacman/pacman-5.0.2.tar.gz | tar -xz - - curl https://git.archlinux.org/pyalpm.git/snapshot/pyalpm-0.8.1.tar.gz | tar -xz - - ( cd libgit2-0.26.0 && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr && make && sudo make install ) - - ( cd pacman-5.0.2 && ./configure --prefix=/usr && make && sudo make install ) - - ( cd pyalpm-0.8.1 && python setup.py build && python setup.py install ) - - pip install mysql-connector-python-rf pygit2==0.26 srcinfo - - pip install bleach Markdown - -script: make -C test -- 2.26.2

2 1

aurweb testing environment using Docker
by Lukas Fleischer 18 May '20

18 May '20

We are currently working on a testing environment using Docker. The idea is to have something that is super easy to set up and still mirrors our production environment as closely as possible. It should ideally allow future contributors to test their patch submissions without having to go through the hassle of manually setting up a full development environment. Shout-out to Yaron (Cc) for sending me an initial draft. One of the big design decisions is whether everything should be in a single Docker container vs. several smaller atomic containers (i.e. separate containers for the HTTP server, the database, the SSH/Git interface, ...), as often done in production environments. The first approach has the benefit of simplicity from a user's perspective and being able to easily set up the environment using just Docker. The second approach has the benefit of atomicity (simplicity from a developer's perspective) and is closer to what we would do if we would actually use Docker for our production environment. It would probably require docker-compose (or another tool) in addition to Docker to provide the same level of convenience we can achieve with the other approach using just Docker, though. Since Yaron and me have conflicting opinions, I would like to hear some more arguments and views. Thanks! Lukas

4 7

[PATCH] Create aurweb.spawn for spawing the test server
by Frédéric Mangano-Tarumi 04 May '20

04 May '20

This program makes it easier for developers to spawn the PHP server since it fetches automatically what it needs from the configuration file, rather than having the user explicitly pass arguments to the php executable. When the setup gets more complicated as we introduce Python, aurweb.spawn will keep providing the same interface, while under the hood it is planned to support running multiple sub-processes. Its Python interface provides an way for the test suite to spawn the test server when it needs to perform HTTP requests to the test server. The current implementation is somewhat weak as it doesn’t detect when a child process dies, but this is not supposed to happen often, and it is only meant for aurweb developers. In the long term, aurweb.spawn will eventually become obsolete, and replaced by Docker or Flask’s tools. --- TESTING | 7 +-- aurweb/spawn.py | 107 +++++++++++++++++++++++++++++++++++++++++++ conf/config.defaults | 3 ++ 3 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 aurweb/spawn.py diff --git a/TESTING b/TESTING index 4a1e6f4c..a5e08cb8 100644 --- a/TESTING +++ b/TESTING @@ -17,7 +17,8 @@ INSTALL. Ensure to enable the pdo_sqlite extension in php.ini. 3) Copy conf/config.defaults to conf/config and adjust the configuration - (pay attention to disable_http_login, enable_maintenance and aur_location). + Pay attention to disable_http_login, enable_maintenance, aur_location and + htmldir. Be sure to change backend to sqlite and name to the file location of your created test database. @@ -31,6 +32,6 @@ INSTALL. $ ./gendummydata.py out.sql $ sqlite3 path/to/aurweb.sqlite3 < out.sql -5) Run the PHP built-in web server: +5) Run the test server: - $ AUR_CONFIG='/path/to/aurweb/conf/config' php -S localhost:8080 -t /path/to/aurweb/web/html + $ AUR_CONFIG='/path/to/aurweb/conf/config' python -m aurweb.spawn diff --git a/aurweb/spawn.py b/aurweb/spawn.py new file mode 100644 index 00000000..5fa646b5 --- /dev/null +++ b/aurweb/spawn.py @@ -0,0 +1,107 @@ +""" +Provide an automatic way of spawing an HTTP test server running aurweb. + +It can be called from the command-line or from another Python module. + +This module uses a global state, since you can’t open two servers with the same +configuration anyway. +""" + + +import atexit +import argparse +import subprocess +import sys +import time +import urllib + +import aurweb.config +import aurweb.schema + + +children = [] +verbosity = 0 + + +class ProcessExceptions(Exception): + """ + Compound exception used by stop() to list all the errors that happened when + terminating child processes. + """ + def __init__(self, message, exceptions): + self.message = message + self.exceptions = exceptions + messages = [message] + [str(e) for e in exceptions] + super().__init__("\n- ".join(messages)) + + +def spawn_child(args): + """Open a subprocess and add it to the global state.""" + if verbosity >= 1: + print(f"Spawning {args}", file=sys.stderr) + children.append(subprocess.Popen(args)) + + +def start(): + """ + Spawn the test server. If it is already running, do nothing. + + The server can be stopped with stop(), or is automatically stopped when the + Python process ends using atexit. + """ + if children: + return + atexit.register(stop) + aur_location = aurweb.config.get("options", "aur_location") + aur_location_parts = urllib.parse.urlsplit(aur_location) + htmldir = aurweb.config.get("options", "htmldir") + spawn_child(["php", "-S", aur_location_parts.netloc, "-t", htmldir]) + + +def stop(): + """ + Stop all the child processes. + + If an exception occurs during the process, the process continues anyway + because we don’t want to leave runaway processes around, and all the + exceptions are finally raised as a single ProcessExceptions. + """ + global children + atexit.unregister(stop) + exceptions = [] + for p in children: + try: + p.terminate() + if verbosity >= 1: + print(f"Sent SIGTERM to {p.args}", file=sys.stderr) + except Exception as e: + exceptions.append(e) + for p in children: + try: + rc = p.wait() + if rc != 0 and rc != -15: + # rc = -15 indicates the process was terminated with SIGTERM, + # which is to be expected since we called terminate on them. + raise Exception(f"Process {p.args} exited with {rc}") + except Exception as e: + exceptions.append(e) + children = [] + if exceptions: + raise ProcessExceptions("Errors terminating the child processes:", + exceptions) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + prog='python -m aurweb.spawn', + description='Start aurweb\'s test server.') + parser.add_argument('-v', '--verbose', action='count', default=0, + help='increase verbosity') + args = parser.parse_args() + verbosity = args.verbose + start() + try: + while True: + time.sleep(60) + except KeyboardInterrupt: + stop() diff --git a/conf/config.defaults b/conf/config.defaults index 447dacac..86fe765c 100644 --- a/conf/config.defaults +++ b/conf/config.defaults @@ -41,6 +41,9 @@ cache = none cache_pkginfo_ttl = 86400 memcache_servers = 127.0.0.1:11211 +; Directory containing aurweb's PHP code, required by aurweb.spawn. +;htmldir = /path/to/web/html + [ratelimit] request_limit = 4000 window_length = 86400 -- 2.26.0

2 1

Append latest commit link to package update notification
by Amin Vakil 01 May '20

01 May '20

I have emailed this 10 days ago in aur-general and as there hasn't been a response to it, I wondered maybe aur-general mailing list is the wrong place to open this issue, therefore: Arch Linux Wiki has said explicitly that they don't have any responsibilities about what a package in AUR does and they also say that it's mandatory to read and understand PKGBUILD (or any other file or patch that a installation requires) before installing any package. But when a package gets updated and those who have enabled notifications want to update the package, they should (I would say everyone must) click on package page, view changes, and click on latest commit to see what has changed. So appending latest commit link "helps" and "encourages" people to check latest update and I don't think in any possible way has a downside except for a little more text to email which I assume it's OK. Best Regards, Amin Vakil

3 2