[PATCH v2] Add rate limit support to API

Florian Pritz bluewind at xinu.at
Thu Feb 1 10:55:44 UTC 2018


This allows us to prevent users from hammering the API every few seconds
to check if any of their packages were updated. Real world users check
as often as every 5 or 10 seconds.

Signed-off-by: Florian Pritz <bluewind at xinu.at>
---

v2:
 - Fix column name scheme
 - Support sqlite
 - Simplify deletion of old limits
 - Put DDL SQL in schema
 - Allow to disable limit by setting to 0 in config

 conf/config.proto         |  4 +++
 schema/aur-schema.sql     | 10 ++++++
 upgrading/4.7.0.txt       | 11 ++++++
 web/lib/aurjson.class.php | 86 +++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 111 insertions(+)
 create mode 100644 upgrading/4.7.0.txt

diff --git a/conf/config.proto b/conf/config.proto
index 1750929..934d369 100644
--- a/conf/config.proto
+++ b/conf/config.proto
@@ -36,6 +36,10 @@ enable-maintenance = 1
 maintenance-exceptions = 127.0.0.1
 render-comment-cmd = /usr/local/bin/aurweb-rendercomment
 
+[ratelimit]
+request_limit = 4000
+window_length = 86400
+
 [notifications]
 notify-cmd = /usr/local/bin/aurweb-notify
 sendmail = /usr/bin/sendmail
diff --git a/schema/aur-schema.sql b/schema/aur-schema.sql
index 45272bb..79de3f2 100644
--- a/schema/aur-schema.sql
+++ b/schema/aur-schema.sql
@@ -399,3 +399,13 @@ CREATE TABLE AcceptedTerms (
 	FOREIGN KEY (UsersID) REFERENCES Users(ID) ON DELETE CASCADE,
 	FOREIGN KEY (TermsID) REFERENCES Terms(ID) ON DELETE CASCADE
 ) ENGINE = InnoDB;
+
+-- Rate limits for API
+--
+CREATE TABLE `ApiRateLimit` (
+  IP VARCHAR(45) NOT NULL,
+  Requests INT(11) NOT NULL,
+  WindowStart BIGINT(20) NOT NULL,
+  PRIMARY KEY (`ip`)
+) ENGINE = InnoDB;
+CREATE INDEX ApiRateLimitWindowStart ON ApiRateLimit (WindowStart);
diff --git a/upgrading/4.7.0.txt b/upgrading/4.7.0.txt
new file mode 100644
index 0000000..820e454
--- /dev/null
+++ b/upgrading/4.7.0.txt
@@ -0,0 +1,11 @@
+1. Add ApiRateLimit table:
+
+---
+CREATE TABLE `ApiRateLimit` (
+  IP VARCHAR(45) NOT NULL,
+  Requests INT(11) NOT NULL,
+  WindowStart BIGINT(20) NOT NULL,
+  PRIMARY KEY (`ip`)
+) ENGINE = InnoDB;
+CREATE INDEX ApiRateLimitWindowStart ON ApiRateLimit (WindowStart);
+---
diff --git a/web/lib/aurjson.class.php b/web/lib/aurjson.class.php
index 9eeaafd..b4cced0 100644
--- a/web/lib/aurjson.class.php
+++ b/web/lib/aurjson.class.php
@@ -96,6 +96,11 @@ public function handle($http_data) {
 
 		$this->dbh = DB::connect();
 
+		if ($this->check_ratelimit($_SERVER['REMOTE_ADDR'])) {
+			header("HTTP/1.1 429 Too Many Requests");
+			return $this->json_error('Rate limit reached');
+		}
+
 		$type = str_replace('-', '_', $http_data['type']);
 		if ($type == 'info' && $this->version >= 5) {
 			$type = 'multiinfo';
@@ -130,6 +135,87 @@ public function handle($http_data) {
 		}
 	}
 
+	/*
+	 * Check if an IP needs to be  rate limited.
+	 *
+	 * @param $ip IP of the current request
+	 *
+	 * @return true if IP needs to be rate limited, false otherwise.
+	 */
+	private function check_ratelimit($ip) {
+		$limit = config_get("ratelimit", "request_limit");
+		if ($limit == 0) {
+			return false;
+		}
+
+		$window_length = config_get("ratelimit", "window_length");
+		$this->update_ratelimit($ip);
+		$stmt = $this->dbh->prepare("
+			SELECT Requests FROM ApiRateLimit
+			WHERE IP = :ip");
+		$stmt->bindParam(":ip", $ip);
+		$result = $stmt->execute();
+
+		if (!$result) {
+			return false;
+		}
+
+		$row = $stmt->fetch(PDO::FETCH_ASSOC);
+		if ($row['Requests'] > $limit) {
+			return true;
+		}
+		return false;
+	}
+
+	/*
+	 * Update a rate limit for an IP by increasing it's requests value by one.
+	 *
+	 * @param $ip IP of the current request
+	 *
+	 * @return void
+	 */
+	private function update_ratelimit($ip) {
+		$window_length = config_get("ratelimit", "window_length");
+		$db_backend = config_get("database", "backend");
+		$time = time();
+
+		// Clean up old windows
+		$deletion_time = $time - $window_length;
+		$stmt = $this->dbh->prepare("
+			DELETE FROM ApiRateLimit
+			WHERE WindowStart < :time");
+		$stmt->bindParam(":time", $deletion_time);
+		$stmt->execute();
+
+		if ($db_backend == "mysql") {
+			$stmt = $this->dbh->prepare("
+				INSERT INTO ApiRateLimit
+				(IP, Requests, WindowStart)
+				VALUES (:ip, 1, :window_start)
+				ON DUPLICATE KEY UPDATE Requests=Requests+1");
+			$stmt->bindParam(":ip", $ip);
+			$stmt->bindParam(":window_start", $time);
+			$stmt->execute();
+		} elseif ($db_backend == "sqlite") {
+			$stmt = $this->dbh->prepare("
+				INSERT OR IGNORE INTO ApiRateLimit
+				(IP, Requests, WindowStart)
+				VALUES (:ip, 0, :window_start);");
+			$stmt->bindParam(":ip", $ip);
+			$stmt->bindParam(":window_start", $time);
+			$stmt->execute();
+
+			$stmt = $this->dbh->prepare("
+				UPDATE ApiRateLimit
+				SET Requests = Requests + 1
+				WHERE IP = :ip");
+			$stmt->bindParam(":ip", $ip);
+			$stmt->execute();
+		} else {
+			throw new RuntimeException("Unknown database backend");
+		}
+	}
+
 	/*
 	 * Returns a JSON formatted error string.
 	 *
-- 
2.16.1


More information about the aur-dev mailing list