vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

build rate limiting: use a rate

Browse source
This commit is contained in:
Jimmy Zelinskie 2016-12-06 13:59:47 -05:00
parent 7877c6ab94
commit 57770493fa
3 changed files with 18 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

8
config.py
View file

@ -400,6 +400,8 @@ class DefaultConfig(object):
# Location of the static marketing site. # Location of the static marketing site.
STATIC_SITE_BUCKET = None STATIC_SITE_BUCKET = None
# Maximum number of builds allowed to be queued per repository before rejecting requests. # Count and duration used to produce a rate of builds allowed to be queued per repository before
# Values less than zero allow queues of infinite size. # rejecting requests. Values less than zero disable rate limiting.
MAX_BUILD_QUEUE_SIZE = -1 # Example: 10 builds per minute is accomplished by setting ITEMS = 10, SECS = 60
MAX_BUILD_QUEUE_RATE_ITEMS = -1
MAX_BUILD_QUEUE_RATE_SECS = -1

6
data/queue.py
View file

@ -77,9 +77,11 @@ class WorkQueue(object):
._available_jobs(now, name_match_query) ._available_jobs(now, name_match_query)
.where(~(QueueItem.queue_name << running_query))) .where(~(QueueItem.queue_name << running_query)))
def num_available_jobs(self, prefix): def num_available_jobs(self, available_min_time, prefix):
prefix = prefix.lstrip('/') prefix = prefix.lstrip('/')
return self._available_jobs(datetime.utcnow(), self._name_match_query() + prefix).count() available = self._available_jobs(datetime.utcnow(),
self._name_match_query() + prefix)
return available.where(QueueItem.available_after >= available_min_time).count()
def _name_match_query(self): def _name_match_query(self):
return '%s%%' % self._canonical_name([self._queue_name] + self._canonical_name_match_list) return '%s%%' % self._canonical_name([self._queue_name] + self._canonical_name_match_list)

13
endpoints/building.py
View file

@ -1,6 +1,8 @@
import logging import logging
import json import json
from datetime import datetime, timedelta
from flask import request from flask import request
from app import app, dockerfile_build_queue from app import app, dockerfile_build_queue
@ -14,7 +16,8 @@ from util.morecollections import AttrDict
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
MAX_BUILD_QUEUE_SIZE = app.config.get('MAX_BUILD_QUEUE_SIZE', -1) MAX_BUILD_QUEUE_ITEMS = app.config.get('MAX_BUILD_QUEUE_ITEMS', -1)
MAX_BUILD_QUEUE_SECS = app.config.get('MAX_BUILD_QUEUE_SECS', -1)
class MaximumBuildsQueuedException(Exception): class MaximumBuildsQueuedException(Exception):
@ -23,9 +26,11 @@ class MaximumBuildsQueuedException(Exception):
def start_build(repository, prepared_build, pull_robot_name=None): def start_build(repository, prepared_build, pull_robot_name=None):
queue_item_prefix = '%s/%s' % repository.namespace_user.username, repository.name queue_item_prefix = '%s/%s' % repository.namespace_user.username, repository.name
available_builds = dockerfile_build_queue.num_available_jobs(queue_item_prefix) if MAX_BUILD_QUEUE_ITEMS > 0 and MAX_BUILD_QUEUE_SECS > 0:
if available_builds >= MAX_BUILD_QUEUE_SIZE and MAX_BUILD_QUEUE_SIZE > -1: available_min = datetime.utcnow() - timedelta(seconds=-MAX_BUILD_QUEUE_SECS)
raise MaximumBuildsQueuedException() available_builds = dockerfile_build_queue.num_available_jobs(available_min, queue_item_prefix)
if available_builds > MAX_BUILD_QUEUE_ITEMS:
raise MaximumBuildsQueuedException()
host = app.config['SERVER_HOSTNAME'] host = app.config['SERVER_HOSTNAME']
repo_path = '%s/%s/%s' % (host, repository.namespace_user.username, repository.name) repo_path = '%s/%s/%s' % (host, repository.namespace_user.username, repository.name)