diff --git a/config.py b/config.py index 3b9eebfc8..ca86ec745 100644 --- a/config.py +++ b/config.py @@ -399,3 +399,7 @@ class DefaultConfig(object): # Location of the static marketing site. STATIC_SITE_BUCKET = None + + # Maximum number of builds allowed to be queued per repository before rejecting requests. + # Values less than zero allow queues of infinite size. + MAX_BUILD_QUEUE_SIZE = -1 diff --git a/data/queue.py b/data/queue.py index 421d7d6e7..059dd7d86 100644 --- a/data/queue.py +++ b/data/queue.py @@ -77,6 +77,10 @@ class WorkQueue(object): ._available_jobs(now, name_match_query) .where(~(QueueItem.queue_name << running_query))) + def num_available_jobs(self, prefix): + prefix = prefix.lstrip('/') + return self._available_jobs(datetime.utcnow(), self._name_match_query() + prefix).count() + def _name_match_query(self): return '%s%%' % self._canonical_name([self._queue_name] + self._canonical_name_match_list) diff --git a/endpoints/api/build.py b/endpoints/api/build.py index 7efedc993..36daf2bde 100644 --- a/endpoints/api/build.py +++ b/endpoints/api/build.py @@ -14,9 +14,9 @@ from buildtrigger.basehandler import BuildTriggerHandler from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource, require_repo_read, require_repo_write, validate_json_request, ApiResource, internal_only, format_date, api, path_param, - require_repo_admin) + require_repo_admin, abort) from endpoints.exception import Unauthorized, NotFound, InvalidRequest -from endpoints.building import start_build, PreparedBuild +from endpoints.building import start_build, PreparedBuild, MaximumBuildsQueuedException from data import database from data import model from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission, @@ -287,7 +287,10 @@ class RepositoryBuildList(RepositoryParamResource): prepared.is_manual = True prepared.metadata = {} - build_request = start_build(repo, prepared, pull_robot_name=pull_robot_name) + try: + build_request = start_build(repo, prepared, pull_robot_name=pull_robot_name) + except MaximumBuildsQueuedException: + abort(429, message='Maximum queued build rate exceeded.') resp = build_status_view(build_request) repo_string = '%s/%s' % (namespace, repository) headers = { diff --git a/endpoints/api/trigger.py b/endpoints/api/trigger.py index da9c98087..c03e9fbd9 100644 --- a/endpoints/api/trigger.py +++ b/endpoints/api/trigger.py @@ -15,10 +15,10 @@ from buildtrigger.triggerutil import (TriggerDeactivationException, RepositoryReadException, TriggerStartException) from endpoints.api import (RepositoryParamResource, nickname, resource, require_repo_admin, log_action, request_error, query_param, parse_args, internal_only, - validate_json_request, api, path_param) + validate_json_request, api, path_param, abort) from endpoints.exception import NotFound, Unauthorized, InvalidRequest from endpoints.api.build import build_status_view, trigger_view, RepositoryBuildStatus -from endpoints.building import start_build +from endpoints.building import start_build, MaximumBuildsQueuedException from data import model from auth.permissions import (UserAdminPermission, AdministerOrganizationPermission, ReadRepositoryPermission, AdministerRepositoryPermission) @@ -436,6 +436,8 @@ class ActivateBuildTrigger(RepositoryParamResource): build_request = start_build(repo, prepared, pull_robot_name=pull_robot_name) except TriggerStartException as tse: raise InvalidRequest(tse.message) + except MaximumBuildsQueuedException: + abort(429, message='Maximum queued build rate exceeded.') resp = build_status_view(build_request) repo_string = '%s/%s' % (namespace_name, repo_name) diff --git a/endpoints/building.py b/endpoints/building.py index 977a964a3..c4aa98e6c 100644 --- a/endpoints/building.py +++ b/endpoints/building.py @@ -11,10 +11,22 @@ from endpoints.notificationhelper import spawn_notification from util.names import escape_tag from util.morecollections import AttrDict + logger = logging.getLogger(__name__) +MAX_BUILD_QUEUE_SIZE = app.config.get('MAX_BUILD_QUEUE_SIZE', -1) + + +class MaximumBuildsQueuedException(Exception): + pass + def start_build(repository, prepared_build, pull_robot_name=None): + queue_item_prefix = '%s/%s' % repository.namespace_user.username, repository.name + available_builds = dockerfile_build_queue.num_available_jobs(queue_item_prefix) + if available_builds >= MAX_BUILD_QUEUE_SIZE and MAX_BUILD_QUEUE_SIZE > -1: + raise MaximumBuildsQueuedException() + host = app.config['SERVER_HOSTNAME'] repo_path = '%s/%s/%s' % (host, repository.namespace_user.username, repository.name) diff --git a/endpoints/webhooks.py b/endpoints/webhooks.py index a65036705..c89d9b26e 100644 --- a/endpoints/webhooks.py +++ b/endpoints/webhooks.py @@ -10,9 +10,9 @@ from util.invoice import renderInvoiceToHtml from util.useremails import send_invoice_email, send_subscription_change, send_payment_failed from util.http import abort from buildtrigger.basehandler import BuildTriggerHandler -from buildtrigger.triggerutil import (ValidationRequestException, SkipRequestException, +from buildtrigger.triggerutil import (ValidationRequestException, SkipRequestException, InvalidPayloadException) -from endpoints.building import start_build +from endpoints.building import start_build, MaximumBuildsQueuedException logger = logging.getLogger(__name__) @@ -104,7 +104,10 @@ def build_trigger_webhook(trigger_uuid, **kwargs): pull_robot_name = model.build.get_pull_robot_name(trigger) repo = model.repository.get_repository(namespace, repository) - start_build(repo, prepared, pull_robot_name=pull_robot_name) + try: + start_build(repo, prepared, pull_robot_name=pull_robot_name) + except MaximumBuildsQueuedException: + abort(429) return make_response('Okay')