add rate limiting to build queues

config.py

@@ -399,3 +399,7 @@ class DefaultConfig(object):
 
   # Location of the static marketing site.
   STATIC_SITE_BUCKET = None
+
+  # Maximum number of builds allowed to be queued per repository before rejecting requests.
+  # Values less than zero allow queues of infinite size.
+  MAX_BUILD_QUEUE_SIZE = -1

data/queue.py

@@ -77,6 +77,10 @@ class WorkQueue(object):
             ._available_jobs(now, name_match_query)
             .where(~(QueueItem.queue_name << running_query)))
 
+  def num_available_jobs(self, prefix):
+    prefix = prefix.lstrip('/')
+    return self._available_jobs(datetime.utcnow(), self._name_match_query() + prefix).count()
+
   def _name_match_query(self):
     return '%s%%' % self._canonical_name([self._queue_name] + self._canonical_name_match_list)
 

endpoints/api/build.py

@@ -14,9 +14,9 @@ from buildtrigger.basehandler import BuildTriggerHandler
 from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource,
                            require_repo_read, require_repo_write, validate_json_request,
                            ApiResource, internal_only, format_date, api, path_param,
-                           require_repo_admin)
+                           require_repo_admin, abort)
 from endpoints.exception import Unauthorized, NotFound, InvalidRequest
-from endpoints.building import start_build, PreparedBuild
+from endpoints.building import start_build, PreparedBuild, MaximumBuildsQueuedException
 from data import database
 from data import model
 from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission,
@@ -287,7 +287,10 @@ class RepositoryBuildList(RepositoryParamResource):
     prepared.is_manual = True
     prepared.metadata = {}
 
-    build_request = start_build(repo, prepared, pull_robot_name=pull_robot_name)
+    try:
+      build_request = start_build(repo, prepared, pull_robot_name=pull_robot_name)
+    except MaximumBuildsQueuedException:
+      abort(429, message='Maximum queued build rate exceeded.')
     resp = build_status_view(build_request)
     repo_string = '%s/%s' % (namespace, repository)
     headers = {

endpoints/api/trigger.py

@@ -15,10 +15,10 @@ from buildtrigger.triggerutil import (TriggerDeactivationException,
                                       RepositoryReadException, TriggerStartException)
 from endpoints.api import (RepositoryParamResource, nickname, resource, require_repo_admin,
                            log_action, request_error, query_param, parse_args, internal_only,
-                           validate_json_request, api, path_param)
+                           validate_json_request, api, path_param, abort)
 from endpoints.exception import NotFound, Unauthorized, InvalidRequest
 from endpoints.api.build import build_status_view, trigger_view, RepositoryBuildStatus
-from endpoints.building import start_build
+from endpoints.building import start_build, MaximumBuildsQueuedException
 from data import model
 from auth.permissions import (UserAdminPermission, AdministerOrganizationPermission,
                               ReadRepositoryPermission, AdministerRepositoryPermission)
@@ -436,6 +436,8 @@ class ActivateBuildTrigger(RepositoryParamResource):
       build_request = start_build(repo, prepared, pull_robot_name=pull_robot_name)
     except TriggerStartException as tse:
       raise InvalidRequest(tse.message)
+    except MaximumBuildsQueuedException:
+      abort(429, message='Maximum queued build rate exceeded.')
 
     resp = build_status_view(build_request)
     repo_string = '%s/%s' % (namespace_name, repo_name)

endpoints/building.py

@@ -11,10 +11,22 @@ from endpoints.notificationhelper import spawn_notification
 from util.names import escape_tag
 from util.morecollections import AttrDict
 
+
 logger = logging.getLogger(__name__)
 
+MAX_BUILD_QUEUE_SIZE = app.config.get('MAX_BUILD_QUEUE_SIZE', -1)
+
+
+class MaximumBuildsQueuedException(Exception):
+  pass
+
 
 def start_build(repository, prepared_build, pull_robot_name=None):
+  queue_item_prefix = '%s/%s' % repository.namespace_user.username, repository.name
+  available_builds = dockerfile_build_queue.num_available_jobs(queue_item_prefix)
+  if available_builds >= MAX_BUILD_QUEUE_SIZE and MAX_BUILD_QUEUE_SIZE > -1:
+    raise MaximumBuildsQueuedException()
+
   host = app.config['SERVER_HOSTNAME']
   repo_path = '%s/%s/%s' % (host, repository.namespace_user.username, repository.name)
 

endpoints/webhooks.py

@@ -10,9 +10,9 @@ from util.invoice import renderInvoiceToHtml
 from util.useremails import send_invoice_email, send_subscription_change, send_payment_failed
 from util.http import abort
 from buildtrigger.basehandler import BuildTriggerHandler
-from buildtrigger.triggerutil import (ValidationRequestException,  SkipRequestException,
+from buildtrigger.triggerutil import (ValidationRequestException, SkipRequestException,
                                       InvalidPayloadException)
-from endpoints.building import start_build
+from endpoints.building import start_build, MaximumBuildsQueuedException
 
 
 logger = logging.getLogger(__name__)
@@ -104,7 +104,10 @@ def build_trigger_webhook(trigger_uuid, **kwargs):
 
     pull_robot_name = model.build.get_pull_robot_name(trigger)
     repo = model.repository.get_repository(namespace, repository)
-    start_build(repo, prepared, pull_robot_name=pull_robot_name)
+    try:
+      start_build(repo, prepared, pull_robot_name=pull_robot_name)
+    except MaximumBuildsQueuedException:
+      abort(429)
 
     return make_response('Okay')