vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #2892 from coreos-inc/joseph.schorr/QS-21/autodisable-triggers

Browse source 
Automatic disabling of failing build triggers
This commit is contained in:
josephschorr 2018-03-02 11:37:38 -05:00 committed by GitHub
commit dbe4258fc4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 571 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
buildman/server.py
View file

@ -14,7 +14,7 @@ from flask import Flask
from buildman.enums import BuildJobResult, BuildServerStatus, RESULT_PHASES
from buildman.jobutil.buildstatus import StatusHandler
from buildman.jobutil.buildjob import BuildJob, BuildJobLoadException
from data import database
from data import database, model
from app import app, metric_queue
logger = logging.getLogger(__name__)
@ -151,6 +151,11 @@ class BuilderServer(object):
else:
self._queue.complete(build_job.job_item)
# Update the trigger failure tracking (if applicable).
if build_job.repo_build.trigger is not None:
model.build.update_trigger_disable_status(build_job.repo_build.trigger,
RESULT_PHASES[job_status])
if update_phase:
status_handler = StatusHandler(self._build_logs, build_job.repo_build.uuid)
yield From(status_handler.set_phase(RESULT_PHASES[job_status]))

8
config.py
View file

@ -519,3 +519,11 @@ class DefaultConfig(ImmutableConfig):
'engine': 'memcached',
'endpoint': ('127.0.0.1', 18080),
}
# Defines the number of successive failures of a build trigger's build before the trigger is
# automatically disabled.
SUCCESSIVE_TRIGGER_FAILURE_DISABLE_THRESHOLD = 100
# Defines the number of successive internal errors of a build trigger's build before the
# trigger is automatically disabled.
SUCCESSIVE_TRIGGER_INTERNAL_ERROR_DISABLE_THRESHOLD = 5

16
data/database.py
View file

@ -694,6 +694,10 @@ class BuildTriggerService(BaseModel):
name = CharField(index=True, unique=True)
class DisableReason(BaseModel):
name = CharField(index=True, unique=True)
class RepositoryBuildTrigger(BaseModel):
uuid = CharField(default=uuid_generator)
service = ForeignKeyField(BuildTriggerService)
@ -705,6 +709,11 @@ class RepositoryBuildTrigger(BaseModel):
write_token = ForeignKeyField(AccessToken, null=True)
pull_robot = QuayUserField(allows_robots=True, null=True, related_name='triggerpullrobot',
robot_null_delete=True)
enabled = BooleanField(default=True)
disabled_reason = EnumField(DisableReason, null=True)
disabled_datetime = DateTimeField(default=datetime.utcnow, null=True, index=True)
successive_failure_count = IntegerField(default=0)
successive_internal_error_count = IntegerField(default=0)
class EmailConfirmation(BaseModel):
@ -883,6 +892,13 @@ class BUILD_PHASE(object):
phase == cls.CANCELLED)
class TRIGGER_DISABLE_REASON(object):
""" Build trigger disable reason enum """
BUILD_FALURES = 'successive_build_failures'
INTERNAL_ERRORS = 'successive_build_internal_errors'
USER_TOGGLED = 'user_toggled'
class QueueItem(BaseModel):
queue_name = CharField(index=True, max_length=1024)
body = TextField()

45
data/migrations/versions/17aff2e1354e_add_automatic_disable_of_build_triggers.py Normal file
View file

@ -0,0 +1,45 @@
"""Add automatic disable of build triggers
Revision ID: 17aff2e1354e
Revises: 61cadbacb9fc
Create Date: 2017-10-18 15:58:03.971526
"""
# revision identifiers, used by Alembic.
revision = '17aff2e1354e'
down_revision = '61cadbacb9fc'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
def upgrade(tables):
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('repositorybuildtrigger', sa.Column('successive_failure_count', sa.Integer(), nullable=False))
op.add_column('repositorybuildtrigger', sa.Column('successive_internal_error_count', sa.Integer(), nullable=False))
# ### end Alembic commands ###
op.bulk_insert(
tables.disablereason,
[
{'id': 2, 'name': 'successive_build_failures'},
{'id': 3, 'name': 'successive_build_internal_errors'},
],
)
def downgrade(tables):
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('repositorybuildtrigger', 'successive_internal_error_count')
op.drop_column('repositorybuildtrigger', 'successive_failure_count')
# ### end Alembic commands ###
op.execute(tables
.disablereason
.delete()
.where(tables.disablereason.c.name == op.inline_literal('successive_internal_error_count')))
op.execute(tables
.disablereason
.delete()
.where(tables.disablereason.c.name == op.inline_literal('successive_failure_count')))

57
data/migrations/versions/61cadbacb9fc_add_ability_for_build_triggers_to_be_.py Normal file
View file

@ -0,0 +1,57 @@
"""Add ability for build triggers to be disabled
Revision ID: 61cadbacb9fc
Revises: b4c2d45bc132
Create Date: 2017-10-18 12:07:26.190901
"""
# revision identifiers, used by Alembic.
revision = '61cadbacb9fc'
down_revision = 'b4c2d45bc132'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
from util.migrate import UTF8CharField
def upgrade(tables):
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('disablereason',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', UTF8CharField(length=255), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_disablereason'))
)
op.create_index('disablereason_name', 'disablereason', ['name'], unique=True)
op.bulk_insert(
tables.disablereason,
[
{'id': 1, 'name': 'user_toggled'},
],
)
op.bulk_insert(tables.logentrykind, [
{'name': 'toggle_repo_trigger'},
])
op.add_column(u'repositorybuildtrigger', sa.Column('disabled_reason_id', sa.Integer(), nullable=True))
op.add_column(u'repositorybuildtrigger', sa.Column('enabled', sa.Boolean(), nullable=False))
op.create_index('repositorybuildtrigger_disabled_reason_id', 'repositorybuildtrigger', ['disabled_reason_id'], unique=False)
op.create_foreign_key(op.f('fk_repositorybuildtrigger_disabled_reason_id_disablereason'), 'repositorybuildtrigger', 'disablereason', ['disabled_reason_id'], ['id'])
# ### end Alembic commands ###
def downgrade(tables):
# ### commands auto generated by Alembic - please adjust! ###
op.drop_constraint(op.f('fk_repositorybuildtrigger_disabled_reason_id_disablereason'), 'repositorybuildtrigger', type_='foreignkey')
op.drop_index('repositorybuildtrigger_disabled_reason_id', table_name='repositorybuildtrigger')
op.drop_column(u'repositorybuildtrigger', 'enabled')
op.drop_column(u'repositorybuildtrigger', 'disabled_reason_id')
op.drop_table('disablereason')
# ### end Alembic commands ###
op.execute(tables
.logentrykind
.delete()
.where(tables.logentrykind.c.name == op.inline_literal('toggle_repo_trigger')))

28
data/migrations/versions/87fbbc224f10_add_disabled_datetime_to_trigger.py Normal file
View file

@ -0,0 +1,28 @@
"""Add disabled datetime to trigger
Revision ID: 87fbbc224f10
Revises: 17aff2e1354e
Create Date: 2017-10-24 14:06:37.658705
"""
# revision identifiers, used by Alembic.
revision = '87fbbc224f10'
down_revision = '17aff2e1354e'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
def upgrade(tables):
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('repositorybuildtrigger', sa.Column('disabled_datetime', sa.DateTime(), nullable=True))
op.create_index('repositorybuildtrigger_disabled_datetime', 'repositorybuildtrigger', ['disabled_datetime'], unique=False)
# ### end Alembic commands ###
def downgrade(tables):
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index('repositorybuildtrigger_disabled_datetime', table_name='repositorybuildtrigger')
op.drop_column('repositorybuildtrigger', 'disabled_datetime')
# ### end Alembic commands ###

6
data/migrations/versions/a6c463dfb9fe_back_fill_build_expand_config.py
View file

@ -10,7 +10,8 @@ Create Date: 2017-03-17 10:00:19.739858
import json
import os
from data.database import RepositoryBuildTrigger
from peewee import *
from data.database import BaseModel
revision = 'a6c463dfb9fe'
down_revision = 'b4df55dea4b3'
@ -18,6 +19,9 @@ down_revision = 'b4df55dea4b3'
from alembic import op
class RepositoryBuildTrigger(BaseModel):
config = TextField(default='{}')
def upgrade(tables):
repostioryBuildTriggers = RepositoryBuildTrigger.select()
for repositoryBuildTrigger in repostioryBuildTriggers:

56
data/model/build.py
View file

@ -6,7 +6,8 @@ from peewee import JOIN_LEFT_OUTER
import features
from data.database import (BuildTriggerService, RepositoryBuildTrigger, Repository, Namespace, User,
RepositoryBuild, BUILD_PHASE, db_random_func, UseThenDisconnect)
RepositoryBuild, BUILD_PHASE, db_random_func, UseThenDisconnect,
DisableReason, TRIGGER_DISABLE_REASON)
from data.model import (InvalidBuildTriggerException, InvalidRepositoryBuildException,
db_transaction, user as user_model, config)
@ -255,3 +256,56 @@ def mark_build_archived(build_uuid):
.where(RepositoryBuild.uuid == build_uuid,
RepositoryBuild.logs_archived == False)
.execute()) > 0
def toggle_build_trigger(trigger, enabled, reason=TRIGGER_DISABLE_REASON.USER_TOGGLED):
""" Toggles the enabled status of a build trigger. """
trigger.enabled = enabled
if not enabled:
trigger.disabled_reason = RepositoryBuildTrigger.disabled_reason.get_id(reason)
trigger.disabled_datetime = datetime.utcnow()
trigger.save()
def update_trigger_disable_status(trigger, final_phase):
""" Updates the disable status of the given build trigger. If the build trigger had a
failure, then the counter is increased and, if we've reached the limit, the trigger is
automatically disabled. Otherwise, if the trigger succeeded, it's counter is reset. This
ensures that triggers that continue to error are eventually automatically disabled.
"""
with db_transaction():
try:
trigger = RepositoryBuildTrigger.get(id=trigger.id)
except RepositoryBuildTrigger.DoesNotExist:
# Already deleted.
return
# If the build completed successfully, then reset the successive counters.
if final_phase == BUILD_PHASE.COMPLETE:
trigger.successive_failure_count = 0
trigger.successive_internal_error_count = 0
trigger.save()
return
# Otherwise, increment the counters and check for trigger disable.
if final_phase == BUILD_PHASE.ERROR:
trigger.successive_failure_count = trigger.successive_failure_count + 1
trigger.successive_internal_error_count = 0
elif final_phase == BUILD_PHASE.INTERNAL_ERROR:
trigger.successive_internal_error_count = trigger.successive_internal_error_count + 1
# Check if we need to disable the trigger.
failure_threshold = config.app_config.get('SUCCESSIVE_TRIGGER_FAILURE_DISABLE_THRESHOLD')
error_threshold = config.app_config.get('SUCCESSIVE_TRIGGER_INTERNAL_ERROR_DISABLE_THRESHOLD')
if failure_threshold and trigger.successive_failure_count >= failure_threshold:
toggle_build_trigger(trigger, False, TRIGGER_DISABLE_REASON.BUILD_FALURES)
elif (error_threshold and
trigger.successive_internal_error_count >= error_threshold):
toggle_build_trigger(trigger, False, TRIGGER_DISABLE_REASON.INTERNAL_ERRORS)
else:
# Save the trigger changes.
trigger.save()

46
data/model/test/test_build.py Normal file
View file

@ -0,0 +1,46 @@
import pytest
from mock import patch
from data.database import BUILD_PHASE, RepositoryBuildTrigger
from data.model.build import update_trigger_disable_status
from test.fixtures import *
TEST_FAIL_THRESHOLD = 5
TEST_INTERNAL_ERROR_THRESHOLD = 2
@pytest.mark.parametrize('starting_failure_count, starting_error_count, status, expected_reason', [
(0, 0, BUILD_PHASE.COMPLETE, None),
(10, 10, BUILD_PHASE.COMPLETE, None),
(TEST_FAIL_THRESHOLD - 1, TEST_INTERNAL_ERROR_THRESHOLD - 1, BUILD_PHASE.COMPLETE, None),
(TEST_FAIL_THRESHOLD - 1, 0, BUILD_PHASE.ERROR, 'successive_build_failures'),
(0, TEST_INTERNAL_ERROR_THRESHOLD - 1, BUILD_PHASE.INTERNAL_ERROR,
'successive_build_internal_errors'),
])
def test_update_trigger_disable_status(starting_failure_count, starting_error_count, status,
expected_reason, initialized_db):
test_config = {
'SUCCESSIVE_TRIGGER_FAILURE_DISABLE_THRESHOLD': TEST_FAIL_THRESHOLD,
'SUCCESSIVE_TRIGGER_INTERNAL_ERROR_DISABLE_THRESHOLD': TEST_INTERNAL_ERROR_THRESHOLD,
}
trigger = model.build.list_build_triggers('devtable', 'building')[0]
trigger.successive_failure_count = starting_failure_count
trigger.successive_internal_error_count = starting_error_count
trigger.enabled = True
trigger.save()
with patch('data.model.config.app_config', test_config):
update_trigger_disable_status(trigger, status)
updated_trigger = RepositoryBuildTrigger.get(uuid=trigger.uuid)
assert updated_trigger.enabled == (expected_reason is None)
if expected_reason is not None:
assert updated_trigger.disabled_reason.name == expected_reason
else:
assert updated_trigger.disabled_reason is None
assert updated_trigger.successive_failure_count == 0
assert updated_trigger.successive_internal_error_count == 0

7
endpoints/api/build.py
View file

@ -22,7 +22,8 @@ from endpoints.api import (RepositoryParamResource, parse_args, query_param, nic
require_repo_read, require_repo_write, validate_json_request,
ApiResource, internal_only, format_date, api, path_param,
require_repo_admin, abort, disallow_for_app_repositories)
from endpoints.building import start_build, PreparedBuild, MaximumBuildsQueuedException
from endpoints.building import (start_build, PreparedBuild, MaximumBuildsQueuedException,
BuildTriggerDisabledException)
from endpoints.exception import Unauthorized, NotFound, InvalidRequest
from util.names import parse_robot_username
@ -69,6 +70,8 @@ def trigger_view(trigger, can_read=False, can_admin=False, for_build=False):
'config': build_trigger.config if can_admin else {},
'can_invoke': can_admin,
'enabled': trigger.enabled,
'disabled_reason': trigger.disabled_reason.name if trigger.disabled_reason else None,
}
if not for_build and can_admin and trigger.pull_robot:
@ -309,6 +312,8 @@ class RepositoryBuildList(RepositoryParamResource):
build_request = start_build(repo, prepared, pull_robot_name=pull_robot_name)
except MaximumBuildsQueuedException:
abort(429, message='Maximum queued build rate exceeded.')
except BuildTriggerDisabledException:
abort(400, message='Build trigger is disabled')
resp = build_status_view(build_request)
repo_string = '%s/%s' % (namespace, repository)

19
endpoints/api/test/test_security.py
View file

@ -14,6 +14,7 @@ from endpoints.api.search import ConductRepositorySearch
from endpoints.api.superuser import SuperUserRepositoryBuildLogs, SuperUserRepositoryBuildResource
from endpoints.api.superuser import SuperUserRepositoryBuildStatus
from endpoints.api.appspecifictokens import AppTokens, AppToken
from endpoints.api.trigger import BuildTrigger
from endpoints.test.shared import client_with_identity, toggle_feature
from test.fixtures import *
@ -24,6 +25,7 @@ REPO_PARAMS = {'repository': 'devtable/someapp'}
SEARCH_PARAMS = {'query': ''}
NOTIFICATION_PARAMS = {'namespace': 'devtable', 'repository': 'devtable/simple', 'uuid': 'some uuid'}
TOKEN_PARAMS = {'token_uuid': 'someuuid'}
TRIGGER_PARAMS = {'repository': 'devtable/simple', 'trigger_uuid': 'someuuid'}
@pytest.mark.parametrize('resource,method,params,body,identity,expected', [
(AppTokens, 'GET', {}, {}, None, 401),
@ -89,7 +91,22 @@ TOKEN_PARAMS = {'token_uuid': 'someuuid'}
(RepositoryTrust, 'POST', REPO_PARAMS, {'trust_enabled': True}, 'freshuser', 403),
(RepositoryTrust, 'POST', REPO_PARAMS, {'trust_enabled': True}, 'reader', 403),
(RepositoryTrust, 'POST', REPO_PARAMS, {'trust_enabled': True}, 'devtable', 404),
(BuildTrigger, 'GET', TRIGGER_PARAMS, {}, None, 401),
(BuildTrigger, 'GET', TRIGGER_PARAMS, {}, 'freshuser', 403),
(BuildTrigger, 'GET', TRIGGER_PARAMS, {}, 'reader', 403),
(BuildTrigger, 'GET', TRIGGER_PARAMS, {}, 'devtable', 404),
(BuildTrigger, 'DELETE', TRIGGER_PARAMS, {}, None, 403),
(BuildTrigger, 'DELETE', TRIGGER_PARAMS, {}, 'freshuser', 403),
(BuildTrigger, 'DELETE', TRIGGER_PARAMS, {}, 'reader', 403),
(BuildTrigger, 'DELETE', TRIGGER_PARAMS, {}, 'devtable', 404),
(BuildTrigger, 'PUT', TRIGGER_PARAMS, {}, None, 403),
(BuildTrigger, 'PUT', TRIGGER_PARAMS, {}, 'freshuser', 403),
(BuildTrigger, 'PUT', TRIGGER_PARAMS, {}, 'reader', 403),
(BuildTrigger, 'PUT', TRIGGER_PARAMS, {}, 'devtable', 400),
(RepositoryUserTransitivePermission, 'GET', {'username': 'A2O9','repository': 'public/publicrepo'}, None, None, 401),
(RepositoryUserTransitivePermission, 'GET', {'username': 'A2O9','repository': 'public/publicrepo'}, None, 'freshuser', 403),
(RepositoryUserTransitivePermission, 'GET', {'username': 'A2O9','repository': 'public/publicrepo'}, None, 'reader', 403),

33
endpoints/api/test/test_trigger.py
View file

@ -1,6 +1,12 @@
import pytest
import json
from data import model
from endpoints.api.trigger_analyzer import is_parent
from endpoints.api.trigger import BuildTrigger
from endpoints.api.test.shared import conduct_api_call
from endpoints.test.shared import client_with_identity
from test.fixtures import *
@pytest.mark.parametrize('context,dockerfile_path,expected', [
@ -20,3 +26,30 @@ from endpoints.api.trigger_analyzer import is_parent
])
def test_super_user_build_endpoints(context, dockerfile_path, expected):
assert is_parent(context, dockerfile_path) == expected
def test_enabled_disabled_trigger(app, client):
trigger = model.build.list_build_triggers('devtable', 'building')[0]
trigger.config = json.dumps({'hook_id': 'someid'})
trigger.save()
params = {
'repository': 'devtable/building',
'trigger_uuid': trigger.uuid,
}
body = {
'enabled': False,
}
with client_with_identity('devtable', client) as cl:
result = conduct_api_call(cl, BuildTrigger, 'PUT', params, body, 200).json
assert not result['enabled']
body = {
'enabled': True,
}
with client_with_identity('devtable', client) as cl:
result = conduct_api_call(cl, BuildTrigger, 'PUT', params, body, 200).json
assert result['enabled']

44
endpoints/api/trigger.py
View file

@ -22,7 +22,8 @@ from endpoints.api import (RepositoryParamResource, nickname, resource, require_
disallow_for_app_repositories)
from endpoints.api.build import build_status_view, trigger_view, RepositoryBuildStatus
from endpoints.api.trigger_analyzer import TriggerAnalyzer
from endpoints.building import start_build, MaximumBuildsQueuedException
from endpoints.building import (start_build, MaximumBuildsQueuedException,
BuildTriggerDisabledException)
from endpoints.exception import NotFound, Unauthorized, InvalidRequest
from util.names import parse_robot_username
@ -62,6 +63,21 @@ class BuildTriggerList(RepositoryParamResource):
@path_param('trigger_uuid', 'The UUID of the build trigger')
class BuildTrigger(RepositoryParamResource):
""" Resource for managing specific build triggers. """
schemas = {
'UpdateTrigger': {
'type': 'object',
'description': 'Options for updating a build trigger',
'required': [
'enabled',
],
'properties': {
'enabled': {
'type': 'boolean',
'description': 'Whether the build trigger is enabled',
},
}
},
}
@require_repo_admin
@disallow_for_app_repositories
@ -70,6 +86,27 @@ class BuildTrigger(RepositoryParamResource):
""" Get information for the specified build trigger. """
return trigger_view(get_trigger(trigger_uuid), can_admin=True)
@require_repo_admin
@disallow_for_app_repositories
@nickname('updateBuildTrigger')
@validate_json_request('UpdateTrigger')
def put(self, namespace_name, repo_name, trigger_uuid):
""" Updates the specified build trigger. """
trigger = get_trigger(trigger_uuid)
handler = BuildTriggerHandler.get_handler(trigger)
if not handler.is_active():
raise InvalidRequest('Cannot update an unactivated trigger')
enable = request.get_json()['enabled']
model.build.toggle_build_trigger(trigger, enable)
log_action('toggle_repo_trigger', namespace_name,
{'repo': repo_name, 'trigger_id': trigger_uuid,
'service': trigger.service.name, 'enabled': enable},
repo=model.repository.get_repository(namespace_name, repo_name))
return trigger_view(trigger)
@require_repo_admin
@disallow_for_app_repositories
@nickname('deleteBuildTrigger')
@ -340,6 +377,8 @@ class ActivateBuildTrigger(RepositoryParamResource):
def post(self, namespace_name, repo_name, trigger_uuid):
""" Manually start a build from the specified trigger. """
trigger = get_trigger(trigger_uuid)
if not trigger.enabled:
raise InvalidRequest('Trigger is not enabled.')
handler = BuildTriggerHandler.get_handler(trigger)
if not handler.is_active():
@ -356,6 +395,8 @@ class ActivateBuildTrigger(RepositoryParamResource):
raise InvalidRequest(tse.message)
except MaximumBuildsQueuedException:
abort(429, message='Maximum queued build rate exceeded.')
except BuildTriggerDisabledException:
abort(400, message='Build trigger is disabled')
resp = build_status_view(build_request)
repo_string = '%s/%s' % (namespace_name, repo_name)
@ -485,3 +526,4 @@ class BuildTriggerSourceNamespaces(RepositoryParamResource):
raise InvalidRequest(rre.message)
else:
raise Unauthorized()

12
endpoints/building.py
View file

@ -25,10 +25,22 @@ class MaximumBuildsQueuedException(Exception):
pass
class BuildTriggerDisabledException(Exception):
"""
This exception is raised when a build is required, but the build trigger has been disabled.
"""
pass
def start_build(repository, prepared_build, pull_robot_name=None):
# Ensure that builds are only run in image repositories.
if repository.kind.name != 'image':
raise Exception('Attempt to start a build for application repository %s' % repository.id)
# Ensure that disabled triggers are not run.
if prepared_build.trigger is not None and not prepared_build.trigger.enabled:
raise BuildTriggerDisabledException
if repository.namespace_user.maximum_queued_builds_count is not None:
queue_item_canonical_name = [repository.namespace_user.username]
alive_builds = dockerfile_build_queue.num_alive_jobs(queue_item_canonical_name)

14
endpoints/test/test_building.py
View file

@ -1,7 +1,8 @@
import pytest
from data import model
from endpoints.building import start_build, PreparedBuild, MaximumBuildsQueuedException
from endpoints.building import (start_build, PreparedBuild, MaximumBuildsQueuedException,
BuildTriggerDisabledException)
from test.fixtures import *
@ -29,3 +30,14 @@ def test_maximum_builds(app):
# Try to queue a second build; should fail.
with pytest.raises(MaximumBuildsQueuedException):
start_build(repo, prepared_build)
def test_start_build_disabled_trigger(app):
trigger = model.build.list_build_triggers('devtable', 'building')[0]
trigger.enabled = False
trigger.save()
build = PreparedBuild(trigger=trigger)
with pytest.raises(BuildTriggerDisabledException):
start_build(trigger.repository, build)

24
endpoints/test/test_webhooks.py Normal file
View file

@ -0,0 +1,24 @@
import base64
import pytest
from flask import url_for
from data import model
from endpoints.test.shared import conduct_call
from test.fixtures import *
def test_start_build_disabled_trigger(app, client):
trigger = model.build.list_build_triggers('devtable', 'building')[0]
trigger.enabled = False
trigger.save()
params = {
'trigger_uuid': trigger.uuid,
}
headers = {
'Authorization': 'Basic ' + base64.b64encode('devtable:password'),
}
conduct_call(client, 'webhooks.build_trigger_webhook', url_for, 'POST', params, None, 400,
headers=headers)

10
endpoints/webhooks.py
View file

@ -12,7 +12,8 @@ from util.http import abort
from buildtrigger.basehandler import BuildTriggerHandler
from buildtrigger.triggerutil import (ValidationRequestException, SkipRequestException,
InvalidPayloadException)
from endpoints.building import start_build, MaximumBuildsQueuedException
from endpoints.building import (start_build, MaximumBuildsQueuedException,
BuildTriggerDisabledException)
logger = logging.getLogger(__name__)
@ -91,9 +92,7 @@ def build_trigger_webhook(trigger_uuid, **kwargs):
namespace = trigger.repository.namespace_user.username
repository = trigger.repository.name
permission = ModifyRepositoryPermission(namespace, repository)
if permission.can():
if ModifyRepositoryPermission(namespace, repository).can():
handler = BuildTriggerHandler.get_handler(trigger)
if trigger.repository.kind.name != 'image':
@ -121,6 +120,9 @@ def build_trigger_webhook(trigger_uuid, **kwargs):
start_build(repo, prepared, pull_robot_name=pull_robot_name)
except MaximumBuildsQueuedException:
abort(429, message='Maximum queued build rate exceeded.')
except BuildTriggerDisabledException:
logger.debug('Build trigger %s is disabled', trigger_uuid)
abort(400, message='This build trigger is currently disabled. Please re-enable to continue.')
return make_response('Okay')

7
initdb.py
View file

@ -20,7 +20,7 @@ from data.database import (db, all_models, beta_classes, Role, TeamRole, Visibil
ExternalNotificationEvent, ExternalNotificationMethod, NotificationKind,
QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode,
ServiceKeyApprovalType, MediaType, LabelSourceType, UserPromptKind,
RepositoryKind, TagKind, BlobPlacementLocation, User,
RepositoryKind, TagKind, BlobPlacementLocation, User, DisableReason,
DeletedNamespace)
from data import model
from data.queue import WorkQueue
@ -353,6 +353,7 @@ def initialize_database():
LogEntryKind.create(name='manifest_label_delete')
LogEntryKind.create(name='change_tag_expiration')
LogEntryKind.create(name='toggle_repo_trigger')
LogEntryKind.create(name='create_app_specific_token')
LogEntryKind.create(name='revoke_app_specific_token')
@ -434,6 +435,10 @@ def initialize_database():
TagKind.create(name='release')
TagKind.create(name='channel')
DisableReason.create(name='user_toggled')
DisableReason.create(name='successive_build_failures')
DisableReason.create(name='successive_build_internal_errors')
def wipe_database():
logger.debug('Wiping all data from the DB.')

20
static/css/directives/repo-view/repo-panel-builds.css
View file

@ -13,4 +13,22 @@
.repo-panel-builds .heading-controls {
white-space: nowrap;
}
}
.repo-panel-builds .trigger-disabled {
background-color: #fcfcfc;
}
.repo-panel-builds .trigger-disabled td {
border-bottom: 0px;
color: #ccc;
}
.repo-panel-builds .trigger-disabled-message {
font-size: 13px;
}
.repo-panel-builds i.fa-exclamation-triangle {
color: #f5c77d;
margin-right: 4px;
}

88
static/directives/repo-view/repo-panel-builds.html
View file

@ -137,34 +137,59 @@
<a ng-click="deleteTrigger(trigger)">Delete Trigger</a>
</td>
</tr>
<tr ng-repeat="trigger in triggers | filter:{'is_active':true}">
<td><trigger-description trigger="trigger"></trigger-description></td>
<td>{{ trigger.config.dockerfile_path || '/Dockerfile' }}</td>
<td>{{ trigger.config.context || '/' }}</td>
<td>{{ trigger.config.branchtag_regex || 'All' }}</td>
<td>
<span class="entity-reference" entity="trigger.pull_robot" ng-if="trigger.pull_robot"></span>
<span class="empty" ng-if="!trigger.pull_robot">(None)</span>
</td>
<td>
<span class="cor-options-menu">
<span ng-if="trigger.config.credentials" class="cor-option" option-click="showTriggerCredentialsModal(trigger)">
<i class="fa fa-unlock-alt"></i> View Credentials
<tbody ng-repeat="trigger in triggers | filter:{'is_active':true}">
<tr ng-class="{'trigger-disabled': !trigger.enabled}">
<td><trigger-description trigger="trigger"></trigger-description></td>
<td>{{ trigger.config.dockerfile_path || '/Dockerfile' }}</td>
<td>{{ trigger.config.context || '/' }}</td>
<td>{{ trigger.config.branchtag_regex || 'All' }}</td>
<td>
<span class="entity-reference" entity="trigger.pull_robot" ng-if="trigger.pull_robot"></span>
<span class="empty" ng-if="!trigger.pull_robot">(None)</span>
</td>
<td>
<span class="cor-options-menu">
<span ng-if="trigger.config.credentials" class="cor-option" option-click="showTriggerCredentialsModal(trigger)">
<i class="fa fa-unlock-alt"></i> View Credentials
</span>
<span class="cor-option" option-click="askRunTrigger(trigger)"
ng-class="trigger.can_invoke && trigger.enabled ? '' : 'disabled'">
<i class="fa fa-chevron-right"></i> Run Trigger Now
</span>
<span class="cor-option" option-click="askToggleTrigger(trigger)">
<i class="fa fa-adjust"></i>
<span ng-if="trigger.enabled">Disable Trigger</span>
<span ng-if="!trigger.enabled">Enable Trigger</span>
</span>
<span class="cor-option" option-click="askDeleteTrigger(trigger)">
<i class="fa fa-times"></i> Delete Trigger
</span>
</span>
<span class="cor-option" option-click="askRunTrigger(trigger)"
ng-class="trigger.can_invoke ? '' : 'disabled'">
<i class="fa fa-chevron-right"></i> Run Trigger Now
</span>
<span class="cor-option" option-click="askDeleteTrigger(trigger)">
<i class="fa fa-times"></i> Delete Trigger
</span>
</span>
</td>
</tr>
</td>
</tr>
<tr class="trigger-disabled-message" ng-if="!trigger.enabled">
<td colspan="5" style="text-align: center"
ng-if="trigger.disabled_reason == 'user_toggled'">
<i class="fa fa-exclamation-triangle"></i>
This build trigger is user disabled and will not build:
<a ng-click="askToggleTrigger(trigger)">Re-enable this trigger</a>
</td>
<td colspan="5" style="text-align: center"
ng-if="trigger.disabled_reason == 'successive_build_failures'">
<i class="fa fa-exclamation-triangle"></i>
This build trigger was automatically disabled due to successive failures:
<a ng-click="askToggleTrigger(trigger)">Re-enable this trigger</a>
</td>
<td colspan="5" style="text-align: center"
ng-if="trigger.disabled_reason == 'successive_build_internal_errors'">
<i class="fa fa-exclamation-triangle"></i>
This build trigger was automatically disabled due to successive internal errors:
<a ng-click="askToggleTrigger(trigger)">Re-enable this trigger</a>
</td>
</tr>
</tbody>
</table>
</div>
</div>
</div> <!-- /Build Triggers -->
@ -174,7 +199,16 @@
<!-- Trigger Credentials dialog -->
<div class="trigger-credentials-dialog" trigger="triggerCredentialsModalTrigger" counter="triggerCredentialsModalCounter"></div>
<!-- Delete Tag Confirm -->
<!-- Toggle Trigger Confirm -->
<div class="cor-confirm-dialog"
dialog-context="toggleTriggerInfo"
dialog-action="toggleTrigger(info.trigger, callback)"
dialog-title="Toggle Trigger"
dialog-action-title="Toggle Trigger">
Are you sure you want to <span ng-if="toggleTriggerInfo.trigger.enabled">disable</span><span ng-if="!toggleTriggerInfo.trigger.enabled">enable</span> this trigger?
</div>
<!-- Delete Trigger Confirm -->
<div class="cor-confirm-dialog"
dialog-context="deleteTriggerInfo"
dialog-action="deleteTrigger(info.trigger, callback)"

38
static/js/directives/repo-view/repo-panel-builds.js
View file

@ -187,6 +187,10 @@ angular.module('quay').directive('repoPanelBuilds', function () {
};
$scope.askRunTrigger = function(trigger) {
if (!trigger.enabled) {
return;
}
if (!trigger.can_invoke) {
bootbox.alert('You do not have permission to manually invoke this trigger');
return;
@ -196,6 +200,40 @@ angular.module('quay').directive('repoPanelBuilds', function () {
$scope.showTriggerStartDialogCounter++;
};
$scope.askToggleTrigger = function(trigger) {
if (!trigger.can_invoke) {
bootbox.alert('You do not have permission to edit this trigger');
return;
}
$scope.toggleTriggerInfo = {
'trigger': trigger
};
};
$scope.toggleTrigger = function(trigger, opt_callback) {
if (!trigger) { return; }
var params = {
'repository': $scope.repository.namespace + '/' + $scope.repository.name,
'trigger_uuid': trigger.id
};
var data = {
'enabled': !trigger.enabled
};
var errorHandler = ApiService.errorDisplay('Could not toggle build trigger', function() {
opt_callback && opt_callback(false);
});
ApiService.updateBuildTrigger(data, params).then(function(resp) {
trigger.enabled = !trigger.enabled;
trigger.disabled_reason = 'user_toggled';
opt_callback && opt_callback(true);
}, errorHandler);
};
$scope.deleteTrigger = function(trigger, opt_callback) {
if (!trigger) { return; }

10
static/js/directives/ui/logs-view.js
View file

@ -223,6 +223,15 @@ angular.module('quay').directive('logsView', function () {
metadata['service'], metadata['config']);
return 'Delete build trigger - ' + triggerDescription;
},
'toggle_repo_trigger': function(metadata) {
var triggerDescription = TriggerService.getDescription(
metadata['service'], metadata['config']);
if (metadata.enabled) {
return 'Build trigger enabled - ' + triggerDescription;
} else {
return 'Build trigger disabled - ' + triggerDescription;
}
},
'create_application': 'Create application {application_name} with client ID {client_id}',
'update_application': 'Update application to {application_name} for client ID {client_id}',
'delete_application': 'Delete application {application_name} with client ID {client_id}',
@ -330,6 +339,7 @@ angular.module('quay').directive('logsView', function () {
'delete_prototype_permission': 'Delete default permission',
'setup_repo_trigger': 'Setup build trigger',
'delete_repo_trigger': 'Delete build trigger',
'toggle_repo_trigger': 'Enable/disable build trigger',
'create_application': 'Create Application',
'update_application': 'Update Application',
'delete_application': 'Delete Application',

4
static/partials/trigger-setup.html
View file

@ -27,6 +27,10 @@
<div class="row">
<div class="col-md-offset-3 col-md-6 col-sm-12 col-lg-6 content">
<h3>Trigger has been successfully activated</h3>
<div class="co-alert co-alert-warning">
<strong>Please note:</strong> If the trigger continuously fails to build, it will be automatically
disabled. It can be re-enabled from the build trigger list.
</div>
<div class="credentials" trigger="trigger"></div>
<div class="button-bar">
<a href="/repository/{{ repository.namespace }}/{{ repository.name }}?tab=builds">

BIN
test/data/test.db
View file

Binary file not shown.

2
test/fixtures.py
View file

@ -17,6 +17,7 @@ from endpoints.appr import appr_bp
from endpoints.web import web
from endpoints.v2 import v2_bp
from endpoints.verbs import verbs as verbs_bp
from endpoints.webhooks import webhooks
from initdb import initialize_database, populate_database
@ -175,6 +176,7 @@ def app(appconfig, initialized_db):
app.register_blueprint(web, url_prefix='/')
app.register_blueprint(verbs_bp, url_prefix='/c1')
app.register_blueprint(v2_bp, url_prefix='/v2')
app.register_blueprint(webhooks, url_prefix='/webhooks')
app.config.update(appconfig)
return app

10
util/config/schema.py
View file

@ -707,6 +707,16 @@ CONFIG_SCHEMA = {
'description': 'If not None, the default maximum number of builds that can be queued in a namespace.',
'x-example': 20,
},
'SUCCESSIVE_TRIGGER_INTERNAL_ERROR_DISABLE_THRESHOLD': {
'type': ['number', 'null'],
'description': 'If not None, the number of successive internal errors that can occur before a build trigger is automatically disabled. Defaults to 5.',
'x-example': 10,
},
'SUCCESSIVE_TRIGGER_FAILURE_DISABLE_THRESHOLD': {
'type': ['number', 'null'],
'description': 'If not None, the number of successive failures that can occur before a build trigger is automatically disabled. Defaults to 100.',
'x-example': 50,
},
# Login
'FEATURE_GITHUB_LOGIN': {