vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add timeout and failure if an EC2 instance could not be found when tagging

Browse source 
Fixes #994
This commit is contained in:
Joseph Schorr 2015-11-30 14:37:15 -05:00
parent eeb4f952dc
commit 946e5fabc0
1 changed files with 15 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
buildman/manager/executor.py
View file

@ -5,6 +5,8 @@ import threading
import boto.ec2 import boto.ec2
import requests import requests
import cachetools import cachetools
import trollius
from jinja2 import FileSystemLoader, Environment from jinja2 import FileSystemLoader, Environment
from trollius import coroutine, From, Return, get_event_loop from trollius import coroutine, From, Return, get_event_loop
@ -19,6 +21,9 @@ logger = logging.getLogger(__name__)
ONE_HOUR = 60*60 ONE_HOUR = 60*60
_TAG_RETRY_COUNT = 3 # Number of times to retry adding tags.
_TAG_RETRY_SLEEP = 2 # Number of seconds to wait between tag retries.
ENV = Environment(loader=FileSystemLoader('buildman/templates')) ENV = Environment(loader=FileSystemLoader('buildman/templates'))
TEMPLATE = ENV.get_template('cloudconfig.yaml') TEMPLATE = ENV.get_template('cloudconfig.yaml')
CloudConfigContext().populate_jinja_environment(ENV) CloudConfigContext().populate_jinja_environment(ENV)
@ -147,7 +152,7 @@ class EC2Executor(BuilderExecutor):
launched = AsyncWrapper(reservation.instances[0]) launched = AsyncWrapper(reservation.instances[0])
for i in range(0, 2): for i in range(0, _TAG_RETRY_COUNT):
try: try:
yield From(launched.add_tags({ yield From(launched.add_tags({
'Name': 'Quay Ephemeral Builder', 'Name': 'Quay Ephemeral Builder',
@ -155,7 +160,15 @@ class EC2Executor(BuilderExecutor):
'Token': token, 'Token': token,
'BuildUUID': build_uuid, 'BuildUUID': build_uuid,
})) }))
except boto.exception.EC2ResponseError: except boto.exception.EC2ResponseError as ec2e:
if ec2e.error_code == 404:
if i < _TAG_RETRY_COUNT - 1:
logger.warning('Failed to write EC2 tags (attempt #%s)', i)
yield From(trollius.sleep(_TAG_RETRY_SLEEP))
continue
raise ExecutorException('Unable to find builder instance.')
logger.exception('Failed to write EC2 tags (attempt #%s)', i) logger.exception('Failed to write EC2 tags (attempt #%s)', i)
raise Return(launched.id) raise Return(launched.id)