vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Lint BuildManager

Browse source
This commit is contained in:
Jimmy Zelinskie 2014-11-18 15:45:56 -05:00
parent 043a30ee96
commit 6df6f28edf
9 changed files with 187 additions and 173 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

146
buildman/buildcomponent.py
View file

@ -10,7 +10,7 @@ from trollius.coroutines import From
from buildman.basecomponent import BaseComponent
from buildman.buildpack import BuildPackage, BuildPackageException
from buildman.buildstatus import StatusHandler
from buildman.server import BUILD_JOB_RESULT
from buildman.server import BuildJobResult
from buildman.workererror import WorkerError
from data.database import BUILD_PHASE
@ -18,9 +18,10 @@ from data.database import BUILD_PHASE
HEARTBEAT_DELTA = datetime.timedelta(seconds=30)
HEARTBEAT_TIMEOUT = 10
logger = logging.getLogger(__name__)
LOGGER = logging.getLogger(__name__)
class COMPONENT_STATUS(object):
class ComponentStatus(object):
""" ComponentStatus represents the possible states of a component. """
JOINING = 'joining'
WAITING = 'waiting'
RUNNING = 'running'
@ -34,7 +35,7 @@ class BuildComponent(BaseComponent):
expected_token = None
builder_realm = None
_component_status = COMPONENT_STATUS.JOINING
_component_status = ComponentStatus.JOINING
_last_heartbeat = None
_current_job = None
_build_status = None
@ -46,33 +47,35 @@ class BuildComponent(BaseComponent):
BaseComponent.__init__(self, config, **kwargs)
def onConnect(self):
def onConnect(self):
self.join(self.builder_realm)
def onJoin(self, details):
logger.debug('Registering methods and listeners for component %s' % self.builder_realm)
LOGGER.debug('Registering methods and listeners for component %s', self.builder_realm)
yield From(self.register(self._on_ready, u'io.quay.buildworker.ready'))
yield From(self.register(self._ping, u'io.quay.buildworker.ping'))
yield From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat'))
yield From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage'))
self._set_status(COMPONENT_STATUS.WAITING)
self._set_status(ComponentStatus.WAITING)
def is_ready(self):
return self._component_status == COMPONENT_STATUS.RUNNING
""" Determines whether a build component is ready to begin a build. """
return self._component_status == ComponentStatus.RUNNING
def start_build(self, build_job):
def start_build(self, build_job):
""" Starts a build. """
self._current_job = build_job
self._build_status = StatusHandler(self.build_logs, build_job.repo_build())
self._image_info = {}
self._set_status(COMPONENT_STATUS.BUILDING)
self._set_status(ComponentStatus.BUILDING)
# Retrieve the job's buildpack.
buildpack_url = self.user_files.get_file_url(build_job.repo_build().resource_key,
requires_cors=False)
logger.debug('Retreiving build package: %s' % buildpack_url)
LOGGER.debug('Retreiving build package: %s', buildpack_url)
buildpack = None
try:
buildpack = BuildPackage.from_url(buildpack_url)
@ -82,7 +85,7 @@ class BuildComponent(BaseComponent):
# Extract the base image information from the Dockerfile.
parsed_dockerfile = None
logger.debug('Parsing dockerfile')
LOGGER.debug('Parsing dockerfile')
build_config = build_job.build_config()
try:
@ -94,11 +97,11 @@ class BuildComponent(BaseComponent):
image_and_tag_tuple = parsed_dockerfile.get_image_and_tag()
if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
self._build_failure('Missing FROM line in Dockerfile')
return
return
base_image_information = {
'repository': image_and_tag_tuple[0],
'tag': image_and_tag_tuple[1]
'repository': image_and_tag_tuple[0],
'tag': image_and_tag_tuple[1]
}
# Extract the number of steps from the Dockerfile.
@ -115,46 +118,47 @@ class BuildComponent(BaseComponent):
repository_name = repo.namespace_user.username + '/' + repo.name
# Parse the build queue item into build arguments.
# build_package: URL to the build package to download and untar/unzip.
# sub_directory: The location within the build package of the Dockerfile and the build context.
# repository: The repository for which this build is occurring.
# registry: The registry for which this build is occuring. Example: 'quay.io', 'staging.quay.io'
# pull_token: The token to use when pulling the cache for building.
# push_token: The token to use to push the built image.
# tag_names: The name(s) of the tag(s) for the newly built image.
# base_image: The image name and credentials to use to conduct the base image pull.
# repository: The repository to pull.
# tag: The tag to pull.
# username: The username for pulling the base image (if any).
# password: The password for pulling the base image (if any).
# build_package: URL to the build package to download and untar/unzip.
# sub_directory: The location within the build package of the Dockerfile and the build context.
# repository: The repository for which this build is occurring.
# registry: The registry for which this build is occuring (e.g. 'quay.io', 'staging.quay.io').
# pull_token: The token to use when pulling the cache for building.
# push_token: The token to use to push the built image.
# tag_names: The name(s) of the tag(s) for the newly built image.
# base_image: The image name and credentials to use to conduct the base image pull.
# repository: The repository to pull.
# tag: The tag to pull.
# username: The username for pulling the base image (if any).
# password: The password for pulling the base image (if any).
build_arguments = {
'build_package': buildpack_url,
'sub_directory': build_config.get('build_subdir', ''),
'repository': repository_name,
'registry': self.server_hostname,
'pull_token': build_job.repo_build().access_token.code,
'push_token': build_job.repo_build().access_token.code,
'tag_names': build_config.get('docker_tags', ['latest']),
'base_image': base_image_information,
'cached_tag': build_job.determine_cached_tag() or ''
'build_package': buildpack_url,
'sub_directory': build_config.get('build_subdir', ''),
'repository': repository_name,
'registry': self.server_hostname,
'pull_token': build_job.repo_build().access_token.code,
'push_token': build_job.repo_build().access_token.code,
'tag_names': build_config.get('docker_tags', ['latest']),
'base_image': base_image_information,
'cached_tag': build_job.determine_cached_tag() or ''
}
# Invoke the build.
logger.debug('Invoking build: %s', self.builder_realm)
logger.debug('With Arguments: %s', build_arguments)
LOGGER.debug('Invoking build: %s', self.builder_realm)
LOGGER.debug('With Arguments: %s', build_arguments)
return (self.call("io.quay.builder.build", **build_arguments)
.add_done_callback(self._build_complete))
.add_done_callback(self._build_complete))
@staticmethod
def __total_completion(statuses, total_images):
def _total_completion(statuses, total_images):
""" Returns the current amount completion relative to the total completion of a build. """
percentage_with_sizes = float(len(statuses.values())) / total_images
sent_bytes = sum([status['current'] for status in statuses.values()])
total_bytes = sum([status['total'] for status in statuses.values()])
return float(sent_bytes) / total_bytes * percentage_with_sizes
@staticmethod
def __process_pushpull_status(status_dict, current_phase, docker_data, images):
def _process_pushpull_status(status_dict, current_phase, docker_data, images):
if not docker_data:
return
@ -178,7 +182,7 @@ class BuildComponent(BaseComponent):
if 'current' in detail and 'total' in detail:
images[image_id] = detail
status_dict[status_completion_key] = \
BuildComponent.__total_completion(images, max(len(images), num_images))
BuildComponent._total_completion(images, max(len(images), num_images))
def _on_log_message(self, phase, json_data):
# Parse any of the JSON data logged.
@ -209,8 +213,8 @@ class BuildComponent(BaseComponent):
# Parse and update the phase and the status_dict. The status dictionary contains
# the pull/push progress, as well as the current step index.
with self._build_status as status_dict:
self._build_status.set_phase(phase)
BuildComponent.__process_pushpull_status(status_dict, phase, docker_data, self._image_info)
self._build_status.set_phase(phase)
BuildComponent._process_pushpull_status(status_dict, phase, docker_data, self._image_info)
# If the current message represents the beginning of a new step, then update the
# current command index.
@ -228,104 +232,110 @@ class BuildComponent(BaseComponent):
def _build_failure(self, error_message, exception=None):
""" Handles and logs a failed build. """
self._build_status.set_error(error_message, {
'internal_error': exception.message if exception else None
'internal_error': exception.message if exception else None
})
build_id = self._current_job.repo_build().uuid
logger.warning('Build %s failed with message: %s', build_id, self._error_message)
LOGGER.warning('Build %s failed with message: %s', build_id, error_message)
# Mark that the build has finished (in an error state)
self._build_finished(BUILD_JOB_RESULT.ERROR)
self._build_finished(BuildJobResult.ERROR)
def _build_complete(self, result):
""" Wraps up a completed build. Handles any errors and calls self._build_finished. """
try:
# Retrieve the result. This will raise an ApplicationError on any error that occurred.
result.result()
self._build_status.set_phase(BUILD_PHASE.COMPLETE)
self._build_finished(BUILD_JOB_RESULT.COMPLETE)
except ApplicationError as ae:
worker_error = WorkerError(ae.error, ae.kwargs.get('base_error'))
self._build_finished(BuildJobResult.COMPLETE)
except ApplicationError as aex:
worker_error = WorkerError(aex.error, aex.kwargs.get('base_error'))
# Write the error to the log.
self._build_status.set_error(worker_error.public_message(), worker_error.extra_data())
# Mark the build as completed.
if worker_error.is_internal_error():
self._build_finished(BUILD_JOB_RESULT.INCOMPLETE)
self._build_finished(BuildJobResult.INCOMPLETE)
else:
self._build_finished(BUILD_JOB_RESULT.ERROR)
self._build_finished(BuildJobResult.ERROR)
def _build_finished(self, job_status):
""" Alerts the parent that a build has completed and sets the status back to running. """
self.parent_manager.job_completed(self._current_job, job_status, self)
self._current_job = None
# Set the component back to a running state.
self._set_status(COMPONENT_STATUS.RUNNING)
self._set_status(ComponentStatus.RUNNING)
def _ping(self):
def _ping():
""" Ping pong. """
return 'pong'
def _on_ready(self, token):
if self._component_status != 'waiting':
logger.warning('Build component with token %s is already connected', self.expected_token)
LOGGER.warning('Build component with token %s is already connected', self.expected_token)
return
if token != self.expected_token:
logger.warning('Builder token mismatch. Expected: %s. Found: %s', self.expected_token, token)
LOGGER.warning('Builder token mismatch. Expected: %s. Found: %s', self.expected_token, token)
return
self._set_status(COMPONENT_STATUS.RUNNING)
self._set_status(ComponentStatus.RUNNING)
# Start the heartbeat check and updating loop.
loop = trollius.get_event_loop()
loop.create_task(self._heartbeat(loop))
logger.debug('Build worker %s is connected and ready' % self.builder_realm)
loop.create_task(self._heartbeat())
LOGGER.debug('Build worker %s is connected and ready', self.builder_realm)
return True
def _set_status(self, phase):
self._component_status = phase
def _on_heartbeat(self):
""" Updates the last known heartbeat. """
self._last_heartbeat = datetime.datetime.now()
def _start_heartbeat(self, loop):
""" Begins an async loop to keep a heartbeat going with a client. """
trollius.set_event_loop(loop)
loop.run_until_complete(self._heartbeat())
@trollius.coroutine
def _heartbeat(self, loop):
def _heartbeat(self):
""" Coroutine that runs every HEARTBEAT_TIMEOUT seconds, both checking the worker's heartbeat
and updating the heartbeat in the build status dictionary (if applicable). This allows
the build system to catch crashes from either end.
"""
while True:
# If the component is no longer running or actively building, nothing more to do.
if (self._component_status != COMPONENT_STATUS.RUNNING and
self._component_status != COMPONENT_STATUS.BUILDING):
if (self._component_status != ComponentStatus.RUNNING and
self._component_status != ComponentStatus.BUILDING):
return
# If there is an active build, write the heartbeat to its status.
build_status = self._build_status
if build_status is not None:
with build_status as status_dict:
status_dict['heartbeat'] = int(time.time())
status_dict['heartbeat'] = int(datetime.time())
# Check the heartbeat from the worker.
logger.debug('Checking heartbeat on realm %s', self.builder_realm)
LOGGER.debug('Checking heartbeat on realm %s', self.builder_realm)
if not self._last_heartbeat:
self._timeout()
return
if self._last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
self._timeout()
return
return
yield From(trollius.sleep(HEARTBEAT_TIMEOUT))
def _timeout(self):
self._set_status(COMPONENT_STATUS.TIMED_OUT)
logger.warning('Build component %s timed out', self.expected_token)
self._set_status(ComponentStatus.TIMED_OUT)
LOGGER.warning('Build component %s timed out', self.expected_token)
self._dispose(timed_out=True)
def _dispose(self, timed_out=False):
@ -335,7 +345,7 @@ class BuildComponent(BaseComponent):
if timed_out:
self._build_status.set_error('Build worker timed out. Build has been requeued')
self.parent_manager.job_completed(self._current_job, BUILD_JOB_RESULT.INCOMPLETE, self)
self.parent_manager.job_completed(self._current_job, BuildJobResult.INCOMPLETE, self)
self._build_status = None
self._current_job = None

11
buildman/buildjob.py
View file

@ -8,7 +8,6 @@ class BuildJobLoadException(Exception):
class BuildJob(object):
""" Represents a single in-progress build job. """
def __init__(self, job_item):
self._job_item = job_item
@ -16,7 +15,8 @@ class BuildJob(object):
self._job_details = json.loads(job_item.body)
except ValueError:
raise BuildJobLoadException(
'Could not parse build queue item config with ID %s' % self._job_details['build_uuid'])
'Could not parse build queue item config with ID %s' % self._job_details['build_uuid']
)
try:
self._repo_build = model.get_repository_build(self._job_details['namespace'],
@ -24,13 +24,14 @@ class BuildJob(object):
self._job_details['build_uuid'])
except model.InvalidRepositoryBuildException:
raise BuildJobLoadException(
'Could not load repository build with ID %s' % self._job_details['build_uuid'])
'Could not load repository build with ID %s' % self._job_details['build_uuid'])
try:
self._build_config = json.loads(self._repo_build.job_config)
except ValueError:
raise BuildJobLoadException(
'Could not parse repository build job config with ID %s' % self._job_details['build_uuid'])
'Could not parse repository build job config with ID %s' % self._job_details['build_uuid']
)
def determine_cached_tag(self):
""" Returns the tag to pull to prime the cache or None if none. """
@ -56,4 +57,4 @@ class BuildJob(object):
def build_config(self):
""" Returns the parsed repository build config for the job. """
return self._build_config
return self._build_config

21
buildman/buildpack.py
View file

@ -4,7 +4,8 @@ import os
from tempfile import TemporaryFile, mkdtemp
from zipfile import ZipFile
from util.dockerfileparse import parse_dockerfile, ParsedDockerfile
from util.dockerfileparse import parse_dockerfile
from util.safetar import safe_extractall
class BuildPackageException(Exception):
""" Exception raised when retrieving or parsing a build package. """
@ -14,15 +15,15 @@ class BuildPackageException(Exception):
class BuildPackage(object):
""" Helper class for easy reading and updating of a Dockerfile build pack. """
def __init__(self, requests_file):
def __init__(self, requests_file):
self._mime_processors = {
'application/zip': BuildPackage.__prepare_zip,
'application/x-zip-compressed': BuildPackage.__prepare_zip,
'text/plain': BuildPackage.__prepare_dockerfile,
'application/octet-stream': BuildPackage.__prepare_dockerfile,
'application/x-tar': BuildPackage.__prepare_tarball,
'application/gzip': BuildPackage.__prepare_tarball,
'application/x-gzip': BuildPackage.__prepare_tarball,
'application/zip': BuildPackage.__prepare_zip,
'application/x-zip-compressed': BuildPackage.__prepare_zip,
'text/plain': BuildPackage.__prepare_dockerfile,
'application/octet-stream': BuildPackage.__prepare_dockerfile,
'application/x-tar': BuildPackage.__prepare_tarball,
'application/gzip': BuildPackage.__prepare_tarball,
'application/x-gzip': BuildPackage.__prepare_tarball,
}
c_type = requests_file.headers['content-type']
@ -84,4 +85,4 @@ class BuildPackage(object):
with tarfile.open(mode='r|*', fileobj=request_file.raw) as tar_stream:
safe_extractall(tar_stream, build_dir)
return build_dir
return build_dir

13
buildman/buildstatus.py
View file

@ -1,4 +1,3 @@
from functools import partial
from data.database import BUILD_PHASE
class StatusHandler(object):
@ -11,10 +10,10 @@ class StatusHandler(object):
self._build_logs = build_logs
self._status = {
'total_commands': None,
'current_command': None,
'push_completion': 0.0,
'pull_completion': 0.0,
'total_commands': None,
'current_command': None,
'push_completion': 0.0,
'pull_completion': 0.0,
}
# Write the initial status.
@ -36,7 +35,7 @@ class StatusHandler(object):
def set_phase(self, phase, extra_data=None):
if phase == self._current_phase:
return
self._current_phase = phase
self._append_log_message(phase, self._build_logs.PHASE, extra_data)
self._repository_build.phase = phase
@ -46,4 +45,4 @@ class StatusHandler(object):
return self._status
def __exit__(self, exc_type, value, traceback):
self._build_logs.set_status(self._uuid, self._status)
self._build_logs.set_status(self._uuid, self._status)

4
buildman/enterprise_builder.py
View file

@ -6,7 +6,7 @@ from app import app, userfiles as user_files, build_logs, dockerfile_build_queue
from buildman.manager.enterprise import EnterpriseManager
from buildman.server import BuilderServer
logger = logging.getLogger(__name__)
LOGGER = logging.getLogger(__name__)
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
@ -17,4 +17,4 @@ if __name__ == '__main__':
server = BuilderServer(app.config['SERVER_HOSTNAME'], dockerfile_build_queue, build_logs,
user_files, EnterpriseManager)
server.run(args.host)
server.run(args.host)

2
buildman/manager/basemanager.py
View file

@ -33,4 +33,4 @@ class BaseManager(object):
""" Method invoked once a job_item has completed, in some manner. The job_status will be
one of: incomplete, error, complete. If incomplete, the job should be requeued.
"""
raise NotImplementedError
raise NotImplementedError

10
buildman/manager/enterprise.py
View file

@ -8,7 +8,7 @@ from buildman.buildcomponent import BuildComponent
from trollius.coroutines import From
REGISTRATION_REALM = 'registration'
logger = logging.getLogger(__name__)
LOGGER = logging.getLogger(__name__)
class DynamicRegistrationComponent(BaseComponent):
""" Component session that handles dynamic registration of the builder components. """
@ -17,12 +17,12 @@ class DynamicRegistrationComponent(BaseComponent):
self.join(REGISTRATION_REALM)
def onJoin(self, details):
logger.debug('Registering registration method')
LOGGER.debug('Registering registration method')
yield From(self.register(self._worker_register, u'io.quay.buildworker.register'))
def _worker_register(self):
realm = self.parent_manager.add_build_component()
logger.debug('Registering new build component+worker with realm %s', realm)
LOGGER.debug('Registering new build component+worker with realm %s', realm)
return realm
@ -35,8 +35,9 @@ class EnterpriseManager(BaseManager):
# Add a component which is used by build workers for dynamic registration. Unlike
# production, build workers in enterprise are long-lived and register dynamically.
self.register_component(REGISTRATION_REALM, DynamicRegistrationComponent)
def add_build_component(self):
""" Adds a new build component for an Enterprise Registry. """
# Generate a new unique realm ID for the build worker.
realm = str(uuid.uuid4())
component = self.register_component(realm, BuildComponent, token="")
@ -44,6 +45,7 @@ class EnterpriseManager(BaseManager):
return realm
def schedule(self, build_job, loop):
""" Schedules a build for an Enterprise Registry. """
if self.shutting_down:
return False

63
buildman/server.py
View file

@ -2,24 +2,23 @@ import logging
import trollius
from autobahn.asyncio.wamp import RouterFactory, RouterSessionFactory
from autobahn.asyncio.websocket import WampWebSocketServerFactory, WampWebSocketServerProtocol
from autobahn.asyncio.websocket import WampWebSocketServerFactory
from autobahn.wamp import types
from autobahn.wamp.exception import ApplicationError
from aiowsgi import create_server as create_wsgi_server
from flask import Flask
from threading import Event, Lock
from threading import Event
from trollius.coroutines import From
from buildjob import BuildJob, BuildJobLoadException
from buildman.buildjob import BuildJob, BuildJobLoadException
logger = logging.getLogger(__name__)
LOGGER = logging.getLogger(__name__)
WORK_CHECK_TIMEOUT = 10
TIMEOUT_PERIOD_MINUTES = 20
RESERVATION_SECONDS = (TIMEOUT_PERIOD_MINUTES + 5) * 60
class BUILD_JOB_RESULT(object):
class BuildJobResult(object):
""" Build job result enum """
INCOMPLETE = 'incomplete'
COMPLETE = 'complete'
@ -29,20 +28,22 @@ class BuilderServer(object):
""" Server which handles both HTTP and WAMP requests, managing the full state of the build
controller.
"""
_loop = None
_current_status = 'starting'
_current_components = []
_job_count = 0
def __init__(self, server_hostname, queue, build_logs, user_files, lifecycle_manager_klass):
self._session_factory = RouterSessionFactory(RouterFactory())
self._loop = None
self._current_status = 'starting'
self._current_components = []
self._job_count = 0
self._session_factory = RouterSessionFactory(RouterFactory())
self._server_hostname = server_hostname
self._queue = queue
self._build_logs = build_logs
self._user_files = user_files
self._lifecycle_manager = lifecycle_manager_klass(
self._register_component, self._unregister_component, self._job_complete)
self._register_component,
self._unregister_component,
self._job_complete
)
self._shutdown_event = Event()
self._current_status = 'running'
@ -69,11 +70,11 @@ class BuilderServer(object):
logging.debug('Starting server on port 8080, with controller on port 8181')
try:
loop.run_forever()
loop.run_forever()
except KeyboardInterrupt:
pass
pass
finally:
loop.close()
loop.close()
def close(self):
logging.debug('Requested server shutdown')
@ -88,7 +89,7 @@ class BuilderServer(object):
"""
logging.debug('Registering component with realm %s', realm)
component = component_klass(types.ComponentConfig(realm = realm), realm=realm, **kwargs)
component = component_klass(types.ComponentConfig(realm=realm), realm=realm, **kwargs)
component.server = self
component.parent_manager = self._lifecycle_manager
component.build_logs = self._build_logs
@ -101,15 +102,15 @@ class BuilderServer(object):
def _unregister_component(self, component):
logging.debug('Unregistering component with realm %s and token %s',
component.builder_realm, component.expected_token)
component.builder_realm, component.expected_token)
self._current_components.remove(component)
self._session_factory.remove(component)
def _job_complete(self, build_job, job_status):
if job_status == BUILD_JOB_RESULT.INCOMPLETE:
if job_status == BuildJobResult.INCOMPLETE:
self._queue.incomplete(build_job.job_item(), restore_retry=True, retry_after=30)
elif job_status == BUILD_JOB_RESULT.ERROR:
elif job_status == BuildJobResult.ERROR:
self._queue.incomplete(build_job.job_item(), restore_retry=False)
else:
self._queue.complete(build_job.job_item())
@ -119,42 +120,42 @@ class BuilderServer(object):
if self._current_status == 'shutting_down' and not self._job_count:
self._shutdown_event.set()
# TODO: check for work here?
# TODO:(jschorr) check for work here?
@trollius.coroutine
def _work_checker(self):
while self._current_status == 'running':
logger.debug('Checking for more work')
LOGGER.debug('Checking for more work')
job_item = self._queue.get(processing_time=RESERVATION_SECONDS)
if job_item is None:
logger.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT)
LOGGER.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT)
yield From(trollius.sleep(WORK_CHECK_TIMEOUT))
continue
try:
build_job = BuildJob(job_item)
except BuildJobLoadException as irbe:
logger.exception(irbe)
LOGGER.exception(irbe)
self._queue.incomplete(job_item, restore_retry=False)
logger.debug('Build job found. Checking for an avaliable worker.')
LOGGER.debug('Build job found. Checking for an avaliable worker.')
if self._lifecycle_manager.schedule(build_job, self._loop):
self._job_count = self._job_count + 1
logger.debug('Build job scheduled. Running: %s', self._job_count)
LOGGER.debug('Build job scheduled. Running: %s', self._job_count)
else:
logger.debug('All workers are busy. Requeuing.')
LOGGER.debug('All workers are busy. Requeuing.')
self._queue.incomplete(job_item, restore_retry=True, retry_after=0)
yield From(trollius.sleep(WORK_CHECK_TIMEOUT))
@trollius.coroutine
def _initialize(self, loop, host):
self._loop = loop
# Create the WAMP server.
transport_factory = WampWebSocketServerFactory(self._session_factory, debug_wamp = False)
transport_factory.setProtocolOptions(failByDrop = True)
transport_factory = WampWebSocketServerFactory(self._session_factory, debug_wamp=False)
transport_factory.setProtocolOptions(failByDrop=True)
# Initialize the controller server and the WAMP server
create_wsgi_server(self._controller_app, loop=loop, host=host, port=8181)

90
buildman/workererror.py
View file

@ -5,59 +5,59 @@ class WorkerError(object):
self._base_message = base_message
self._error_handlers = {
'io.quay.builder.buildpackissue': {
'message': 'Could not load build package',
'is_internal': True
},
'io.quay.builder.buildpackissue': {
'message': 'Could not load build package',
'is_internal': True
},
'io.quay.builder.cannotextractbuildpack': {
'message': 'Could not extract the contents of the build package'
},
'io.quay.builder.cannotextractbuildpack': {
'message': 'Could not extract the contents of the build package'
},
'io.quay.builder.cannotpullforcache': {
'message': 'Could not pull cached image',
'is_internal': True
},
'io.quay.builder.cannotpullforcache': {
'message': 'Could not pull cached image',
'is_internal': True
},
'io.quay.builder.cannotpullbaseimage': {
'message': 'Could not pull base image',
'show_base_error': True
},
'io.quay.builder.cannotpullbaseimage': {
'message': 'Could not pull base image',
'show_base_error': True
},
'io.quay.builder.internalerror': {
'message': 'An internal error occurred while building. Please submit a ticket.'
},
'io.quay.builder.internalerror': {
'message': 'An internal error occurred while building. Please submit a ticket.'
},
'io.quay.builder.buildrunerror': {
'message': 'Could not start the build process',
'is_internal': True
},
'io.quay.builder.buildrunerror': {
'message': 'Could not start the build process',
'is_internal': True
},
'io.quay.builder.builderror': {
'message': 'A build step failed',
'show_base_error': True
},
'io.quay.builder.builderror': {
'message': 'A build step failed',
'show_base_error': True
},
'io.quay.builder.tagissue': {
'message': 'Could not tag built image',
'is_internal': True
},
'io.quay.builder.tagissue': {
'message': 'Could not tag built image',
'is_internal': True
},
'io.quay.builder.pushissue': {
'message': 'Could not push built image',
'show_base_error': True,
'is_internal': True
},
'io.quay.builder.pushissue': {
'message': 'Could not push built image',
'show_base_error': True,
'is_internal': True
},
'io.quay.builder.dockerconnecterror': {
'message': 'Could not connect to Docker daemon',
'is_internal': True
},
'io.quay.builder.dockerconnecterror': {
'message': 'Could not connect to Docker daemon',
'is_internal': True
},
'io.quay.builder.missingorinvalidargument': {
'message': 'Missing required arguments for builder',
'is_internal': True
}
'io.quay.builder.missingorinvalidargument': {
'message': 'Missing required arguments for builder',
'is_internal': True
}
}
def is_internal_error(self):
@ -81,7 +81,7 @@ class WorkerError(object):
def extra_data(self):
if self._base_message:
return {
'base_error': self._base_message
'base_error': self._base_message
}
return {}
return {}