diff --git a/buildman/buildcomponent.py b/buildman/buildcomponent.py index de63aa0b5..3e2458c4a 100644 --- a/buildman/buildcomponent.py +++ b/buildman/buildcomponent.py @@ -10,7 +10,7 @@ from trollius.coroutines import From from buildman.basecomponent import BaseComponent from buildman.buildpack import BuildPackage, BuildPackageException from buildman.buildstatus import StatusHandler -from buildman.server import BUILD_JOB_RESULT +from buildman.server import BuildJobResult from buildman.workererror import WorkerError from data.database import BUILD_PHASE @@ -18,9 +18,10 @@ from data.database import BUILD_PHASE HEARTBEAT_DELTA = datetime.timedelta(seconds=30) HEARTBEAT_TIMEOUT = 10 -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) -class COMPONENT_STATUS(object): +class ComponentStatus(object): + """ ComponentStatus represents the possible states of a component. """ JOINING = 'joining' WAITING = 'waiting' RUNNING = 'running' @@ -34,7 +35,7 @@ class BuildComponent(BaseComponent): expected_token = None builder_realm = None - _component_status = COMPONENT_STATUS.JOINING + _component_status = ComponentStatus.JOINING _last_heartbeat = None _current_job = None _build_status = None @@ -46,33 +47,35 @@ class BuildComponent(BaseComponent): BaseComponent.__init__(self, config, **kwargs) - def onConnect(self): + def onConnect(self): self.join(self.builder_realm) def onJoin(self, details): - logger.debug('Registering methods and listeners for component %s' % self.builder_realm) + LOGGER.debug('Registering methods and listeners for component %s', self.builder_realm) yield From(self.register(self._on_ready, u'io.quay.buildworker.ready')) yield From(self.register(self._ping, u'io.quay.buildworker.ping')) yield From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat')) yield From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage')) - self._set_status(COMPONENT_STATUS.WAITING) + self._set_status(ComponentStatus.WAITING) def is_ready(self): - return self._component_status == COMPONENT_STATUS.RUNNING + """ Determines whether a build component is ready to begin a build. """ + return self._component_status == ComponentStatus.RUNNING - def start_build(self, build_job): + def start_build(self, build_job): + """ Starts a build. """ self._current_job = build_job self._build_status = StatusHandler(self.build_logs, build_job.repo_build()) self._image_info = {} - self._set_status(COMPONENT_STATUS.BUILDING) + self._set_status(ComponentStatus.BUILDING) # Retrieve the job's buildpack. buildpack_url = self.user_files.get_file_url(build_job.repo_build().resource_key, requires_cors=False) - logger.debug('Retreiving build package: %s' % buildpack_url) + LOGGER.debug('Retreiving build package: %s', buildpack_url) buildpack = None try: buildpack = BuildPackage.from_url(buildpack_url) @@ -82,7 +85,7 @@ class BuildComponent(BaseComponent): # Extract the base image information from the Dockerfile. parsed_dockerfile = None - logger.debug('Parsing dockerfile') + LOGGER.debug('Parsing dockerfile') build_config = build_job.build_config() try: @@ -94,11 +97,11 @@ class BuildComponent(BaseComponent): image_and_tag_tuple = parsed_dockerfile.get_image_and_tag() if image_and_tag_tuple is None or image_and_tag_tuple[0] is None: self._build_failure('Missing FROM line in Dockerfile') - return + return base_image_information = { - 'repository': image_and_tag_tuple[0], - 'tag': image_and_tag_tuple[1] + 'repository': image_and_tag_tuple[0], + 'tag': image_and_tag_tuple[1] } # Extract the number of steps from the Dockerfile. @@ -115,46 +118,47 @@ class BuildComponent(BaseComponent): repository_name = repo.namespace_user.username + '/' + repo.name # Parse the build queue item into build arguments. - # build_package: URL to the build package to download and untar/unzip. - # sub_directory: The location within the build package of the Dockerfile and the build context. - # repository: The repository for which this build is occurring. - # registry: The registry for which this build is occuring. Example: 'quay.io', 'staging.quay.io' - # pull_token: The token to use when pulling the cache for building. - # push_token: The token to use to push the built image. - # tag_names: The name(s) of the tag(s) for the newly built image. - # base_image: The image name and credentials to use to conduct the base image pull. - # repository: The repository to pull. - # tag: The tag to pull. - # username: The username for pulling the base image (if any). - # password: The password for pulling the base image (if any). + # build_package: URL to the build package to download and untar/unzip. + # sub_directory: The location within the build package of the Dockerfile and the build context. + # repository: The repository for which this build is occurring. + # registry: The registry for which this build is occuring (e.g. 'quay.io', 'staging.quay.io'). + # pull_token: The token to use when pulling the cache for building. + # push_token: The token to use to push the built image. + # tag_names: The name(s) of the tag(s) for the newly built image. + # base_image: The image name and credentials to use to conduct the base image pull. + # repository: The repository to pull. + # tag: The tag to pull. + # username: The username for pulling the base image (if any). + # password: The password for pulling the base image (if any). build_arguments = { - 'build_package': buildpack_url, - 'sub_directory': build_config.get('build_subdir', ''), - 'repository': repository_name, - 'registry': self.server_hostname, - 'pull_token': build_job.repo_build().access_token.code, - 'push_token': build_job.repo_build().access_token.code, - 'tag_names': build_config.get('docker_tags', ['latest']), - 'base_image': base_image_information, - 'cached_tag': build_job.determine_cached_tag() or '' + 'build_package': buildpack_url, + 'sub_directory': build_config.get('build_subdir', ''), + 'repository': repository_name, + 'registry': self.server_hostname, + 'pull_token': build_job.repo_build().access_token.code, + 'push_token': build_job.repo_build().access_token.code, + 'tag_names': build_config.get('docker_tags', ['latest']), + 'base_image': base_image_information, + 'cached_tag': build_job.determine_cached_tag() or '' } # Invoke the build. - logger.debug('Invoking build: %s', self.builder_realm) - logger.debug('With Arguments: %s', build_arguments) + LOGGER.debug('Invoking build: %s', self.builder_realm) + LOGGER.debug('With Arguments: %s', build_arguments) return (self.call("io.quay.builder.build", **build_arguments) - .add_done_callback(self._build_complete)) + .add_done_callback(self._build_complete)) @staticmethod - def __total_completion(statuses, total_images): + def _total_completion(statuses, total_images): + """ Returns the current amount completion relative to the total completion of a build. """ percentage_with_sizes = float(len(statuses.values())) / total_images sent_bytes = sum([status['current'] for status in statuses.values()]) total_bytes = sum([status['total'] for status in statuses.values()]) return float(sent_bytes) / total_bytes * percentage_with_sizes @staticmethod - def __process_pushpull_status(status_dict, current_phase, docker_data, images): + def _process_pushpull_status(status_dict, current_phase, docker_data, images): if not docker_data: return @@ -178,7 +182,7 @@ class BuildComponent(BaseComponent): if 'current' in detail and 'total' in detail: images[image_id] = detail status_dict[status_completion_key] = \ - BuildComponent.__total_completion(images, max(len(images), num_images)) + BuildComponent._total_completion(images, max(len(images), num_images)) def _on_log_message(self, phase, json_data): # Parse any of the JSON data logged. @@ -209,8 +213,8 @@ class BuildComponent(BaseComponent): # Parse and update the phase and the status_dict. The status dictionary contains # the pull/push progress, as well as the current step index. with self._build_status as status_dict: - self._build_status.set_phase(phase) - BuildComponent.__process_pushpull_status(status_dict, phase, docker_data, self._image_info) + self._build_status.set_phase(phase) + BuildComponent._process_pushpull_status(status_dict, phase, docker_data, self._image_info) # If the current message represents the beginning of a new step, then update the # current command index. @@ -228,104 +232,110 @@ class BuildComponent(BaseComponent): def _build_failure(self, error_message, exception=None): + """ Handles and logs a failed build. """ self._build_status.set_error(error_message, { - 'internal_error': exception.message if exception else None + 'internal_error': exception.message if exception else None }) build_id = self._current_job.repo_build().uuid - logger.warning('Build %s failed with message: %s', build_id, self._error_message) + LOGGER.warning('Build %s failed with message: %s', build_id, error_message) # Mark that the build has finished (in an error state) - self._build_finished(BUILD_JOB_RESULT.ERROR) + self._build_finished(BuildJobResult.ERROR) def _build_complete(self, result): + """ Wraps up a completed build. Handles any errors and calls self._build_finished. """ try: # Retrieve the result. This will raise an ApplicationError on any error that occurred. result.result() self._build_status.set_phase(BUILD_PHASE.COMPLETE) - self._build_finished(BUILD_JOB_RESULT.COMPLETE) - except ApplicationError as ae: - worker_error = WorkerError(ae.error, ae.kwargs.get('base_error')) + self._build_finished(BuildJobResult.COMPLETE) + except ApplicationError as aex: + worker_error = WorkerError(aex.error, aex.kwargs.get('base_error')) # Write the error to the log. self._build_status.set_error(worker_error.public_message(), worker_error.extra_data()) # Mark the build as completed. if worker_error.is_internal_error(): - self._build_finished(BUILD_JOB_RESULT.INCOMPLETE) + self._build_finished(BuildJobResult.INCOMPLETE) else: - self._build_finished(BUILD_JOB_RESULT.ERROR) + self._build_finished(BuildJobResult.ERROR) def _build_finished(self, job_status): + """ Alerts the parent that a build has completed and sets the status back to running. """ self.parent_manager.job_completed(self._current_job, job_status, self) self._current_job = None # Set the component back to a running state. - self._set_status(COMPONENT_STATUS.RUNNING) + self._set_status(ComponentStatus.RUNNING) - def _ping(self): + def _ping(): + """ Ping pong. """ return 'pong' def _on_ready(self, token): if self._component_status != 'waiting': - logger.warning('Build component with token %s is already connected', self.expected_token) + LOGGER.warning('Build component with token %s is already connected', self.expected_token) return if token != self.expected_token: - logger.warning('Builder token mismatch. Expected: %s. Found: %s', self.expected_token, token) + LOGGER.warning('Builder token mismatch. Expected: %s. Found: %s', self.expected_token, token) return - self._set_status(COMPONENT_STATUS.RUNNING) + self._set_status(ComponentStatus.RUNNING) # Start the heartbeat check and updating loop. loop = trollius.get_event_loop() - loop.create_task(self._heartbeat(loop)) - logger.debug('Build worker %s is connected and ready' % self.builder_realm) + loop.create_task(self._heartbeat()) + LOGGER.debug('Build worker %s is connected and ready', self.builder_realm) return True def _set_status(self, phase): self._component_status = phase def _on_heartbeat(self): + """ Updates the last known heartbeat. """ self._last_heartbeat = datetime.datetime.now() def _start_heartbeat(self, loop): + """ Begins an async loop to keep a heartbeat going with a client. """ trollius.set_event_loop(loop) loop.run_until_complete(self._heartbeat()) @trollius.coroutine - def _heartbeat(self, loop): + def _heartbeat(self): """ Coroutine that runs every HEARTBEAT_TIMEOUT seconds, both checking the worker's heartbeat and updating the heartbeat in the build status dictionary (if applicable). This allows the build system to catch crashes from either end. """ while True: # If the component is no longer running or actively building, nothing more to do. - if (self._component_status != COMPONENT_STATUS.RUNNING and - self._component_status != COMPONENT_STATUS.BUILDING): + if (self._component_status != ComponentStatus.RUNNING and + self._component_status != ComponentStatus.BUILDING): return # If there is an active build, write the heartbeat to its status. build_status = self._build_status if build_status is not None: with build_status as status_dict: - status_dict['heartbeat'] = int(time.time()) + status_dict['heartbeat'] = int(datetime.time()) # Check the heartbeat from the worker. - logger.debug('Checking heartbeat on realm %s', self.builder_realm) + LOGGER.debug('Checking heartbeat on realm %s', self.builder_realm) if not self._last_heartbeat: self._timeout() return if self._last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA: self._timeout() - return + return yield From(trollius.sleep(HEARTBEAT_TIMEOUT)) def _timeout(self): - self._set_status(COMPONENT_STATUS.TIMED_OUT) - logger.warning('Build component %s timed out', self.expected_token) + self._set_status(ComponentStatus.TIMED_OUT) + LOGGER.warning('Build component %s timed out', self.expected_token) self._dispose(timed_out=True) def _dispose(self, timed_out=False): @@ -335,7 +345,7 @@ class BuildComponent(BaseComponent): if timed_out: self._build_status.set_error('Build worker timed out. Build has been requeued') - self.parent_manager.job_completed(self._current_job, BUILD_JOB_RESULT.INCOMPLETE, self) + self.parent_manager.job_completed(self._current_job, BuildJobResult.INCOMPLETE, self) self._build_status = None self._current_job = None diff --git a/buildman/buildjob.py b/buildman/buildjob.py index bfd9363ec..2b9dbf35c 100644 --- a/buildman/buildjob.py +++ b/buildman/buildjob.py @@ -8,7 +8,6 @@ class BuildJobLoadException(Exception): class BuildJob(object): """ Represents a single in-progress build job. """ - def __init__(self, job_item): self._job_item = job_item @@ -16,7 +15,8 @@ class BuildJob(object): self._job_details = json.loads(job_item.body) except ValueError: raise BuildJobLoadException( - 'Could not parse build queue item config with ID %s' % self._job_details['build_uuid']) + 'Could not parse build queue item config with ID %s' % self._job_details['build_uuid'] + ) try: self._repo_build = model.get_repository_build(self._job_details['namespace'], @@ -24,13 +24,14 @@ class BuildJob(object): self._job_details['build_uuid']) except model.InvalidRepositoryBuildException: raise BuildJobLoadException( - 'Could not load repository build with ID %s' % self._job_details['build_uuid']) + 'Could not load repository build with ID %s' % self._job_details['build_uuid']) try: self._build_config = json.loads(self._repo_build.job_config) except ValueError: raise BuildJobLoadException( - 'Could not parse repository build job config with ID %s' % self._job_details['build_uuid']) + 'Could not parse repository build job config with ID %s' % self._job_details['build_uuid'] + ) def determine_cached_tag(self): """ Returns the tag to pull to prime the cache or None if none. """ @@ -56,4 +57,4 @@ class BuildJob(object): def build_config(self): """ Returns the parsed repository build config for the job. """ - return self._build_config \ No newline at end of file + return self._build_config diff --git a/buildman/buildpack.py b/buildman/buildpack.py index 62dab38e5..cdc4d4b07 100644 --- a/buildman/buildpack.py +++ b/buildman/buildpack.py @@ -4,7 +4,8 @@ import os from tempfile import TemporaryFile, mkdtemp from zipfile import ZipFile -from util.dockerfileparse import parse_dockerfile, ParsedDockerfile +from util.dockerfileparse import parse_dockerfile +from util.safetar import safe_extractall class BuildPackageException(Exception): """ Exception raised when retrieving or parsing a build package. """ @@ -14,15 +15,15 @@ class BuildPackageException(Exception): class BuildPackage(object): """ Helper class for easy reading and updating of a Dockerfile build pack. """ - def __init__(self, requests_file): + def __init__(self, requests_file): self._mime_processors = { - 'application/zip': BuildPackage.__prepare_zip, - 'application/x-zip-compressed': BuildPackage.__prepare_zip, - 'text/plain': BuildPackage.__prepare_dockerfile, - 'application/octet-stream': BuildPackage.__prepare_dockerfile, - 'application/x-tar': BuildPackage.__prepare_tarball, - 'application/gzip': BuildPackage.__prepare_tarball, - 'application/x-gzip': BuildPackage.__prepare_tarball, + 'application/zip': BuildPackage.__prepare_zip, + 'application/x-zip-compressed': BuildPackage.__prepare_zip, + 'text/plain': BuildPackage.__prepare_dockerfile, + 'application/octet-stream': BuildPackage.__prepare_dockerfile, + 'application/x-tar': BuildPackage.__prepare_tarball, + 'application/gzip': BuildPackage.__prepare_tarball, + 'application/x-gzip': BuildPackage.__prepare_tarball, } c_type = requests_file.headers['content-type'] @@ -84,4 +85,4 @@ class BuildPackage(object): with tarfile.open(mode='r|*', fileobj=request_file.raw) as tar_stream: safe_extractall(tar_stream, build_dir) - return build_dir \ No newline at end of file + return build_dir diff --git a/buildman/buildstatus.py b/buildman/buildstatus.py index d09876e10..17a7c9d1e 100644 --- a/buildman/buildstatus.py +++ b/buildman/buildstatus.py @@ -1,4 +1,3 @@ -from functools import partial from data.database import BUILD_PHASE class StatusHandler(object): @@ -11,10 +10,10 @@ class StatusHandler(object): self._build_logs = build_logs self._status = { - 'total_commands': None, - 'current_command': None, - 'push_completion': 0.0, - 'pull_completion': 0.0, + 'total_commands': None, + 'current_command': None, + 'push_completion': 0.0, + 'pull_completion': 0.0, } # Write the initial status. @@ -36,7 +35,7 @@ class StatusHandler(object): def set_phase(self, phase, extra_data=None): if phase == self._current_phase: return - + self._current_phase = phase self._append_log_message(phase, self._build_logs.PHASE, extra_data) self._repository_build.phase = phase @@ -46,4 +45,4 @@ class StatusHandler(object): return self._status def __exit__(self, exc_type, value, traceback): - self._build_logs.set_status(self._uuid, self._status) \ No newline at end of file + self._build_logs.set_status(self._uuid, self._status) diff --git a/buildman/enterprise_builder.py b/buildman/enterprise_builder.py index 62b67a14c..cc9ce8432 100644 --- a/buildman/enterprise_builder.py +++ b/buildman/enterprise_builder.py @@ -6,7 +6,7 @@ from app import app, userfiles as user_files, build_logs, dockerfile_build_queue from buildman.manager.enterprise import EnterpriseManager from buildman.server import BuilderServer -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) @@ -17,4 +17,4 @@ if __name__ == '__main__': server = BuilderServer(app.config['SERVER_HOSTNAME'], dockerfile_build_queue, build_logs, user_files, EnterpriseManager) - server.run(args.host) + server.run(args.host) diff --git a/buildman/manager/basemanager.py b/buildman/manager/basemanager.py index 7b5816c9b..c856159b5 100644 --- a/buildman/manager/basemanager.py +++ b/buildman/manager/basemanager.py @@ -33,4 +33,4 @@ class BaseManager(object): """ Method invoked once a job_item has completed, in some manner. The job_status will be one of: incomplete, error, complete. If incomplete, the job should be requeued. """ - raise NotImplementedError \ No newline at end of file + raise NotImplementedError diff --git a/buildman/manager/enterprise.py b/buildman/manager/enterprise.py index 1b59a07eb..584a6daf4 100644 --- a/buildman/manager/enterprise.py +++ b/buildman/manager/enterprise.py @@ -8,7 +8,7 @@ from buildman.buildcomponent import BuildComponent from trollius.coroutines import From REGISTRATION_REALM = 'registration' -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) class DynamicRegistrationComponent(BaseComponent): """ Component session that handles dynamic registration of the builder components. """ @@ -17,12 +17,12 @@ class DynamicRegistrationComponent(BaseComponent): self.join(REGISTRATION_REALM) def onJoin(self, details): - logger.debug('Registering registration method') + LOGGER.debug('Registering registration method') yield From(self.register(self._worker_register, u'io.quay.buildworker.register')) def _worker_register(self): realm = self.parent_manager.add_build_component() - logger.debug('Registering new build component+worker with realm %s', realm) + LOGGER.debug('Registering new build component+worker with realm %s', realm) return realm @@ -35,8 +35,9 @@ class EnterpriseManager(BaseManager): # Add a component which is used by build workers for dynamic registration. Unlike # production, build workers in enterprise are long-lived and register dynamically. self.register_component(REGISTRATION_REALM, DynamicRegistrationComponent) - + def add_build_component(self): + """ Adds a new build component for an Enterprise Registry. """ # Generate a new unique realm ID for the build worker. realm = str(uuid.uuid4()) component = self.register_component(realm, BuildComponent, token="") @@ -44,6 +45,7 @@ class EnterpriseManager(BaseManager): return realm def schedule(self, build_job, loop): + """ Schedules a build for an Enterprise Registry. """ if self.shutting_down: return False diff --git a/buildman/server.py b/buildman/server.py index 76e6eacc4..e8fa237c7 100644 --- a/buildman/server.py +++ b/buildman/server.py @@ -2,24 +2,23 @@ import logging import trollius from autobahn.asyncio.wamp import RouterFactory, RouterSessionFactory -from autobahn.asyncio.websocket import WampWebSocketServerFactory, WampWebSocketServerProtocol +from autobahn.asyncio.websocket import WampWebSocketServerFactory from autobahn.wamp import types -from autobahn.wamp.exception import ApplicationError from aiowsgi import create_server as create_wsgi_server from flask import Flask -from threading import Event, Lock +from threading import Event from trollius.coroutines import From -from buildjob import BuildJob, BuildJobLoadException +from buildman.buildjob import BuildJob, BuildJobLoadException -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) WORK_CHECK_TIMEOUT = 10 TIMEOUT_PERIOD_MINUTES = 20 RESERVATION_SECONDS = (TIMEOUT_PERIOD_MINUTES + 5) * 60 -class BUILD_JOB_RESULT(object): +class BuildJobResult(object): """ Build job result enum """ INCOMPLETE = 'incomplete' COMPLETE = 'complete' @@ -29,20 +28,22 @@ class BuilderServer(object): """ Server which handles both HTTP and WAMP requests, managing the full state of the build controller. """ - _loop = None - _current_status = 'starting' - _current_components = [] - _job_count = 0 - def __init__(self, server_hostname, queue, build_logs, user_files, lifecycle_manager_klass): - self._session_factory = RouterSessionFactory(RouterFactory()) + self._loop = None + self._current_status = 'starting' + self._current_components = [] + self._job_count = 0 + self._session_factory = RouterSessionFactory(RouterFactory()) self._server_hostname = server_hostname self._queue = queue self._build_logs = build_logs self._user_files = user_files self._lifecycle_manager = lifecycle_manager_klass( - self._register_component, self._unregister_component, self._job_complete) + self._register_component, + self._unregister_component, + self._job_complete + ) self._shutdown_event = Event() self._current_status = 'running' @@ -69,11 +70,11 @@ class BuilderServer(object): logging.debug('Starting server on port 8080, with controller on port 8181') try: - loop.run_forever() + loop.run_forever() except KeyboardInterrupt: - pass + pass finally: - loop.close() + loop.close() def close(self): logging.debug('Requested server shutdown') @@ -88,7 +89,7 @@ class BuilderServer(object): """ logging.debug('Registering component with realm %s', realm) - component = component_klass(types.ComponentConfig(realm = realm), realm=realm, **kwargs) + component = component_klass(types.ComponentConfig(realm=realm), realm=realm, **kwargs) component.server = self component.parent_manager = self._lifecycle_manager component.build_logs = self._build_logs @@ -101,15 +102,15 @@ class BuilderServer(object): def _unregister_component(self, component): logging.debug('Unregistering component with realm %s and token %s', - component.builder_realm, component.expected_token) + component.builder_realm, component.expected_token) self._current_components.remove(component) self._session_factory.remove(component) def _job_complete(self, build_job, job_status): - if job_status == BUILD_JOB_RESULT.INCOMPLETE: + if job_status == BuildJobResult.INCOMPLETE: self._queue.incomplete(build_job.job_item(), restore_retry=True, retry_after=30) - elif job_status == BUILD_JOB_RESULT.ERROR: + elif job_status == BuildJobResult.ERROR: self._queue.incomplete(build_job.job_item(), restore_retry=False) else: self._queue.complete(build_job.job_item()) @@ -119,42 +120,42 @@ class BuilderServer(object): if self._current_status == 'shutting_down' and not self._job_count: self._shutdown_event.set() - # TODO: check for work here? + # TODO:(jschorr) check for work here? @trollius.coroutine def _work_checker(self): while self._current_status == 'running': - logger.debug('Checking for more work') + LOGGER.debug('Checking for more work') job_item = self._queue.get(processing_time=RESERVATION_SECONDS) if job_item is None: - logger.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT) + LOGGER.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT) yield From(trollius.sleep(WORK_CHECK_TIMEOUT)) continue try: build_job = BuildJob(job_item) except BuildJobLoadException as irbe: - logger.exception(irbe) + LOGGER.exception(irbe) self._queue.incomplete(job_item, restore_retry=False) - logger.debug('Build job found. Checking for an avaliable worker.') + LOGGER.debug('Build job found. Checking for an avaliable worker.') if self._lifecycle_manager.schedule(build_job, self._loop): self._job_count = self._job_count + 1 - logger.debug('Build job scheduled. Running: %s', self._job_count) + LOGGER.debug('Build job scheduled. Running: %s', self._job_count) else: - logger.debug('All workers are busy. Requeuing.') + LOGGER.debug('All workers are busy. Requeuing.') self._queue.incomplete(job_item, restore_retry=True, retry_after=0) - + yield From(trollius.sleep(WORK_CHECK_TIMEOUT)) @trollius.coroutine def _initialize(self, loop, host): self._loop = loop - + # Create the WAMP server. - transport_factory = WampWebSocketServerFactory(self._session_factory, debug_wamp = False) - transport_factory.setProtocolOptions(failByDrop = True) + transport_factory = WampWebSocketServerFactory(self._session_factory, debug_wamp=False) + transport_factory.setProtocolOptions(failByDrop=True) # Initialize the controller server and the WAMP server create_wsgi_server(self._controller_app, loop=loop, host=host, port=8181) diff --git a/buildman/workererror.py b/buildman/workererror.py index 023817e00..c5a5adbd0 100644 --- a/buildman/workererror.py +++ b/buildman/workererror.py @@ -5,59 +5,59 @@ class WorkerError(object): self._base_message = base_message self._error_handlers = { - 'io.quay.builder.buildpackissue': { - 'message': 'Could not load build package', - 'is_internal': True - }, + 'io.quay.builder.buildpackissue': { + 'message': 'Could not load build package', + 'is_internal': True + }, - 'io.quay.builder.cannotextractbuildpack': { - 'message': 'Could not extract the contents of the build package' - }, + 'io.quay.builder.cannotextractbuildpack': { + 'message': 'Could not extract the contents of the build package' + }, - 'io.quay.builder.cannotpullforcache': { - 'message': 'Could not pull cached image', - 'is_internal': True - }, + 'io.quay.builder.cannotpullforcache': { + 'message': 'Could not pull cached image', + 'is_internal': True + }, - 'io.quay.builder.cannotpullbaseimage': { - 'message': 'Could not pull base image', - 'show_base_error': True - }, + 'io.quay.builder.cannotpullbaseimage': { + 'message': 'Could not pull base image', + 'show_base_error': True + }, - 'io.quay.builder.internalerror': { - 'message': 'An internal error occurred while building. Please submit a ticket.' - }, + 'io.quay.builder.internalerror': { + 'message': 'An internal error occurred while building. Please submit a ticket.' + }, - 'io.quay.builder.buildrunerror': { - 'message': 'Could not start the build process', - 'is_internal': True - }, + 'io.quay.builder.buildrunerror': { + 'message': 'Could not start the build process', + 'is_internal': True + }, - 'io.quay.builder.builderror': { - 'message': 'A build step failed', - 'show_base_error': True - }, + 'io.quay.builder.builderror': { + 'message': 'A build step failed', + 'show_base_error': True + }, - 'io.quay.builder.tagissue': { - 'message': 'Could not tag built image', - 'is_internal': True - }, + 'io.quay.builder.tagissue': { + 'message': 'Could not tag built image', + 'is_internal': True + }, - 'io.quay.builder.pushissue': { - 'message': 'Could not push built image', - 'show_base_error': True, - 'is_internal': True - }, + 'io.quay.builder.pushissue': { + 'message': 'Could not push built image', + 'show_base_error': True, + 'is_internal': True + }, - 'io.quay.builder.dockerconnecterror': { - 'message': 'Could not connect to Docker daemon', - 'is_internal': True - }, + 'io.quay.builder.dockerconnecterror': { + 'message': 'Could not connect to Docker daemon', + 'is_internal': True + }, - 'io.quay.builder.missingorinvalidargument': { - 'message': 'Missing required arguments for builder', - 'is_internal': True - } + 'io.quay.builder.missingorinvalidargument': { + 'message': 'Missing required arguments for builder', + 'is_internal': True + } } def is_internal_error(self): @@ -81,7 +81,7 @@ class WorkerError(object): def extra_data(self): if self._base_message: return { - 'base_error': self._base_message + 'base_error': self._base_message } - return {} \ No newline at end of file + return {}