Lint BuildManager
This commit is contained in:
Jimmy Zelinskie
parent
043a30ee96
commit
6df6f28edf
9 changed files with 187 additions and 173 deletions
|
|
@ -10,7 +10,7 @@ from trollius.coroutines import From
|
|||
from buildman.basecomponent import BaseComponent
|
||||
from buildman.buildpack import BuildPackage, BuildPackageException
|
||||
from buildman.buildstatus import StatusHandler
|
||||
from buildman.server import BUILD_JOB_RESULT
|
||||
from buildman.server import BuildJobResult
|
||||
from buildman.workererror import WorkerError
|
||||
|
||||
from data.database import BUILD_PHASE
|
||||
|
|
@ -18,9 +18,10 @@ from data.database import BUILD_PHASE
|
|||
HEARTBEAT_DELTA = datetime.timedelta(seconds=30)
|
||||
HEARTBEAT_TIMEOUT = 10
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
class COMPONENT_STATUS(object):
|
||||
class ComponentStatus(object):
|
||||
""" ComponentStatus represents the possible states of a component. """
|
||||
JOINING = 'joining'
|
||||
WAITING = 'waiting'
|
||||
RUNNING = 'running'
|
||||
|
|
@ -34,7 +35,7 @@ class BuildComponent(BaseComponent):
|
|||
expected_token = None
|
||||
builder_realm = None
|
||||
|
||||
_component_status = COMPONENT_STATUS.JOINING
|
||||
_component_status = ComponentStatus.JOINING
|
||||
_last_heartbeat = None
|
||||
_current_job = None
|
||||
_build_status = None
|
||||
|
|
@ -46,33 +47,35 @@ class BuildComponent(BaseComponent):
|
|||
|
||||
BaseComponent.__init__(self, config, **kwargs)
|
||||
|
||||
def onConnect(self):
|
||||
def onConnect(self):
|
||||
self.join(self.builder_realm)
|
||||
|
||||
def onJoin(self, details):
|
||||
logger.debug('Registering methods and listeners for component %s' % self.builder_realm)
|
||||
LOGGER.debug('Registering methods and listeners for component %s', self.builder_realm)
|
||||
yield From(self.register(self._on_ready, u'io.quay.buildworker.ready'))
|
||||
yield From(self.register(self._ping, u'io.quay.buildworker.ping'))
|
||||
yield From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat'))
|
||||
yield From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage'))
|
||||
|
||||
self._set_status(COMPONENT_STATUS.WAITING)
|
||||
self._set_status(ComponentStatus.WAITING)
|
||||
|
||||
def is_ready(self):
|
||||
return self._component_status == COMPONENT_STATUS.RUNNING
|
||||
""" Determines whether a build component is ready to begin a build. """
|
||||
return self._component_status == ComponentStatus.RUNNING
|
||||
|
||||
def start_build(self, build_job):
|
||||
def start_build(self, build_job):
|
||||
""" Starts a build. """
|
||||
self._current_job = build_job
|
||||
self._build_status = StatusHandler(self.build_logs, build_job.repo_build())
|
||||
self._image_info = {}
|
||||
|
||||
self._set_status(COMPONENT_STATUS.BUILDING)
|
||||
self._set_status(ComponentStatus.BUILDING)
|
||||
|
||||
# Retrieve the job's buildpack.
|
||||
buildpack_url = self.user_files.get_file_url(build_job.repo_build().resource_key,
|
||||
requires_cors=False)
|
||||
|
||||
logger.debug('Retreiving build package: %s' % buildpack_url)
|
||||
LOGGER.debug('Retreiving build package: %s', buildpack_url)
|
||||
buildpack = None
|
||||
try:
|
||||
buildpack = BuildPackage.from_url(buildpack_url)
|
||||
|
|
@ -82,7 +85,7 @@ class BuildComponent(BaseComponent):
|
|||
|
||||
# Extract the base image information from the Dockerfile.
|
||||
parsed_dockerfile = None
|
||||
logger.debug('Parsing dockerfile')
|
||||
LOGGER.debug('Parsing dockerfile')
|
||||
|
||||
build_config = build_job.build_config()
|
||||
try:
|
||||
|
|
@ -94,11 +97,11 @@ class BuildComponent(BaseComponent):
|
|||
image_and_tag_tuple = parsed_dockerfile.get_image_and_tag()
|
||||
if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
|
||||
self._build_failure('Missing FROM line in Dockerfile')
|
||||
return
|
||||
return
|
||||
|
||||
base_image_information = {
|
||||
'repository': image_and_tag_tuple[0],
|
||||
'tag': image_and_tag_tuple[1]
|
||||
'repository': image_and_tag_tuple[0],
|
||||
'tag': image_and_tag_tuple[1]
|
||||
}
|
||||
|
||||
# Extract the number of steps from the Dockerfile.
|
||||
|
|
@ -115,46 +118,47 @@ class BuildComponent(BaseComponent):
|
|||
repository_name = repo.namespace_user.username + '/' + repo.name
|
||||
|
||||
# Parse the build queue item into build arguments.
|
||||
# build_package: URL to the build package to download and untar/unzip.
|
||||
# sub_directory: The location within the build package of the Dockerfile and the build context.
|
||||
# repository: The repository for which this build is occurring.
|
||||
# registry: The registry for which this build is occuring. Example: 'quay.io', 'staging.quay.io'
|
||||
# pull_token: The token to use when pulling the cache for building.
|
||||
# push_token: The token to use to push the built image.
|
||||
# tag_names: The name(s) of the tag(s) for the newly built image.
|
||||
# base_image: The image name and credentials to use to conduct the base image pull.
|
||||
# repository: The repository to pull.
|
||||
# tag: The tag to pull.
|
||||
# username: The username for pulling the base image (if any).
|
||||
# password: The password for pulling the base image (if any).
|
||||
# build_package: URL to the build package to download and untar/unzip.
|
||||
# sub_directory: The location within the build package of the Dockerfile and the build context.
|
||||
# repository: The repository for which this build is occurring.
|
||||
# registry: The registry for which this build is occuring (e.g. 'quay.io', 'staging.quay.io').
|
||||
# pull_token: The token to use when pulling the cache for building.
|
||||
# push_token: The token to use to push the built image.
|
||||
# tag_names: The name(s) of the tag(s) for the newly built image.
|
||||
# base_image: The image name and credentials to use to conduct the base image pull.
|
||||
# repository: The repository to pull.
|
||||
# tag: The tag to pull.
|
||||
# username: The username for pulling the base image (if any).
|
||||
# password: The password for pulling the base image (if any).
|
||||
build_arguments = {
|
||||
'build_package': buildpack_url,
|
||||
'sub_directory': build_config.get('build_subdir', ''),
|
||||
'repository': repository_name,
|
||||
'registry': self.server_hostname,
|
||||
'pull_token': build_job.repo_build().access_token.code,
|
||||
'push_token': build_job.repo_build().access_token.code,
|
||||
'tag_names': build_config.get('docker_tags', ['latest']),
|
||||
'base_image': base_image_information,
|
||||
'cached_tag': build_job.determine_cached_tag() or ''
|
||||
'build_package': buildpack_url,
|
||||
'sub_directory': build_config.get('build_subdir', ''),
|
||||
'repository': repository_name,
|
||||
'registry': self.server_hostname,
|
||||
'pull_token': build_job.repo_build().access_token.code,
|
||||
'push_token': build_job.repo_build().access_token.code,
|
||||
'tag_names': build_config.get('docker_tags', ['latest']),
|
||||
'base_image': base_image_information,
|
||||
'cached_tag': build_job.determine_cached_tag() or ''
|
||||
}
|
||||
|
||||
# Invoke the build.
|
||||
logger.debug('Invoking build: %s', self.builder_realm)
|
||||
logger.debug('With Arguments: %s', build_arguments)
|
||||
LOGGER.debug('Invoking build: %s', self.builder_realm)
|
||||
LOGGER.debug('With Arguments: %s', build_arguments)
|
||||
|
||||
return (self.call("io.quay.builder.build", **build_arguments)
|
||||
.add_done_callback(self._build_complete))
|
||||
.add_done_callback(self._build_complete))
|
||||
|
||||
@staticmethod
|
||||
def __total_completion(statuses, total_images):
|
||||
def _total_completion(statuses, total_images):
|
||||
""" Returns the current amount completion relative to the total completion of a build. """
|
||||
percentage_with_sizes = float(len(statuses.values())) / total_images
|
||||
sent_bytes = sum([status['current'] for status in statuses.values()])
|
||||
total_bytes = sum([status['total'] for status in statuses.values()])
|
||||
return float(sent_bytes) / total_bytes * percentage_with_sizes
|
||||
|
||||
@staticmethod
|
||||
def __process_pushpull_status(status_dict, current_phase, docker_data, images):
|
||||
def _process_pushpull_status(status_dict, current_phase, docker_data, images):
|
||||
if not docker_data:
|
||||
return
|
||||
|
||||
|
|
@ -178,7 +182,7 @@ class BuildComponent(BaseComponent):
|
|||
if 'current' in detail and 'total' in detail:
|
||||
images[image_id] = detail
|
||||
status_dict[status_completion_key] = \
|
||||
BuildComponent.__total_completion(images, max(len(images), num_images))
|
||||
BuildComponent._total_completion(images, max(len(images), num_images))
|
||||
|
||||
def _on_log_message(self, phase, json_data):
|
||||
# Parse any of the JSON data logged.
|
||||
|
|
@ -209,8 +213,8 @@ class BuildComponent(BaseComponent):
|
|||
# Parse and update the phase and the status_dict. The status dictionary contains
|
||||
# the pull/push progress, as well as the current step index.
|
||||
with self._build_status as status_dict:
|
||||
self._build_status.set_phase(phase)
|
||||
BuildComponent.__process_pushpull_status(status_dict, phase, docker_data, self._image_info)
|
||||
self._build_status.set_phase(phase)
|
||||
BuildComponent._process_pushpull_status(status_dict, phase, docker_data, self._image_info)
|
||||
|
||||
# If the current message represents the beginning of a new step, then update the
|
||||
# current command index.
|
||||
|
|
@ -228,104 +232,110 @@ class BuildComponent(BaseComponent):
|
|||
|
||||
|
||||
def _build_failure(self, error_message, exception=None):
|
||||
""" Handles and logs a failed build. """
|
||||
self._build_status.set_error(error_message, {
|
||||
'internal_error': exception.message if exception else None
|
||||
'internal_error': exception.message if exception else None
|
||||
})
|
||||
|
||||
build_id = self._current_job.repo_build().uuid
|
||||
logger.warning('Build %s failed with message: %s', build_id, self._error_message)
|
||||
LOGGER.warning('Build %s failed with message: %s', build_id, error_message)
|
||||
|
||||
# Mark that the build has finished (in an error state)
|
||||
self._build_finished(BUILD_JOB_RESULT.ERROR)
|
||||
self._build_finished(BuildJobResult.ERROR)
|
||||
|
||||
def _build_complete(self, result):
|
||||
""" Wraps up a completed build. Handles any errors and calls self._build_finished. """
|
||||
try:
|
||||
# Retrieve the result. This will raise an ApplicationError on any error that occurred.
|
||||
result.result()
|
||||
self._build_status.set_phase(BUILD_PHASE.COMPLETE)
|
||||
self._build_finished(BUILD_JOB_RESULT.COMPLETE)
|
||||
except ApplicationError as ae:
|
||||
worker_error = WorkerError(ae.error, ae.kwargs.get('base_error'))
|
||||
self._build_finished(BuildJobResult.COMPLETE)
|
||||
except ApplicationError as aex:
|
||||
worker_error = WorkerError(aex.error, aex.kwargs.get('base_error'))
|
||||
|
||||
# Write the error to the log.
|
||||
self._build_status.set_error(worker_error.public_message(), worker_error.extra_data())
|
||||
|
||||
# Mark the build as completed.
|
||||
if worker_error.is_internal_error():
|
||||
self._build_finished(BUILD_JOB_RESULT.INCOMPLETE)
|
||||
self._build_finished(BuildJobResult.INCOMPLETE)
|
||||
else:
|
||||
self._build_finished(BUILD_JOB_RESULT.ERROR)
|
||||
self._build_finished(BuildJobResult.ERROR)
|
||||
|
||||
def _build_finished(self, job_status):
|
||||
""" Alerts the parent that a build has completed and sets the status back to running. """
|
||||
self.parent_manager.job_completed(self._current_job, job_status, self)
|
||||
self._current_job = None
|
||||
|
||||
# Set the component back to a running state.
|
||||
self._set_status(COMPONENT_STATUS.RUNNING)
|
||||
self._set_status(ComponentStatus.RUNNING)
|
||||
|
||||
def _ping(self):
|
||||
def _ping():
|
||||
""" Ping pong. """
|
||||
return 'pong'
|
||||
|
||||
def _on_ready(self, token):
|
||||
if self._component_status != 'waiting':
|
||||
logger.warning('Build component with token %s is already connected', self.expected_token)
|
||||
LOGGER.warning('Build component with token %s is already connected', self.expected_token)
|
||||
return
|
||||
|
||||
if token != self.expected_token:
|
||||
logger.warning('Builder token mismatch. Expected: %s. Found: %s', self.expected_token, token)
|
||||
LOGGER.warning('Builder token mismatch. Expected: %s. Found: %s', self.expected_token, token)
|
||||
return
|
||||
|
||||
self._set_status(COMPONENT_STATUS.RUNNING)
|
||||
self._set_status(ComponentStatus.RUNNING)
|
||||
|
||||
# Start the heartbeat check and updating loop.
|
||||
loop = trollius.get_event_loop()
|
||||
loop.create_task(self._heartbeat(loop))
|
||||
logger.debug('Build worker %s is connected and ready' % self.builder_realm)
|
||||
loop.create_task(self._heartbeat())
|
||||
LOGGER.debug('Build worker %s is connected and ready', self.builder_realm)
|
||||
return True
|
||||
|
||||
def _set_status(self, phase):
|
||||
self._component_status = phase
|
||||
|
||||
def _on_heartbeat(self):
|
||||
""" Updates the last known heartbeat. """
|
||||
self._last_heartbeat = datetime.datetime.now()
|
||||
|
||||
def _start_heartbeat(self, loop):
|
||||
""" Begins an async loop to keep a heartbeat going with a client. """
|
||||
trollius.set_event_loop(loop)
|
||||
loop.run_until_complete(self._heartbeat())
|
||||
|
||||
@trollius.coroutine
|
||||
def _heartbeat(self, loop):
|
||||
def _heartbeat(self):
|
||||
""" Coroutine that runs every HEARTBEAT_TIMEOUT seconds, both checking the worker's heartbeat
|
||||
and updating the heartbeat in the build status dictionary (if applicable). This allows
|
||||
the build system to catch crashes from either end.
|
||||
"""
|
||||
while True:
|
||||
# If the component is no longer running or actively building, nothing more to do.
|
||||
if (self._component_status != COMPONENT_STATUS.RUNNING and
|
||||
self._component_status != COMPONENT_STATUS.BUILDING):
|
||||
if (self._component_status != ComponentStatus.RUNNING and
|
||||
self._component_status != ComponentStatus.BUILDING):
|
||||
return
|
||||
|
||||
# If there is an active build, write the heartbeat to its status.
|
||||
build_status = self._build_status
|
||||
if build_status is not None:
|
||||
with build_status as status_dict:
|
||||
status_dict['heartbeat'] = int(time.time())
|
||||
status_dict['heartbeat'] = int(datetime.time())
|
||||
|
||||
# Check the heartbeat from the worker.
|
||||
logger.debug('Checking heartbeat on realm %s', self.builder_realm)
|
||||
LOGGER.debug('Checking heartbeat on realm %s', self.builder_realm)
|
||||
if not self._last_heartbeat:
|
||||
self._timeout()
|
||||
return
|
||||
|
||||
if self._last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
|
||||
self._timeout()
|
||||
return
|
||||
return
|
||||
|
||||
yield From(trollius.sleep(HEARTBEAT_TIMEOUT))
|
||||
|
||||
def _timeout(self):
|
||||
self._set_status(COMPONENT_STATUS.TIMED_OUT)
|
||||
logger.warning('Build component %s timed out', self.expected_token)
|
||||
self._set_status(ComponentStatus.TIMED_OUT)
|
||||
LOGGER.warning('Build component %s timed out', self.expected_token)
|
||||
self._dispose(timed_out=True)
|
||||
|
||||
def _dispose(self, timed_out=False):
|
||||
|
|
@ -335,7 +345,7 @@ class BuildComponent(BaseComponent):
|
|||
if timed_out:
|
||||
self._build_status.set_error('Build worker timed out. Build has been requeued')
|
||||
|
||||
self.parent_manager.job_completed(self._current_job, BUILD_JOB_RESULT.INCOMPLETE, self)
|
||||
self.parent_manager.job_completed(self._current_job, BuildJobResult.INCOMPLETE, self)
|
||||
self._build_status = None
|
||||
self._current_job = None
|
||||
|
||||
|
|
|
|||
Reference in a new issue