vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

When speaking to version 0.2-beta of the build worker, properly lookup the cached commands and see if we have a matching image/tag in the repository

Browse source
This commit is contained in:
Joseph Schorr 2014-12-11 18:03:40 +02:00
parent 81846f6a5f
commit 6601e83285
4 changed files with 123 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

81
buildman/component/buildcomponent.py
View file

@ -20,7 +20,7 @@ HEARTBEAT_DELTA = datetime.timedelta(seconds=30)
HEARTBEAT_TIMEOUT = 10
INITIAL_TIMEOUT = 25
SUPPORTED_WORKER_VERSIONS = ['0.1-beta']
SUPPORTED_WORKER_VERSIONS = ['0.1-beta', '0.2-beta']
logger = logging.getLogger(__name__)
@ -46,6 +46,7 @@ class BuildComponent(BaseComponent):
self._current_job = None
self._build_status = None
self._image_info = None
self._worker_version = None
BaseComponent.__init__(self, config, **kwargs)
@ -55,6 +56,7 @@ class BuildComponent(BaseComponent):
def onJoin(self, details):
logger.debug('Registering methods and listeners for component %s', self.builder_realm)
yield From(self.register(self._on_ready, u'io.quay.buildworker.ready'))
yield From(self.register(self._check_cache, u'io.quay.buildworker.checkcache'))
yield From(self.register(self._ping, u'io.quay.buildworker.ping'))
yield From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat'))
yield From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage'))
@ -73,42 +75,45 @@ class BuildComponent(BaseComponent):
self._set_status(ComponentStatus.BUILDING)
# Retrieve the job's buildpack.
base_image_information = {}
build_config = build_job.build_config()
# Retrieve the job's buildpack url.
buildpack_url = self.user_files.get_file_url(build_job.repo_build().resource_key,
requires_cors=False)
logger.debug('Retreiving build package: %s', buildpack_url)
buildpack = None
try:
buildpack = BuildPackage.from_url(buildpack_url)
except BuildPackageException as bpe:
self._build_failure('Could not retrieve build package', bpe)
return
# TODO(jschorr): Remove this block andthe buildpack package once we move everyone over
# to version 0.2 or higher
if self._worker_version == '0.1-beta':
logger.debug('Retreiving build package: %s', buildpack_url)
buildpack = None
try:
buildpack = BuildPackage.from_url(buildpack_url)
except BuildPackageException as bpe:
self._build_failure('Could not retrieve build package', bpe)
return
# Extract the base image information from the Dockerfile.
parsed_dockerfile = None
logger.debug('Parsing dockerfile')
# Extract the base image information from the Dockerfile.
parsed_dockerfile = None
logger.debug('Parsing dockerfile')
build_config = build_job.build_config()
try:
parsed_dockerfile = buildpack.parse_dockerfile(build_config.get('build_subdir'))
except BuildPackageException as bpe:
self._build_failure('Could not find Dockerfile in build package', bpe)
return
try:
parsed_dockerfile = buildpack.parse_dockerfile(build_config.get('build_subdir'))
except BuildPackageException as bpe:
self._build_failure('Could not find Dockerfile in build package', bpe)
return
image_and_tag_tuple = parsed_dockerfile.get_image_and_tag()
if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
self._build_failure('Missing FROM line in Dockerfile')
return
image_and_tag_tuple = parsed_dockerfile.get_image_and_tag()
if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
self._build_failure('Missing FROM line in Dockerfile')
return
base_image_information = {
'repository': image_and_tag_tuple[0],
'tag': image_and_tag_tuple[1]
}
base_image_information['repository'] = image_and_tag_tuple[0]
base_image_information['tag'] = image_and_tag_tuple[1]
# Extract the number of steps from the Dockerfile.
with self._build_status as status_dict:
status_dict['total_commands'] = len(parsed_dockerfile.commands)
# Extract the number of steps from the Dockerfile.
with self._build_status as status_dict:
status_dict['total_commands'] = len(parsed_dockerfile.commands)
# Add the pull robot information, if any.
if build_config.get('pull_credentials') is not None:
@ -128,20 +133,20 @@ class BuildComponent(BaseComponent):
# push_token: The token to use to push the built image.
# tag_names: The name(s) of the tag(s) for the newly built image.
# base_image: The image name and credentials to use to conduct the base image pull.
# repository: The repository to pull.
# tag: The tag to pull.
# repository: The repository to pull (DEPRECATED)
# tag: The tag to pull (DEPRECATED)
# username: The username for pulling the base image (if any).
# password: The password for pulling the base image (if any).
build_arguments = {
'build_package': buildpack_url,
'sub_directory': build_config.get('build_subdir', ''),
'repository': repository_name,
'registry': self.server_hostname,
'registry': '10.0.2.2:5000' or self.server_hostname,
'pull_token': build_job.repo_build().access_token.code,
'push_token': build_job.repo_build().access_token.code,
'tag_names': build_config.get('docker_tags', ['latest']),
'base_image': base_image_information,
'cached_tag': build_job.determine_cached_tag() or ''
'cached_tag': build_job.determine_cached_tag() or '' # Remove after V0.1-beta is deprecated
}
# Invoke the build.
@ -283,6 +288,15 @@ class BuildComponent(BaseComponent):
""" Ping pong. """
return 'pong'
def _check_cache(self, cache_commands, base_image_name, base_image_tag, base_image_id):
with self._build_status as status_dict:
status_dict['total_commands'] = len(cache_commands) + 1
logger.debug('Checking cache on realm %s. Base image: %s:%s (%s)', self.builder_realm,
base_image_name, base_image_tag, base_image_id)
return self._current_job.determine_cached_tag(base_image_id, cache_commands) or ''
def _on_ready(self, token, version):
if not version in SUPPORTED_WORKER_VERSIONS:
logger.warning('Build component (token "%s") is running an out-of-date version: %s', version)
@ -296,6 +310,7 @@ class BuildComponent(BaseComponent):
logger.warning('Builder token mismatch. Expected: "%s". Found: "%s"', self.expected_token, token)
return False
self._worker_version = version
self._set_status(ComponentStatus.RUNNING)
# Start the heartbeat check and updating loop.