Better organize the source file structure of the build manager and change it to choose a lifecycle manager based on the config
This commit is contained in:
parent
c48559ee3d
commit
660a640de6
12 changed files with 34 additions and 20 deletions
0
buildman/jobutil/__init__.py
Normal file
0
buildman/jobutil/__init__.py
Normal file
60
buildman/jobutil/buildjob.py
Normal file
60
buildman/jobutil/buildjob.py
Normal file
|
@ -0,0 +1,60 @@
|
|||
from data import model
|
||||
|
||||
import json
|
||||
|
||||
class BuildJobLoadException(Exception):
|
||||
""" Exception raised if a build job could not be instantiated for some reason. """
|
||||
pass
|
||||
|
||||
class BuildJob(object):
|
||||
""" Represents a single in-progress build job. """
|
||||
def __init__(self, job_item):
|
||||
self._job_item = job_item
|
||||
|
||||
try:
|
||||
self._job_details = json.loads(job_item.body)
|
||||
except ValueError:
|
||||
raise BuildJobLoadException(
|
||||
'Could not parse build queue item config with ID %s' % self._job_details['build_uuid']
|
||||
)
|
||||
|
||||
try:
|
||||
self._repo_build = model.get_repository_build(self._job_details['namespace'],
|
||||
self._job_details['repository'],
|
||||
self._job_details['build_uuid'])
|
||||
except model.InvalidRepositoryBuildException:
|
||||
raise BuildJobLoadException(
|
||||
'Could not load repository build with ID %s' % self._job_details['build_uuid'])
|
||||
|
||||
try:
|
||||
self._build_config = json.loads(self._repo_build.job_config)
|
||||
except ValueError:
|
||||
raise BuildJobLoadException(
|
||||
'Could not parse repository build job config with ID %s' % self._job_details['build_uuid']
|
||||
)
|
||||
|
||||
def determine_cached_tag(self):
|
||||
""" Returns the tag to pull to prime the cache or None if none. """
|
||||
# TODO(jschorr): Change this to use the more complicated caching rules, once we have caching
|
||||
# be a pull of things besides the constructed tags.
|
||||
tags = self._build_config.get('docker_tags', ['latest'])
|
||||
existing_tags = model.list_repository_tags(self._job_details['namespace'],
|
||||
self._job_details['repository'])
|
||||
|
||||
cached_tags = set(tags) & set([tag.name for tag in existing_tags])
|
||||
if cached_tags:
|
||||
return list(cached_tags)[0]
|
||||
|
||||
return None
|
||||
|
||||
def job_item(self):
|
||||
""" Returns the job's queue item. """
|
||||
return self._job_item
|
||||
|
||||
def repo_build(self):
|
||||
""" Returns the repository build DB row for the job. """
|
||||
return self._repo_build
|
||||
|
||||
def build_config(self):
|
||||
""" Returns the parsed repository build config for the job. """
|
||||
return self._build_config
|
88
buildman/jobutil/buildpack.py
Normal file
88
buildman/jobutil/buildpack.py
Normal file
|
@ -0,0 +1,88 @@
|
|||
import tarfile
|
||||
import requests
|
||||
import os
|
||||
|
||||
from tempfile import TemporaryFile, mkdtemp
|
||||
from zipfile import ZipFile
|
||||
from util.dockerfileparse import parse_dockerfile
|
||||
from util.safetar import safe_extractall
|
||||
|
||||
class BuildPackageException(Exception):
|
||||
""" Exception raised when retrieving or parsing a build package. """
|
||||
pass
|
||||
|
||||
|
||||
class BuildPackage(object):
|
||||
""" Helper class for easy reading and updating of a Dockerfile build pack. """
|
||||
|
||||
def __init__(self, requests_file):
|
||||
self._mime_processors = {
|
||||
'application/zip': BuildPackage._prepare_zip,
|
||||
'application/x-zip-compressed': BuildPackage._prepare_zip,
|
||||
'text/plain': BuildPackage._prepare_dockerfile,
|
||||
'application/octet-stream': BuildPackage._prepare_dockerfile,
|
||||
'application/x-tar': BuildPackage._prepare_tarball,
|
||||
'application/gzip': BuildPackage._prepare_tarball,
|
||||
'application/x-gzip': BuildPackage._prepare_tarball,
|
||||
}
|
||||
|
||||
c_type = requests_file.headers['content-type']
|
||||
c_type = c_type.split(';')[0] if ';' in c_type else c_type
|
||||
|
||||
if c_type not in self._mime_processors:
|
||||
raise BuildPackageException('Unknown build package mime type: %s' % c_type)
|
||||
|
||||
self._package_directory = None
|
||||
try:
|
||||
self._package_directory = self._mime_processors[c_type](requests_file)
|
||||
except Exception as ex:
|
||||
raise BuildPackageException(ex.message)
|
||||
|
||||
def parse_dockerfile(self, subdirectory):
|
||||
dockerfile_path = os.path.join(self._package_directory, subdirectory, 'Dockerfile')
|
||||
if not os.path.exists(dockerfile_path):
|
||||
if subdirectory:
|
||||
message = 'Build package did not contain a Dockerfile at sub directory %s.' % subdirectory
|
||||
else:
|
||||
message = 'Build package did not contain a Dockerfile at the root directory.'
|
||||
|
||||
raise BuildPackageException(message)
|
||||
|
||||
with open(dockerfile_path, 'r') as dockerfileobj:
|
||||
return parse_dockerfile(dockerfileobj.read())
|
||||
|
||||
@staticmethod
|
||||
def from_url(url):
|
||||
buildpack_resource = requests.get(url, stream=True)
|
||||
return BuildPackage(buildpack_resource)
|
||||
|
||||
@staticmethod
|
||||
def _prepare_zip(request_file):
|
||||
build_dir = mkdtemp(prefix='docker-build-')
|
||||
|
||||
# Save the zip file to temp somewhere
|
||||
with TemporaryFile() as zip_file:
|
||||
zip_file.write(request_file.content)
|
||||
to_extract = ZipFile(zip_file)
|
||||
to_extract.extractall(build_dir)
|
||||
|
||||
return build_dir
|
||||
|
||||
@staticmethod
|
||||
def _prepare_dockerfile(request_file):
|
||||
build_dir = mkdtemp(prefix='docker-build-')
|
||||
dockerfile_path = os.path.join(build_dir, "Dockerfile")
|
||||
with open(dockerfile_path, 'w') as dockerfile:
|
||||
dockerfile.write(request_file.content)
|
||||
|
||||
return build_dir
|
||||
|
||||
@staticmethod
|
||||
def _prepare_tarball(request_file):
|
||||
build_dir = mkdtemp(prefix='docker-build-')
|
||||
|
||||
# Save the zip file to temp somewhere
|
||||
with tarfile.open(mode='r|*', fileobj=request_file.raw) as tar_stream:
|
||||
safe_extractall(tar_stream, build_dir)
|
||||
|
||||
return build_dir
|
52
buildman/jobutil/buildstatus.py
Normal file
52
buildman/jobutil/buildstatus.py
Normal file
|
@ -0,0 +1,52 @@
|
|||
from data.database import BUILD_PHASE
|
||||
|
||||
class StatusHandler(object):
|
||||
""" Context wrapper for writing status to build logs. """
|
||||
|
||||
def __init__(self, build_logs, repository_build):
|
||||
self._current_phase = None
|
||||
self._repository_build = repository_build
|
||||
self._uuid = repository_build.uuid
|
||||
self._build_logs = build_logs
|
||||
|
||||
self._status = {
|
||||
'total_commands': None,
|
||||
'current_command': None,
|
||||
'push_completion': 0.0,
|
||||
'pull_completion': 0.0,
|
||||
}
|
||||
|
||||
# Write the initial status.
|
||||
self.__exit__(None, None, None)
|
||||
|
||||
def _append_log_message(self, log_message, log_type=None, log_data=None):
|
||||
self._build_logs.append_log_message(self._uuid, log_message, log_type, log_data)
|
||||
|
||||
def append_log(self, log_message, extra_data=None):
|
||||
self._append_log_message(log_message, log_data=extra_data)
|
||||
|
||||
def set_command(self, command, extra_data=None):
|
||||
self._append_log_message(command, self._build_logs.COMMAND, extra_data)
|
||||
|
||||
def set_error(self, error_message, extra_data=None, internal_error=False):
|
||||
self.set_phase(BUILD_PHASE.INTERNAL_ERROR if internal_error else BUILD_PHASE.ERROR)
|
||||
|
||||
extra_data = extra_data or {}
|
||||
extra_data['internal_error'] = internal_error
|
||||
self._append_log_message(error_message, self._build_logs.ERROR, extra_data)
|
||||
|
||||
def set_phase(self, phase, extra_data=None):
|
||||
if phase == self._current_phase:
|
||||
return False
|
||||
|
||||
self._current_phase = phase
|
||||
self._append_log_message(phase, self._build_logs.PHASE, extra_data)
|
||||
self._repository_build.phase = phase
|
||||
self._repository_build.save()
|
||||
return True
|
||||
|
||||
def __enter__(self):
|
||||
return self._status
|
||||
|
||||
def __exit__(self, exc_type, value, traceback):
|
||||
self._build_logs.set_status(self._uuid, self._status)
|
84
buildman/jobutil/workererror.py
Normal file
84
buildman/jobutil/workererror.py
Normal file
|
@ -0,0 +1,84 @@
|
|||
class WorkerError(object):
|
||||
""" Helper class which represents errors raised by a build worker. """
|
||||
def __init__(self, error_code, base_message=None):
|
||||
self._error_code = error_code
|
||||
self._base_message = base_message
|
||||
|
||||
self._error_handlers = {
|
||||
'io.quay.builder.buildpackissue': {
|
||||
'message': 'Could not load build package',
|
||||
'is_internal': True
|
||||
},
|
||||
|
||||
'io.quay.builder.cannotextractbuildpack': {
|
||||
'message': 'Could not extract the contents of the build package'
|
||||
},
|
||||
|
||||
'io.quay.builder.cannotpullforcache': {
|
||||
'message': 'Could not pull cached image',
|
||||
'is_internal': True
|
||||
},
|
||||
|
||||
'io.quay.builder.cannotpullbaseimage': {
|
||||
'message': 'Could not pull base image',
|
||||
'show_base_error': True
|
||||
},
|
||||
|
||||
'io.quay.builder.internalerror': {
|
||||
'message': 'An internal error occurred while building. Please submit a ticket.'
|
||||
},
|
||||
|
||||
'io.quay.builder.buildrunerror': {
|
||||
'message': 'Could not start the build process',
|
||||
'is_internal': True
|
||||
},
|
||||
|
||||
'io.quay.builder.builderror': {
|
||||
'message': 'A build step failed',
|
||||
'show_base_error': True
|
||||
},
|
||||
|
||||
'io.quay.builder.tagissue': {
|
||||
'message': 'Could not tag built image',
|
||||
'is_internal': True
|
||||
},
|
||||
|
||||
'io.quay.builder.pushissue': {
|
||||
'message': 'Could not push built image',
|
||||
'show_base_error': True,
|
||||
'is_internal': True
|
||||
},
|
||||
|
||||
'io.quay.builder.dockerconnecterror': {
|
||||
'message': 'Could not connect to Docker daemon',
|
||||
'is_internal': True
|
||||
},
|
||||
|
||||
'io.quay.builder.missingorinvalidargument': {
|
||||
'message': 'Missing required arguments for builder',
|
||||
'is_internal': True
|
||||
}
|
||||
}
|
||||
|
||||
def is_internal_error(self):
|
||||
handler = self._error_handlers.get(self._error_code)
|
||||
return handler.get('is_internal', False) if handler else True
|
||||
|
||||
def public_message(self):
|
||||
handler = self._error_handlers.get(self._error_code)
|
||||
if not handler:
|
||||
return 'An unknown error occurred'
|
||||
|
||||
message = handler['message']
|
||||
if handler.get('show_base_error', False) and self._base_message:
|
||||
message = message + ': ' + self._base_message
|
||||
|
||||
return message
|
||||
|
||||
def extra_data(self):
|
||||
if self._base_message:
|
||||
return {
|
||||
'base_error': self._base_message
|
||||
}
|
||||
|
||||
return {}
|
Reference in a new issue