Switch to the redis backed build logs and status.
This commit is contained in:
yackob03
parent
5cc59e67a9
commit
5270066d6d
10 changed files with 292 additions and 141 deletions
|
|
@ -25,6 +25,28 @@ formatter = logging.Formatter(FORMAT)
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
user_files = app.config['USERFILES']
|
||||
build_logs = app.config['BUILDLOGS']
|
||||
|
||||
|
||||
class StatusWrapper(object):
|
||||
def __init__(self, build_uuid):
|
||||
self._uuid = build_uuid
|
||||
self._status = {
|
||||
'total_commands': None,
|
||||
'current_command': None,
|
||||
'push_completion': 0.0,
|
||||
'image_completion': {},
|
||||
}
|
||||
|
||||
self.__exit__(None, None, None)
|
||||
|
||||
def __enter__(self):
|
||||
return self._status
|
||||
|
||||
def __exit__(self, exc_type, value, traceback):
|
||||
build_logs.set_status(self._uuid, self._status)
|
||||
|
||||
|
||||
class DockerfileBuildWorker(Worker):
|
||||
def __init__(self, *vargs, **kwargs):
|
||||
|
|
@ -75,102 +97,94 @@ class DockerfileBuildWorker(Worker):
|
|||
return float(sent_bytes)/total_bytes*percentage_with_sizes
|
||||
|
||||
@staticmethod
|
||||
def __build_image(build_dir, tag_name, num_steps, result_object):
|
||||
try:
|
||||
logger.debug('Starting build.')
|
||||
docker_cl = Client(timeout=1200)
|
||||
def __build_image(build_dir, tag_name, num_steps, result_object, uuid):
|
||||
logger.debug('Starting build.')
|
||||
docker_cl = Client(timeout=1200)
|
||||
|
||||
result_object['status'] = 'building'
|
||||
build_status = docker_cl.build(path=build_dir, tag=tag_name, stream=True)
|
||||
with result_object as status:
|
||||
status['total_commands'] = num_steps
|
||||
|
||||
current_step = 0
|
||||
built_image = None
|
||||
for status in build_status:
|
||||
# logger.debug('Status: %s', str(status))
|
||||
step_increment = re.search(r'Step ([0-9]+) :', status)
|
||||
if step_increment:
|
||||
current_step = int(step_increment.group(1))
|
||||
logger.debug('Step now: %s/%s' % (current_step, num_steps))
|
||||
result_object['current_command'] = current_step
|
||||
continue
|
||||
build_status = docker_cl.build(path=build_dir, tag=tag_name, stream=True)
|
||||
|
||||
complete = re.match(r'Successfully built ([a-z0-9]+)$', status)
|
||||
if complete:
|
||||
built_image = complete.group(1)
|
||||
logger.debug('Final image ID is: %s' % built_image)
|
||||
continue
|
||||
current_step = 0
|
||||
built_image = None
|
||||
for status in build_status:
|
||||
logger.debug('Status: %s', str(status))
|
||||
build_logs.append_log_message(uuid, str(status))
|
||||
step_increment = re.search(r'Step ([0-9]+) :', status)
|
||||
if step_increment:
|
||||
current_step = int(step_increment.group(1))
|
||||
logger.debug('Step now: %s/%s' % (current_step, num_steps))
|
||||
with result_object as status:
|
||||
status['current_command'] = current_step
|
||||
continue
|
||||
|
||||
shutil.rmtree(build_dir)
|
||||
complete = re.match(r'Successfully built ([a-z0-9]+)$', status)
|
||||
if complete:
|
||||
built_image = complete.group(1)
|
||||
logger.debug('Final image ID is: %s' % built_image)
|
||||
continue
|
||||
|
||||
# Get the image count
|
||||
if not built_image:
|
||||
result_object['status'] = 'error'
|
||||
result_object['message'] = 'Unable to build dockerfile.'
|
||||
return
|
||||
shutil.rmtree(build_dir)
|
||||
|
||||
return built_image
|
||||
except Exception as exc:
|
||||
logger.exception('Exception when processing request.')
|
||||
result_object['status'] = 'error'
|
||||
result_object['message'] = str(exc.message)
|
||||
# Get the image count
|
||||
if not built_image:
|
||||
return
|
||||
|
||||
return built_image
|
||||
|
||||
@staticmethod
|
||||
def __push_image(built_image, token, tag_name, result_object):
|
||||
try:
|
||||
# Login to the registry
|
||||
host = re.match(r'([a-z0-9.:]+)/.+/.+$', tag_name)
|
||||
if not host:
|
||||
raise Exception('Invalid tag name: %s' % tag_name)
|
||||
# Login to the registry
|
||||
host = re.match(r'([a-z0-9.:]+)/.+/.+$', tag_name)
|
||||
if not host:
|
||||
raise RuntimeError('Invalid tag name: %s' % tag_name)
|
||||
|
||||
docker_cl = Client(timeout=1200)
|
||||
docker_cl = Client(timeout=1200)
|
||||
|
||||
for protocol in ['https', 'http']:
|
||||
registry_endpoint = '%s://%s/v1/' % (protocol, host.group(1))
|
||||
logger.debug('Attempting login to registry: %s' % registry_endpoint)
|
||||
for protocol in ['https', 'http']:
|
||||
registry_endpoint = '%s://%s/v1/' % (protocol, host.group(1))
|
||||
logger.debug('Attempting login to registry: %s' % registry_endpoint)
|
||||
|
||||
try:
|
||||
docker_cl.login('$token', token, registry=registry_endpoint)
|
||||
break
|
||||
except APIError:
|
||||
pass # Probably the wrong protocol
|
||||
try:
|
||||
docker_cl.login('$token', token, registry=registry_endpoint)
|
||||
break
|
||||
except APIError:
|
||||
pass # Probably the wrong protocol
|
||||
|
||||
history = json.loads(docker_cl.history(built_image))
|
||||
num_images = len(history)
|
||||
result_object['total_images'] = num_images
|
||||
history = json.loads(docker_cl.history(built_image))
|
||||
num_images = len(history)
|
||||
with result_object as status:
|
||||
status['total_images'] = num_images
|
||||
|
||||
result_object['status'] = 'pushing'
|
||||
logger.debug('Pushing to tag name: %s' % tag_name)
|
||||
resp = docker_cl.push(tag_name, stream=True)
|
||||
logger.debug('Pushing to tag name: %s' % tag_name)
|
||||
resp = docker_cl.push(tag_name, stream=True)
|
||||
|
||||
for status_str in resp:
|
||||
status = json.loads(status_str)
|
||||
logger.debug('Status: %s', status_str)
|
||||
if u'status' in status:
|
||||
status_msg = status[u'status']
|
||||
for status_str in resp:
|
||||
status = json.loads(status_str)
|
||||
logger.debug('Status: %s', status_str)
|
||||
if u'status' in status:
|
||||
status_msg = status[u'status']
|
||||
|
||||
if status_msg == 'Pushing':
|
||||
if u'progressDetail' in status and u'id' in status:
|
||||
image_id = status[u'id']
|
||||
detail = status[u'progressDetail']
|
||||
if status_msg == 'Pushing':
|
||||
if u'progressDetail' in status and u'id' in status:
|
||||
image_id = status[u'id']
|
||||
detail = status[u'progressDetail']
|
||||
|
||||
if u'current' in detail and 'total' in detail:
|
||||
images = result_object['image_completion']
|
||||
if u'current' in detail and 'total' in detail:
|
||||
with result_object as status:
|
||||
images = status['image_completion']
|
||||
|
||||
images[image_id] = detail
|
||||
result_object['push_completion'] = \
|
||||
status['push_completion'] = \
|
||||
DockerfileBuildWorker.__total_completion(images, num_images)
|
||||
|
||||
elif u'errorDetail' in status:
|
||||
result_object['status'] = 'error'
|
||||
if u'message' in status[u'errorDetail']:
|
||||
result_object['message'] = str(status[u'errorDetail'][u'message'])
|
||||
return
|
||||
elif u'errorDetail' in status:
|
||||
message = 'Error pushing image.'
|
||||
if u'message' in status[u'errorDetail']:
|
||||
message = str(status[u'errorDetail'][u'message'])
|
||||
|
||||
result_object['status'] = 'complete'
|
||||
except Exception as exc:
|
||||
logger.exception('Exception when processing request.')
|
||||
result_object['status'] = 'error'
|
||||
result_object['message'] = str(exc.message)
|
||||
raise RuntimeError(message)
|
||||
|
||||
@staticmethod
|
||||
def __cleanup():
|
||||
|
|
@ -215,47 +229,75 @@ class DockerfileBuildWorker(Worker):
|
|||
raise RuntimeError('Image was not removed: %s' % image['Id'])
|
||||
|
||||
def process_queue_item(self, job_details):
|
||||
repository_build = model.get_repository_build(job_details['build_id'])
|
||||
repository_build = model.get_repository_build(job_details['namespace'],
|
||||
job_details['repository'],
|
||||
job_details['build_uuid'])
|
||||
|
||||
user_files = app.config['USERFILES']
|
||||
resource_url = user_files.get_file_url(repository_build.resource_key)
|
||||
tag_name = repository_build.tag
|
||||
access_token = repository_build.access_token.code
|
||||
|
||||
feedback = {
|
||||
'total_commands': None,
|
||||
'current_command': None,
|
||||
'push_completion': 0.0,
|
||||
'status': 'waiting',
|
||||
'message': None,
|
||||
'image_completion': {},
|
||||
}
|
||||
result_object = StatusWrapper(repository_build.uuid)
|
||||
|
||||
logger.debug('Starting job with resource url: %s tag: %s and token: %s' %
|
||||
start_msg = ('Starting job with resource url: %s tag: %s and token: %s' %
|
||||
(resource_url, tag_name, access_token))
|
||||
logger.debug(start_msg)
|
||||
build_logs.append_log_message(repository_build.uuid, start_msg)
|
||||
|
||||
docker_resource = requests.get(resource_url)
|
||||
c_type = docker_resource.headers['content-type']
|
||||
|
||||
logger.info('Request to build file of type: %s with tag: %s' %
|
||||
(c_type, tag_name))
|
||||
filetype_msg = ('Request to build file of type: %s with tag: %s' %
|
||||
(c_type, tag_name))
|
||||
logger.info(filetype_msg)
|
||||
build_logs.append_log_message(repository_build.uuid, filetype_msg)
|
||||
|
||||
if c_type not in self._mime_processors:
|
||||
raise Exception('Invalid dockerfile content type: %s' % c_type)
|
||||
raise RuntimeError('Invalid dockerfile content type: %s' % c_type)
|
||||
|
||||
build_dir = self._mime_processors[c_type](docker_resource)
|
||||
|
||||
dockerfile_path = os.path.join(build_dir, "Dockerfile")
|
||||
num_steps = DockerfileBuildWorker.__count_steps(dockerfile_path)
|
||||
logger.debug('Dockerfile had %s steps' % num_steps)
|
||||
|
||||
steps_msg = 'Dockerfile had %s steps' % num_steps
|
||||
logger.debug(steps_msg)
|
||||
build_logs.append_log_message(repository_build.uuid, steps_msg)
|
||||
|
||||
built_image = DockerfileBuildWorker.__build_image(build_dir, tag_name,
|
||||
num_steps, feedback)
|
||||
uuid = repository_build.uuid
|
||||
repository_build.phase = 'building'
|
||||
repository_build.save()
|
||||
|
||||
DockerfileBuildWorker.__push_image(built_image, access_token, tag_name,
|
||||
feedback)
|
||||
try:
|
||||
built_image = DockerfileBuildWorker.__build_image(build_dir, tag_name,
|
||||
num_steps,
|
||||
result_object, uuid)
|
||||
|
||||
DockerfileBuildWorker.__cleanup()
|
||||
if not built_image:
|
||||
repository_build.phase = 'error'
|
||||
repository_build.save()
|
||||
build_logs.append_log_message(uuid, 'Unable to build dockerfile.')
|
||||
return False
|
||||
|
||||
repository_build.phase = 'pushing'
|
||||
repository_build.save()
|
||||
|
||||
DockerfileBuildWorker.__push_image(built_image, access_token, tag_name,
|
||||
result_object)
|
||||
|
||||
repository_build.phase = 'complete'
|
||||
repository_build.save()
|
||||
|
||||
# TODO turn cleanup on before pushing to prod
|
||||
# DockerfileBuildWorker.__cleanup()
|
||||
except Exception as exc:
|
||||
logger.exception('Exception when processing request.')
|
||||
repository_build.phase = 'error'
|
||||
repository_build.save()
|
||||
build_logs.append_log_message(uuid, exc.message)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
desc = 'Worker daemon to monitor dockerfile build'
|
||||
|
|
|
|||
Reference in a new issue