Move the source dockerfile processing to a modular system. Move actual build to a thread. Wire up the status API. Fix a bug with the way current_image was computed. Add untested support for raw dockerfile upload.
This commit is contained in:
yackob03
parent
cd28ae6b1a
commit
5d5d027cc0
3 changed files with 143 additions and 64 deletions
|
|
@ -4,9 +4,11 @@ import shutil
|
|||
import os
|
||||
import re
|
||||
|
||||
from flask import Flask, request, send_file, make_response, jsonify
|
||||
from flask import Flask, request, send_file, jsonify, redirect, url_for, abort
|
||||
from zipfile import ZipFile
|
||||
from tempfile import TemporaryFile, mkdtemp
|
||||
from uuid import uuid4
|
||||
from multiprocessing.pool import ThreadPool
|
||||
|
||||
|
||||
BUFFER_SIZE = 8 * 1024
|
||||
|
|
@ -17,97 +19,173 @@ app = Flask(__name__)
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def count_steps(dockerfileobj):
|
||||
steps = 0
|
||||
for line in dockerfileobj.readlines():
|
||||
stripped = line.strip()
|
||||
if stripped and stripped[0] is not '#':
|
||||
steps += 1
|
||||
return steps
|
||||
|
||||
@app.route('/')
|
||||
def index():
|
||||
return send_file('test.html')
|
||||
|
||||
|
||||
@app.route('/start/zip/<path:tag_name>', methods=['POST'])
|
||||
def start_build(tag_name):
|
||||
docker_zip = request.files['dockerfile']
|
||||
def count_steps(dockerfile_path):
|
||||
with open(dockerfile_path, 'r') as dockerfileobj:
|
||||
steps = 0
|
||||
for line in dockerfileobj.readlines():
|
||||
stripped = line.strip()
|
||||
if stripped and stripped[0] is not '#':
|
||||
steps += 1
|
||||
return steps
|
||||
|
||||
|
||||
def prepare_zip(request_file):
|
||||
build_dir = mkdtemp(prefix='docker-build-')
|
||||
|
||||
# Save the zip file to temp somewhere
|
||||
with TemporaryFile() as zip_file:
|
||||
docker_zip.save(zip_file)
|
||||
request_file.save(zip_file)
|
||||
to_extract = ZipFile(zip_file)
|
||||
to_extract.extractall(build_dir)
|
||||
|
||||
docker_cl = docker.Client(version='1.5')
|
||||
return build_dir
|
||||
|
||||
build_status = docker_cl.build(path=build_dir, tag=tag_name)
|
||||
|
||||
def prepare_dockerfile(request_file):
|
||||
build_dir = mkdtemp(prefix='docker-build-')
|
||||
dockerfile_path = os.path.join(build_dir, "Dockerfile")
|
||||
with open(dockerfile_path, 'r') as dockerfileobj:
|
||||
num_steps = count_steps(dockerfileobj)
|
||||
logger.debug('Dockerfile had %s steps' % num_steps)
|
||||
request_file.save(dockerfile_path)
|
||||
|
||||
current_step = 0
|
||||
built_image = None
|
||||
for status in build_status:
|
||||
step_increment = re.search(r'Step ([0-9]+) :', status)
|
||||
if step_increment:
|
||||
current_step = int(step_increment.group(1))
|
||||
logger.debug('Step now: %s/%s' % (current_step, num_steps))
|
||||
continue
|
||||
return build_dir
|
||||
|
||||
complete = re.match(r'Successfully built ([a-z0-9]+)', status)
|
||||
if complete:
|
||||
built_image = complete.group(1)
|
||||
logger.debug('Final image ID is: %s' % built_image)
|
||||
continue
|
||||
|
||||
shutil.rmtree(build_dir)
|
||||
MIME_PROCESSORS = {
|
||||
'application/zip': prepare_zip,
|
||||
'text/plain': prepare_dockerfile,
|
||||
}
|
||||
|
||||
# Get the image count
|
||||
if not built_image:
|
||||
abort(500)
|
||||
|
||||
history = docker_cl.history(built_image)
|
||||
num_images = len(history)
|
||||
builds = {}
|
||||
pool = ThreadPool(1)
|
||||
|
||||
logger.debug('Pushing to tag name: %s' % tag_name)
|
||||
resp = docker_cl.push(tag_name)
|
||||
|
||||
current_image = 0
|
||||
image_progress = 0
|
||||
for status in resp:
|
||||
if u'status' in status:
|
||||
status_msg = status[u'status']
|
||||
def build_image(build_dir, tag_name, num_steps, result_object):
|
||||
try:
|
||||
logger.debug('Does this show up?')
|
||||
docker_cl = docker.Client(version='1.5')
|
||||
result_object['status'] = 'building'
|
||||
build_status = docker_cl.build(path=build_dir, tag=tag_name)
|
||||
|
||||
next_image = r'(Pushing:|Image) [a-z0-9]+( already pushed, skipping)?'
|
||||
match = re.match(next_image, status_msg)
|
||||
if match:
|
||||
current_image += 1
|
||||
image_progress = 0
|
||||
logger.debug('Now pushing image %s/%s' % (current_image, num_images))
|
||||
current_step = 0
|
||||
built_image = None
|
||||
for status in build_status:
|
||||
step_increment = re.search(r'Step ([0-9]+) :', status)
|
||||
if step_increment:
|
||||
current_step = int(step_increment.group(1))
|
||||
logger.debug('Step now: %s/%s' % (current_step, num_steps))
|
||||
result_object['current_command'] = current_step
|
||||
continue
|
||||
|
||||
if status_msg == u'Pushing' and u'progress' in status:
|
||||
percent = r'\(([0-9]+)%\)'
|
||||
match = re.search(percent, status[u'progress'])
|
||||
complete = re.match(r'Successfully built ([a-z0-9]+)$', status)
|
||||
if complete:
|
||||
built_image = complete.group(1)
|
||||
logger.debug('Final image ID is: %s' % built_image)
|
||||
continue
|
||||
|
||||
shutil.rmtree(build_dir)
|
||||
|
||||
# Get the image count
|
||||
if not built_image:
|
||||
result_object['status'] = 'error'
|
||||
result_object['message'] = 'Unable to build dockerfile.'
|
||||
return
|
||||
|
||||
history = docker_cl.history(built_image)
|
||||
num_images = len(history)
|
||||
result_object['total_images'] = num_images
|
||||
|
||||
result_object['status'] = 'pushing'
|
||||
logger.debug('Pushing to tag name: %s' % tag_name)
|
||||
resp = docker_cl.push(tag_name)
|
||||
|
||||
current_image = 0
|
||||
image_progress = 0
|
||||
for status in resp:
|
||||
logger.debug(status)
|
||||
|
||||
if u'status' in status:
|
||||
status_msg = status[u'status']
|
||||
|
||||
next_image = r'(Pushing|Image) [a-z0-9]+( already pushed, skipping)?$'
|
||||
match = re.match(next_image, status_msg)
|
||||
if match:
|
||||
image_progress = int(match.group(1))
|
||||
continue
|
||||
current_image += 1
|
||||
image_progress = 0
|
||||
logger.debug('Now pushing image %s/%s' %
|
||||
(current_image, num_images))
|
||||
|
||||
return jsonify({
|
||||
'images_pushed': num_images,
|
||||
'commands_run': num_steps,
|
||||
})
|
||||
elif status_msg == u'Pushing' and u'progress' in status:
|
||||
percent = r'\(([0-9]+)%\)'
|
||||
match = re.search(percent, status[u'progress'])
|
||||
if match:
|
||||
image_progress = int(match.group(1))
|
||||
|
||||
result_object['current_image'] = current_image
|
||||
result_object['image_completion_percent'] = image_progress
|
||||
|
||||
elif u'errorDetail' in status:
|
||||
result_object['status'] = 'error'
|
||||
if u'message' in status[u'errorDetail']:
|
||||
result_object['message'] = status[u'errorDetail'][u'message']
|
||||
return
|
||||
|
||||
result_object['status'] = 'complete'
|
||||
except Exception as e:
|
||||
logger.exception('Exception when processing request.')
|
||||
result_object['status'] = 'error'
|
||||
result_object['message'] = e.message
|
||||
|
||||
|
||||
@app.route('/status')
|
||||
def get_status():
|
||||
return 'building'
|
||||
@app.route('/build', methods=['POST'])
|
||||
def start_build():
|
||||
docker_input = request.files['dockerfile']
|
||||
c_type = docker_input.content_type
|
||||
tag_name = request.values['tag']
|
||||
|
||||
logger.info('Request to build file of type: %s with tag: %s' %
|
||||
(c_type, tag_name))
|
||||
|
||||
if c_type not in MIME_PROCESSORS:
|
||||
logger.error('Invalid dockerfile content type: %s' % c_type)
|
||||
abort(400)
|
||||
|
||||
build_dir = MIME_PROCESSORS[c_type](docker_input)
|
||||
|
||||
dockerfile_path = os.path.join(build_dir, "Dockerfile")
|
||||
num_steps = count_steps(dockerfile_path)
|
||||
logger.debug('Dockerfile had %s steps' % num_steps)
|
||||
|
||||
job_id = str(uuid4())
|
||||
logger.info('Sending job to builder pool: %s' % job_id)
|
||||
|
||||
result_object = {
|
||||
'id': job_id,
|
||||
'total_commands': num_steps,
|
||||
'total_images': None,
|
||||
'current_command': 0,
|
||||
'current_image': 0,
|
||||
'image_completion_percent': 0,
|
||||
'status': 'waiting',
|
||||
'message': None,
|
||||
}
|
||||
builds[job_id] = result_object
|
||||
pool.apply_async(build_image, [build_dir, tag_name, num_steps,
|
||||
result_object])
|
||||
|
||||
return redirect(url_for('get_status', job_id=job_id))
|
||||
|
||||
|
||||
@app.route('/status/<job_id>')
|
||||
def get_status(job_id):
|
||||
if job_id not in builds:
|
||||
abort(400)
|
||||
|
||||
return jsonify(builds[job_id])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -1,2 +1,2 @@
|
|||
flask
|
||||
-e git+git://github.com/dotcloud/docker-py.git#egg=docker-py
|
||||
-e git+git://github.com/DevTable/docker-py.git#egg=docker-py
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
<html>
|
||||
<body>
|
||||
<form enctype='multipart/form-data' method="post" action="/start/zip/localhost:5000/devtable/newtag">
|
||||
<form enctype='multipart/form-data' method="post" action="/build">
|
||||
<input type="file" name="dockerfile">
|
||||
<input type="text" name="tag">
|
||||
<button type="submit">Send</button>
|
||||
</form>
|
||||
</body>
|
||||
|
|
|
|||
Reference in a new issue