Switch to using environment variables to pass information to the build node, this closes down a security loophole with the admin endpoint of the server.
This commit is contained in:
yackob03
parent
a313a77a6b
commit
c91b40f356
2 changed files with 76 additions and 99 deletions
|
|
@ -6,7 +6,7 @@ import re
|
|||
import requests
|
||||
import json
|
||||
|
||||
from flask import Flask, request, jsonify, url_for, abort, make_response
|
||||
from flask import Flask, jsonify, url_for, abort, make_response
|
||||
from zipfile import ZipFile
|
||||
from tempfile import TemporaryFile, mkdtemp
|
||||
from uuid import uuid4
|
||||
|
|
@ -134,16 +134,19 @@ MIME_PROCESSORS = {
|
|||
}
|
||||
|
||||
|
||||
builds = {}
|
||||
build = {
|
||||
'total_commands': None,
|
||||
'total_images': None,
|
||||
'current_command': None,
|
||||
'current_image': None,
|
||||
'image_completion_percent': None,
|
||||
'status': 'waiting',
|
||||
'message': None,
|
||||
}
|
||||
pool = ThreadPool(1)
|
||||
|
||||
|
||||
@app.route('/build/', methods=['POST'])
|
||||
def start_build():
|
||||
resource_url = request.values['resource_url']
|
||||
tag_name = request.values['tag']
|
||||
acccess_token = request.values['token']
|
||||
|
||||
def start_build(resource_url, tag_name, acccess_token):
|
||||
# Save the token
|
||||
host = re.match(r'([a-z0-9.:]+)/.+/.+$', tag_name)
|
||||
if host:
|
||||
|
|
@ -160,8 +163,7 @@ def start_build():
|
|||
dockercfg.write(json.dumps(payload))
|
||||
|
||||
else:
|
||||
logger.warning('Invalid tag name: %s' % tag_name)
|
||||
abort(400)
|
||||
raise Exception('Invalid tag name: %s' % tag_name)
|
||||
|
||||
docker_resource = requests.get(resource_url)
|
||||
c_type = docker_resource.headers['content-type']
|
||||
|
|
@ -170,8 +172,7 @@ def start_build():
|
|||
(c_type, tag_name))
|
||||
|
||||
if c_type not in MIME_PROCESSORS:
|
||||
logger.error('Invalid dockerfile content type: %s' % c_type)
|
||||
abort(400)
|
||||
raise Exception('Invalid dockerfile content type: %s' % c_type)
|
||||
|
||||
build_dir = MIME_PROCESSORS[c_type](docker_resource)
|
||||
|
||||
|
|
@ -179,43 +180,34 @@ def start_build():
|
|||
num_steps = count_steps(dockerfile_path)
|
||||
logger.debug('Dockerfile had %s steps' % num_steps)
|
||||
|
||||
job_id = str(uuid4())
|
||||
logger.info('Sending job to builder pool: %s' % job_id)
|
||||
logger.info('Sending job to builder pool.')
|
||||
build['total_commands'] = num_steps
|
||||
|
||||
result_object = {
|
||||
'id': job_id,
|
||||
'total_commands': num_steps,
|
||||
'total_images': None,
|
||||
'current_command': None,
|
||||
'current_image': None,
|
||||
'image_completion_percent': None,
|
||||
'status': 'waiting',
|
||||
'message': None,
|
||||
}
|
||||
builds[job_id] = result_object
|
||||
pool.apply_async(build_image, [build_dir, tag_name, num_steps,
|
||||
result_object])
|
||||
|
||||
resp = make_response('Created', 201)
|
||||
resp.headers['Location'] = url_for('get_status', job_id=job_id)
|
||||
return resp
|
||||
|
||||
|
||||
@app.route('/build/<job_id>')
|
||||
def get_status(job_id):
|
||||
if job_id not in builds:
|
||||
abort(400)
|
||||
|
||||
return jsonify(builds[job_id])
|
||||
build])
|
||||
|
||||
|
||||
@app.route('/build/', methods=['GET'])
|
||||
def get_all_status():
|
||||
return jsonify({
|
||||
'builds': builds,
|
||||
})
|
||||
def get_status():
|
||||
if build:
|
||||
return jsonify(build)
|
||||
abort(404)
|
||||
|
||||
|
||||
@app.route('/status/', methods=['GET'])
|
||||
def health_check():
|
||||
return make_response('Running')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
|
||||
app.run(host='0.0.0.0', port=5002, debug=True, threaded=True)
|
||||
resource_url = os.environ['RESOURCE_URL']
|
||||
tag_name = os.environ['TAG']
|
||||
acccess_token = os.environ['TOKEN']
|
||||
|
||||
logger.debug('Starting job with resource url: %s tag: %s and token: %s' %
|
||||
(resource_url, tag_name, acccess_token))
|
||||
|
||||
start_build(resource_url, tag_name, acccess_token)
|
||||
|
||||
app.run(host='0.0.0.0', port=5002)
|
||||
|
|
|
|||
|
|
@ -26,30 +26,19 @@ formatter = logging.Formatter(FORMAT)
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BUILD_SERVER_CMD = ('docker run -d -lxc-conf="lxc.aa_profile=unconfined" ' +
|
||||
'-privileged -e \'RESOURCE_URL=%s\' -e \'TAG=%s\' ' +
|
||||
'-e \'TOKEN=%s\' quay.io/quay/buildserver')
|
||||
|
||||
def try_connection(url, retries=5, period=5):
|
||||
|
||||
def retry_command(to_call, args=[], kwargs={}, retries=5, period=5):
|
||||
try:
|
||||
return requests.get(url)
|
||||
except ConnectionError as ex:
|
||||
if retries:
|
||||
logger.debug('Retrying connection to url: %s after %ss' % (url, period))
|
||||
time.sleep(period)
|
||||
return try_connection(url, retries-1, period)
|
||||
raise ex
|
||||
|
||||
|
||||
def try_connect_ssh(client, ip_addr, port, user, key_filename, retries=5,
|
||||
period=5):
|
||||
try:
|
||||
client.connect(ip_addr, port, user, look_for_keys=False,
|
||||
key_filename=key_filename)
|
||||
return to_call(*args, **kwargs)
|
||||
except Exception as ex:
|
||||
if retries:
|
||||
logger.debug('Retrying connection to ssh ip: %s:%s after %ss' %
|
||||
(ip_addr, port, period))
|
||||
logger.debug('Retrying command after %ss' % period)
|
||||
time.sleep(period)
|
||||
return try_connect_ssh(client, ip_addr, port, user, key_filename,
|
||||
retries-1, period)
|
||||
return retry_command(to_call, args, kwargs, retries-1, period)
|
||||
raise ex
|
||||
|
||||
|
||||
|
|
@ -77,11 +66,12 @@ def babysit_builder(request):
|
|||
logger.debug('Cleaning up old DO node: %s' % old_id)
|
||||
old_droplet = digitalocean.Droplet(id=old_id, client_id=do_client_id,
|
||||
api_key=do_api_key)
|
||||
old_droplet.destroy()
|
||||
retry_command(old_droplet.destroy)
|
||||
|
||||
# Pick the region for the new droplet
|
||||
allowed_regions = app.config['DO_ALLOWED_REGIONS']
|
||||
available_regions = {region.id for region in manager.get_all_regions()}
|
||||
regions = retry_command(manager.get_all_regions)
|
||||
available_regions = {region.id for region in regions}
|
||||
regions = available_regions.intersection(allowed_regions)
|
||||
if not regions:
|
||||
logger.error('No droplets in our allowed regtions, available: %s' %
|
||||
|
|
@ -98,19 +88,21 @@ def babysit_builder(request):
|
|||
image_id=1004145, # Docker on 13.04
|
||||
size_id=66, # 512MB,
|
||||
backup_active=False)
|
||||
droplet.create(ssh_key_ids=[app.config['DO_SSH_KEY_ID']])
|
||||
retry_command(droplet.create, [],
|
||||
{'ssh_key_ids': [app.config['DO_SSH_KEY_ID']]})
|
||||
repository_build.build_node_id = droplet.id
|
||||
repository_build.phase = 'starting'
|
||||
repository_build.save()
|
||||
|
||||
startup = droplet.get_events()[0]
|
||||
startup.load()
|
||||
logger.debug('Waiting for DO node to be available.')
|
||||
|
||||
startup = retry_command(droplet.get_events)[0]
|
||||
while not startup.percentage or int(startup.percentage) != 100:
|
||||
logger.debug('Droplet startup percentage: %s' % startup.percentage)
|
||||
time.sleep(5)
|
||||
startup.load()
|
||||
retry_command(startup.load)
|
||||
|
||||
droplet.load()
|
||||
retry_command(droplet.load)
|
||||
logger.debug('Droplet started at ip address: %s' % droplet.ip_address)
|
||||
|
||||
# connect to it with ssh
|
||||
|
|
@ -119,8 +111,10 @@ def babysit_builder(request):
|
|||
|
||||
ssh_client = paramiko.SSHClient()
|
||||
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
try_connect_ssh(ssh_client, droplet.ip_address, 22, 'root',
|
||||
key_filename=app.config['DO_SSH_PRIVATE_KEY_FILENAME'])
|
||||
|
||||
retry_command(ssh_client.connect, [droplet.ip_address, 22, 'root'],
|
||||
{'look_for_keys': False,
|
||||
'key_filename': app.config['DO_SSH_PRIVATE_KEY_FILENAME']})
|
||||
|
||||
# Load the node with the pull token
|
||||
token = app.config['BUILD_NODE_PULL_TOKEN']
|
||||
|
|
@ -142,6 +136,7 @@ def babysit_builder(request):
|
|||
|
||||
if pull_status != 0:
|
||||
logger.error('Pull command failed for host: %s' % droplet.ip_address)
|
||||
return False
|
||||
else:
|
||||
logger.debug('Pull status was: %s' % pull_status)
|
||||
|
||||
|
|
@ -149,56 +144,46 @@ def babysit_builder(request):
|
|||
remove_auth_cmd = 'rm .dockercfg'
|
||||
ssh_client.exec_command(remove_auth_cmd)
|
||||
|
||||
# Start the build server
|
||||
start_cmd = 'docker run -d -privileged -lxc-conf="lxc.aa_profile=unconfined" quay.io/quay/buildserver'
|
||||
ssh_client.exec_command(start_cmd)
|
||||
|
||||
# wait for the server to be ready
|
||||
logger.debug('Waiting for buildserver to be ready')
|
||||
build_endpoint = 'http://%s:5002/build/' % droplet.ip_address
|
||||
try:
|
||||
try_connection(build_endpoint)
|
||||
except ConnectionError:
|
||||
#TODO cleanup
|
||||
pass
|
||||
|
||||
# send it the job
|
||||
logger.debug('Sending build server request')
|
||||
|
||||
# Prepare the signed resource url the build node can fetch the job from
|
||||
user_files = UserRequestFiles(app.config['AWS_ACCESS_KEY'],
|
||||
app.config['AWS_SECRET_KEY'],
|
||||
app.config['REGISTRY_S3_BUCKET'])
|
||||
resource_url = user_files.get_file_url(repository_build.resource_key)
|
||||
|
||||
repo = repository_build.repository
|
||||
payload = {
|
||||
'tag': repository_build.tag,
|
||||
'resource_url': user_files.get_file_url(repository_build.resource_key),
|
||||
'token': repository_build.access_token.code,
|
||||
}
|
||||
start_build = requests.post(build_endpoint, data=payload)
|
||||
# Start the build server
|
||||
start_cmd = BUILD_SERVER_CMD % (resource_url, repository_build.tag,
|
||||
repository_build.access_token.code)
|
||||
logger.debug('Sending build server request with command: %s' % start_cmd)
|
||||
ssh_client.exec_command(start_cmd)
|
||||
|
||||
status_endpoint = 'http://%s:5002/build/' % droplet.ip_address
|
||||
# wait for the server to be ready
|
||||
logger.debug('Waiting for buildserver to be ready')
|
||||
retry_command(requests.get, [status_endpoint])
|
||||
|
||||
# wait for the job to be complete
|
||||
status_url = start_build.headers['Location']
|
||||
repository_build.phase = 'building'
|
||||
repository_build.status_url = status_url
|
||||
repository_build.status_url = status_endpoint
|
||||
repository_build.save()
|
||||
|
||||
logger.debug('Waiting for job to be complete')
|
||||
status = get_status(status_url)
|
||||
status = get_status(status_endpoint)
|
||||
while status != 'error' and status != 'complete':
|
||||
logger.debug('Job status is: %s' % status)
|
||||
time.sleep(5)
|
||||
status = get_status(status_url)
|
||||
status = get_status(status_endpoint)
|
||||
|
||||
logger.debug('Job complete with status: %s' % status)
|
||||
if status == 'error':
|
||||
error_message = requests.get(status_endpoint).json()['message']
|
||||
logger.warning('Job error: %s' % error_message)
|
||||
repository_build.phase = 'error'
|
||||
else:
|
||||
repository_build.phase = 'complete'
|
||||
|
||||
# clean up the DO node
|
||||
logger.debug('Cleaning up DO node.')
|
||||
droplet.destroy()
|
||||
# retry_command(droplet.destroy)
|
||||
|
||||
repository_build.status_url = None
|
||||
repository_build.build_node_id = None;
|
||||
|
|
|
|||
Reference in a new issue