Switch to using environment variables to pass information to the build node, this closes down a security loophole with the admin endpoint of the server.

buildserver/buildserver.py

@@ -6,7 +6,7 @@ import re
 import requests
 import json
 
-from flask import Flask, request, jsonify, url_for, abort, make_response
+from flask import Flask, jsonify, url_for, abort, make_response
 from zipfile import ZipFile
 from tempfile import TemporaryFile, mkdtemp
 from uuid import uuid4
@@ -134,16 +134,19 @@ MIME_PROCESSORS = {
 }
 
 
-builds = {}
+build = {
+  'total_commands': None,
+  'total_images': None,
+  'current_command': None,
+  'current_image': None,
+  'image_completion_percent': None,
+  'status': 'waiting',
+  'message': None,
+}
 pool = ThreadPool(1)
 
 
-@app.route('/build/', methods=['POST'])
+def start_build(resource_url, tag_name, acccess_token):
-def start_build():
-  resource_url = request.values['resource_url']
-  tag_name = request.values['tag']
-  acccess_token = request.values['token']
-
   # Save the token
   host = re.match(r'([a-z0-9.:]+)/.+/.+$', tag_name)
   if host:
@@ -160,8 +163,7 @@ def start_build():
       dockercfg.write(json.dumps(payload))
 
   else:
-    logger.warning('Invalid tag name: %s' % tag_name)
+    raise Exception('Invalid tag name: %s' % tag_name)
-    abort(400)
 
   docker_resource = requests.get(resource_url)
   c_type = docker_resource.headers['content-type']
@@ -170,8 +172,7 @@ def start_build():
               (c_type, tag_name))
 
   if c_type not in MIME_PROCESSORS:
-    logger.error('Invalid dockerfile content type: %s' % c_type)
+    raise Exception('Invalid dockerfile content type: %s' % c_type)
-    abort(400)
 
   build_dir = MIME_PROCESSORS[c_type](docker_resource)
 
@@ -179,43 +180,34 @@ def start_build():
   num_steps = count_steps(dockerfile_path)
   logger.debug('Dockerfile had %s steps' % num_steps)
 
-  job_id = str(uuid4())
+  logger.info('Sending job to builder pool.')
-  logger.info('Sending job to builder pool: %s' % job_id)
+  build['total_commands'] = num_steps
 
-  result_object = {
-    'id': job_id,
-    'total_commands': num_steps,
-    'total_images': None,
-    'current_command': None,
-    'current_image': None,
-    'image_completion_percent': None,
-    'status': 'waiting',
-    'message': None,
-  }
-  builds[job_id] = result_object
   pool.apply_async(build_image, [build_dir, tag_name, num_steps,
-                                 result_object])
+                                 build])
-
-  resp = make_response('Created', 201)
-  resp.headers['Location'] = url_for('get_status', job_id=job_id)
-  return resp
-
-
-@app.route('/build/<job_id>')
-def get_status(job_id):
-  if job_id not in builds:
-    abort(400)
-
-  return jsonify(builds[job_id])
 
 
 @app.route('/build/', methods=['GET'])
-def get_all_status():
+def get_status():
-  return jsonify({
+  if build:
-    'builds': builds,
+    return jsonify(build)
-  })
+  abort(404)
+
+
+@app.route('/status/', methods=['GET'])
+def health_check():
+  return make_response('Running')
 
 
 if __name__ == '__main__':
   logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
-  app.run(host='0.0.0.0', port=5002, debug=True, threaded=True)
+  resource_url = os.environ['RESOURCE_URL']
+  tag_name = os.environ['TAG']
+  acccess_token = os.environ['TOKEN']
+
+  logger.debug('Starting job with resource url: %s tag: %s and token: %s' %
+               (resource_url, tag_name, acccess_token))
+
+  start_build(resource_url, tag_name, acccess_token)
+
+  app.run(host='0.0.0.0', port=5002)

workers/dockerfilebuild.py

@@ -26,30 +26,19 @@ formatter = logging.Formatter(FORMAT)
 
 logger = logging.getLogger(__name__)
 
+BUILD_SERVER_CMD = ('docker run -d -lxc-conf="lxc.aa_profile=unconfined" ' +
+                    '-privileged -e \'RESOURCE_URL=%s\' -e \'TAG=%s\' ' +
+                    '-e \'TOKEN=%s\' quay.io/quay/buildserver')
 
-def try_connection(url, retries=5, period=5):
+
+def retry_command(to_call, args=[], kwargs={}, retries=5, period=5):
   try:
-    return requests.get(url)
+    return to_call(*args, **kwargs)
-  except ConnectionError as ex:
-    if retries:
-      logger.debug('Retrying connection to url: %s after %ss' % (url, period))
-      time.sleep(period)
-      return try_connection(url, retries-1, period)
-    raise ex
-
-
-def try_connect_ssh(client, ip_addr, port, user, key_filename, retries=5,
-                    period=5):
-  try:
-    client.connect(ip_addr, port, user, look_for_keys=False,
-                   key_filename=key_filename)
   except Exception as ex:
     if retries:
-      logger.debug('Retrying connection to ssh ip: %s:%s after %ss' %
+      logger.debug('Retrying command after %ss' % period)
-                   (ip_addr, port, period))
       time.sleep(period)
-      return try_connect_ssh(client, ip_addr, port, user, key_filename,
+      return retry_command(to_call, args, kwargs, retries-1, period)
-                             retries-1, period)
     raise ex
 
 
@@ -77,11 +66,12 @@ def babysit_builder(request):
       logger.debug('Cleaning up old DO node: %s' % old_id)
       old_droplet = digitalocean.Droplet(id=old_id, client_id=do_client_id,
                                          api_key=do_api_key)
-      old_droplet.destroy()
+      retry_command(old_droplet.destroy)
 
     # Pick the region for the new droplet
     allowed_regions = app.config['DO_ALLOWED_REGIONS']
-    available_regions = {region.id for region in manager.get_all_regions()}
+    regions = retry_command(manager.get_all_regions)
+    available_regions = {region.id for region in regions}
     regions = available_regions.intersection(allowed_regions)
     if not regions:
       logger.error('No droplets in our allowed regtions, available: %s' %
@@ -98,19 +88,21 @@ def babysit_builder(request):
                                    image_id=1004145,  # Docker on 13.04
                                    size_id=66,  # 512MB,
                                    backup_active=False)
-    droplet.create(ssh_key_ids=[app.config['DO_SSH_KEY_ID']])
+    retry_command(droplet.create, [],
+                  {'ssh_key_ids': [app.config['DO_SSH_KEY_ID']]})
     repository_build.build_node_id = droplet.id  
     repository_build.phase = 'starting'
     repository_build.save()
 
-    startup = droplet.get_events()[0]
+    logger.debug('Waiting for DO node to be available.')
-    startup.load()
+
+    startup = retry_command(droplet.get_events)[0]
     while not startup.percentage or int(startup.percentage) != 100:
       logger.debug('Droplet startup percentage: %s' % startup.percentage)
       time.sleep(5)
-      startup.load()
+      retry_command(startup.load)
 
-    droplet.load()
+    retry_command(droplet.load)
     logger.debug('Droplet started at ip address: %s' % droplet.ip_address)
 
     # connect to it with ssh
@@ -119,8 +111,10 @@ def babysit_builder(request):
 
     ssh_client = paramiko.SSHClient()
     ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-    try_connect_ssh(ssh_client, droplet.ip_address, 22, 'root',
+
-                    key_filename=app.config['DO_SSH_PRIVATE_KEY_FILENAME'])
+    retry_command(ssh_client.connect, [droplet.ip_address, 22, 'root'],
+                  {'look_for_keys': False, 
+                   'key_filename': app.config['DO_SSH_PRIVATE_KEY_FILENAME']})
 
     # Load the node with the pull token
     token = app.config['BUILD_NODE_PULL_TOKEN']
@@ -142,6 +136,7 @@ def babysit_builder(request):
 
     if pull_status != 0:
       logger.error('Pull command failed for host: %s' % droplet.ip_address)
+      return False
     else:
       logger.debug('Pull status was: %s' % pull_status)
 
@@ -149,56 +144,46 @@ def babysit_builder(request):
     remove_auth_cmd = 'rm .dockercfg'
     ssh_client.exec_command(remove_auth_cmd)
 
-    # Start the build server
+    # Prepare the signed resource url the build node can fetch the job from
-    start_cmd = 'docker run -d -privileged -lxc-conf="lxc.aa_profile=unconfined" quay.io/quay/buildserver'
-    ssh_client.exec_command(start_cmd)
-
-    # wait for the server to be ready
-    logger.debug('Waiting for buildserver to be ready')
-    build_endpoint = 'http://%s:5002/build/' % droplet.ip_address
-    try:
-      try_connection(build_endpoint)
-    except ConnectionError:
-      #TODO cleanup
-      pass
-
-    # send it the job
-    logger.debug('Sending build server request')
-
     user_files = UserRequestFiles(app.config['AWS_ACCESS_KEY'],
                                   app.config['AWS_SECRET_KEY'],
                                   app.config['REGISTRY_S3_BUCKET'])
+    resource_url = user_files.get_file_url(repository_build.resource_key)
 
-    repo = repository_build.repository
+    # Start the build server
-    payload = {
+    start_cmd = BUILD_SERVER_CMD % (resource_url, repository_build.tag,
-      'tag': repository_build.tag,
+                                    repository_build.access_token.code)
-      'resource_url': user_files.get_file_url(repository_build.resource_key),
+    logger.debug('Sending build server request with command: %s' % start_cmd)
-      'token': repository_build.access_token.code,
+    ssh_client.exec_command(start_cmd)
-    }
+
-    start_build = requests.post(build_endpoint, data=payload)
+    status_endpoint = 'http://%s:5002/build/' % droplet.ip_address
+    # wait for the server to be ready
+    logger.debug('Waiting for buildserver to be ready')
+    retry_command(requests.get, [status_endpoint])
 
     # wait for the job to be complete
-    status_url = start_build.headers['Location']
     repository_build.phase = 'building'
-    repository_build.status_url = status_url
+    repository_build.status_url = status_endpoint
     repository_build.save()
 
     logger.debug('Waiting for job to be complete')
-    status = get_status(status_url)
+    status = get_status(status_endpoint)
     while status != 'error' and status != 'complete':
       logger.debug('Job status is: %s' % status)
       time.sleep(5)
-      status = get_status(status_url)
+      status = get_status(status_endpoint)
 
     logger.debug('Job complete with status: %s' % status)
     if status == 'error':
+      error_message = requests.get(status_endpoint).json()['message']
+      logger.warning('Job error: %s' % error_message)
       repository_build.phase = 'error'
     else:
       repository_build.phase = 'complete'
 
     # clean up the DO node
     logger.debug('Cleaning up DO node.')
-    droplet.destroy()
+    # retry_command(droplet.destroy)
 
     repository_build.status_url = None
     repository_build.build_node_id = None;