From eacf3f01d2917ecea11eded6a7e53de6e53037fb Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Tue, 11 Nov 2014 18:23:15 -0500
Subject: [PATCH 01/26] WIP: Start implementation of the build
 manager/controller. This code is not yet working completely.

---
 buildman/__init__.py            |   0
 buildman/basecomponent.py       |  11 ++
 buildman/buildcomponent.py      | 212 ++++++++++++++++++++++++++++++++
 buildman/buildpack.py           |  86 +++++++++++++
 buildman/enterprise_builder.py  |  20 +++
 buildman/manager/__init__.py    |   0
 buildman/manager/basemanager.py |  36 ++++++
 buildman/manager/enterprise.py  |  65 ++++++++++
 buildman/server.py              | 146 ++++++++++++++++++++++
 9 files changed, 576 insertions(+)
 create mode 100644 buildman/__init__.py
 create mode 100644 buildman/basecomponent.py
 create mode 100644 buildman/buildcomponent.py
 create mode 100644 buildman/buildpack.py
 create mode 100644 buildman/enterprise_builder.py
 create mode 100644 buildman/manager/__init__.py
 create mode 100644 buildman/manager/basemanager.py
 create mode 100644 buildman/manager/enterprise.py
 create mode 100644 buildman/server.py

diff --git a/buildman/__init__.py b/buildman/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/buildman/basecomponent.py b/buildman/basecomponent.py
new file mode 100644
index 000000000..3cf5bc26b
--- /dev/null
+++ b/buildman/basecomponent.py
@@ -0,0 +1,11 @@
+from autobahn.asyncio.wamp import ApplicationSession
+
+class BaseComponent(ApplicationSession):
+  """ Base class for all registered component sessions in the server. """
+  server = None
+  parent_manager = None
+  build_logs = None
+  user_files = None
+
+  def __init__(self, config, **kwargs):
+    ApplicationSession.__init__(self, config)
diff --git a/buildman/buildcomponent.py b/buildman/buildcomponent.py
new file mode 100644
index 000000000..c67b4cb8b
--- /dev/null
+++ b/buildman/buildcomponent.py
@@ -0,0 +1,212 @@
+import datetime
+import logging
+import json
+import trollius
+
+from trollius.coroutines import From
+from buildman.basecomponent import BaseComponent
+
+HEARTBEAT_DELTA = datetime.timedelta(seconds=15)
+
+logger = logging.getLogger(__name__)
+
+class BuildComponent(BaseComponent):
+  """ An application session component which conducts one (or more) builds. """
+
+  server_hostname = None
+  expected_token = None
+  builder_realm = None
+  last_heartbeat = None
+
+  current_phase = 'joining'
+  current_job = None
+
+  def __init__(self, config, realm=None, token=None, **kwargs):
+    self.expected_token = token
+    self.builder_realm = realm
+
+    BaseComponent.__init__(self, config, **kwargs)
+
+  def onConnect(self):    
+    self.join(self.builder_realm)
+
+  def onJoin(self, details):
+    logger.debug('Registering methods and listeners for component %s' % self.builder_realm)
+    yield From(self.register(self._on_ready, u'io.quay.buildworker.ready'))
+    yield From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat'))
+    yield From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage'))
+
+    self._set_phase('waiting')
+
+  def is_ready(self):
+    return self.current_phase == 'running'
+
+  def start_build(self, job_item):
+    if not self.is_ready():
+      return False
+
+    self.job_item = job_item
+    self._set_phase('building')
+
+    # Parse the build job's config.
+    logger.debug('Parsing job JSON configuration block')
+    try:
+      job_config = json.loads(job_item.body)
+    except ValueError:
+      self._build_failure('Could not parse build job configuration')
+      return False
+
+    # Retrieve the job's buildpack.
+    buildpack_url = self.user_files.get_file_url(job_item.resource_key, requires_cors=False)
+    logger.debug('Retreiving build package: %s' % buildpack_url)
+
+    buildpack = None
+    try:
+      buildpack = BuildPack.from_url(buildpack_url)
+    except BuildPackageException as bpe:
+      self._build_failure('Could not retrieve build package', bpe)
+      return False
+
+    # Extract the base image information from the Dockerfile.
+    parsed_dockerfile = None
+    logger.debug('Parsing dockerfile')
+
+    try:
+      parsed_dockerfile = buildpack.parse_dockerfile(job_config.get('build_subdir'))
+    except BuildPackageException as bpe:
+      self._build_failure('Could not find Dockerfile in build package', bpe)
+      return False
+
+    image_and_tag_tuple = parsed_dockerfile.get_image_and_tag()
+    if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
+      self._build_failure('Missing FROM line Dockerfile')
+      return False      
+
+    base_image_information = {
+      'repository': image_and_tag_tuple[0],
+      'tag': image_and_tag_tuple[1]
+    }
+
+    # Add the pull robot information, if any.
+    if job_config.get('pull_credentials') is not None:
+      base_image_information['username'] = job_config['pull_credentials'].get('username', '')
+      base_image_information['password'] = job_config['pull_credentials'].get('password', '')
+
+    # Retrieve the repository's full name.
+    repo = job_config.repository
+    repository_name = repo.namespace_user.username + '/' + repo.name
+
+    # Parse the build queue item into build arguments.
+    #   build_package: URL to the build package to download and untar/unzip.
+    #   sub_directory: The location within the build package of the Dockerfile and the build context.
+    #   repository: The repository for which this build is occurring.
+    #   registry: The registry for which this build is occuring. Example: 'quay.io', 'staging.quay.io'
+    #   pull_token: The token to use when pulling the cache for building.
+    #   push_token: The token to use to push the built image.
+    #   tag_names: The name(s) of the tag(s) for the newly built image.
+    #   base_image: The image name and credentials to use to conduct the base image pull.
+    #     repository: The repository to pull.
+    #     tag: The tag to pull.
+    #     username: The username for pulling the base image (if any).
+    #     password: The password for pulling the base image (if any).
+    build_arguments = {
+      'build_package': buildpack_url,
+      'sub_directory': job_config.get('build_subdir', ''),
+      'repository': repository_name,
+      'registry': self.server_hostname,
+      'pull_token': job_item.access_token.code,
+      'push_token': job_item.access_token.code,
+      'tag_names': job_config.get('docker_tags', ['latest']),
+      'base_image': base_image_information
+    }
+
+    # Invoke the build.    
+    logger.debug('Invoking build: %s', token)
+    logger.debug('With Arguments: %s', build_arguments)
+
+    (self.call("io.quay.builder.build", **build_arguments)
+         .add_done_callback(self._build_complete))
+
+    return True
+
+  def _build_failure(self, error_message, exception=None):
+    # TODO: log this message
+    print error_kind
+    self._set_phase('running')
+
+  def _build_complete(self, result):
+    try:
+      status = result.result()
+      # TODO: log the success
+      print status
+    except ApplicationError as ae:
+      error_kind = ae.error
+      # TODO: log the error
+      print error_kind
+    finally:
+      self._set_phase('running')
+
+  def _on_ready(self, token):
+    if self.current_phase != 'waiting':
+      logger.warning('Build component with token %s is already connected', self.expected_token)
+      return
+
+    if token != self.expected_token:
+      logger.warning('Builder token mismatch. Expected: %s. Found: %s', self.expected_token, token)
+      return
+
+    self._set_phase('running')
+
+    # Start the heartbeat check.
+    loop = trollius.get_event_loop()
+    loop.create_task(self._check_heartbeat(loop))
+    logger.debug('Build worker %s is connected and ready' % self.builder_realm)
+    return True
+
+  def _on_log_message(self, status, json):
+    # TODO: log the message
+    print json
+
+  def _set_phase(self, phase):
+    self.current_phase = phase
+
+  def _on_heartbeat(self):
+    self.last_heartbeat = datetime.datetime.now()
+
+  def _start_heartbeat_check(self, loop):
+    trollius.set_event_loop(loop)
+    loop.run_until_complete(self._check_heartbeat())
+
+  @trollius.coroutine
+  def _check_heartbeat(self, loop):
+    while True:
+      if self.current_phase != 'running' or self.current_phase != 'building':
+        return
+
+      logger.debug('Checking heartbeat on realm %s and build %s',
+        self.builder_realm, self.expected_token)
+
+      if not self.last_heartbeat:
+        self._timeout()
+        return
+
+      if self.last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
+        self._timeout()
+        return      
+
+      yield From(trollius.sleep(5))
+
+  def _timeout(self):
+    self._set_phase('timeout')
+    logger.warning('Build component %s timed out', self.expected_token)
+    self._dispose(timed_out=True)
+
+  def _dispose(self, timed_out=False):
+    # If we still have a running job, then it has not completed and we need to tell the parent
+    # manager.
+    if self.job_item is not None:
+      self.parent_manager.job_completed(job_item, 'incomplete', self)
+      self.job_item = None
+
+    # Unregister the current component so that it cannot be invoked again.
+    self.parent_manager.build_component_disposed(self, timed_out)
diff --git a/buildman/buildpack.py b/buildman/buildpack.py
new file mode 100644
index 000000000..f3e0347d2
--- /dev/null
+++ b/buildman/buildpack.py
@@ -0,0 +1,86 @@
+import tarfile
+
+from tempfile import TemporaryFile, mkdtemp
+from zipfile import ZipFile
+from util.dockerfileparse import parse_dockerfile, ParsedDockerfile
+
+
+class BuildPackageException(Exception):
+  """ Exception raised when retrieving or parsing a build package. """
+  pass
+
+
+class BuildPackage(object):
+  """ Helper class for easy reading and updating of a Dockerfile build pack. """
+
+  def __init__(self, requests_file):  
+    self._mime_processors = {
+      'application/zip': BuildPack.__prepare_zip,
+      'application/x-zip-compressed': BuildPack.__prepare_zip,
+      'text/plain': BuildPack.__prepare_dockerfile,
+      'application/octet-stream': BuildPack.__prepare_dockerfile,
+      'application/x-tar': BuildPack.__prepare_tarball,
+      'application/gzip': BuildPack.__prepare_tarball,
+      'application/x-gzip': BuildPack.__prepare_tarball,
+    }
+
+    c_type = buildpack_resource.headers['content-type']
+    c_type = c_type.split(';')[0] if ';' in c_type else c_type
+
+    if c_type not in self._mime_processors:
+      raise BuildPackageException('Unknown build package mime type: %s' % c_type)
+
+    self._package_directory = None
+    try:
+      self._package_directory = self._mime_processors[c_type](requests_file)
+    except Exception as ex:
+      raise BuildPackageException(ex.message)
+
+  def parse_dockerfile(self, build_subdirectory):
+    dockerfile_path = os.path.join(self._package_directory, subdirectory, 'Dockerfile')
+    if not os.path.exists(dockerfile_path):
+      if subdirectory:
+        message = 'Build package did not contain a Dockerfile at sub directory %s.' % subdirectory
+      else:
+        message = 'Build package did not contain a Dockerfile at the root directory.'
+
+      raise BuildPackageException(message)
+
+    with open(dockerfile_path, 'r') as dockerfileobj:
+      return parse_dockerfile(dockerfileobj.read())
+
+  @classmethod
+  def from_url(url):
+    buildpack_resource = requests.get(buildpack_url, stream=True)
+    return BuildPackage(buildpack_resource, c_type)
+
+  @staticmethod
+  def __prepare_zip(request_file):
+    build_dir = mkdtemp(prefix='docker-build-')
+
+    # Save the zip file to temp somewhere
+    with TemporaryFile() as zip_file:
+      zip_file.write(request_file.content)
+      to_extract = ZipFile(zip_file)
+      to_extract.extractall(build_dir)
+
+    return build_dir
+
+  @staticmethod
+  def __prepare_dockerfile(request_file):
+    build_dir = mkdtemp(prefix='docker-build-')
+    dockerfile_path = os.path.join(build_dir, "Dockerfile")
+    with open(dockerfile_path, 'w') as dockerfile:
+      dockerfile.write(request_file.content)
+
+    return build_dir
+
+  @staticmethod
+  def __prepare_tarball(request_file):
+    build_dir = mkdtemp(prefix='docker-build-')
+
+    # Save the zip file to temp somewhere
+    with tarfile.open(mode='r|*', fileobj=request_file.raw) as tar_stream:
+      safe_extractall(tar_stream, build_dir)
+
+    return build_dir
\ No newline at end of file
diff --git a/buildman/enterprise_builder.py b/buildman/enterprise_builder.py
new file mode 100644
index 000000000..62b67a14c
--- /dev/null
+++ b/buildman/enterprise_builder.py
@@ -0,0 +1,20 @@
+import argparse
+import logging
+
+from app import app, userfiles as user_files, build_logs, dockerfile_build_queue
+
+from buildman.manager.enterprise import EnterpriseManager
+from buildman.server import BuilderServer
+
+logger = logging.getLogger(__name__)
+
+if __name__ == '__main__':
+  logging.basicConfig(level=logging.DEBUG)
+
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--host", type = str, default = "127.0.0.1", help = 'Host IP.')
+  args = parser.parse_args()
+
+  server = BuilderServer(app.config['SERVER_HOSTNAME'], dockerfile_build_queue, build_logs,
+                         user_files, EnterpriseManager)
+  server.run(args.host)  
diff --git a/buildman/manager/__init__.py b/buildman/manager/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/buildman/manager/basemanager.py b/buildman/manager/basemanager.py
new file mode 100644
index 000000000..1a5d101f7
--- /dev/null
+++ b/buildman/manager/basemanager.py
@@ -0,0 +1,36 @@
+class BaseManager(object):
+  """ Base for all worker managers. """
+  def __init__(self, register_component, unregister_component, job_complete_callback):
+    self.register_component = register_component
+    self.unregister_component = unregister_component
+    self.job_complete_callback = job_complete_callback
+
+  def shutdown(self):
+    """ Indicates that the build controller server is in a shutdown state and that no new jobs
+        or workers should be performed. Existing workers should be cleaned up once their jobs
+        have completed
+    """
+    raise NotImplementedError
+
+  def schedule(self, job_item):
+    """ Schedules a queue item to be built. Returns True if the item was properly scheduled
+        and False if all workers are busy.
+    """
+    raise NotImplementedError
+
+  def initialize(self):
+    """ Runs any initialization code for the manager. Called once the server is in a ready state.
+    """
+    raise NotImplementedError
+
+  def build_component_disposed(self, build_component, timed_out):
+    """ Method invoked whenever a build component has been disposed. The timed_out boolean indicates
+        whether the component's heartbeat timed out.
+    """
+    raise NotImplementedError
+
+  def job_completed(self, job_item, job_status, build_component):
+    """ Method invoked once a job_item has completed, in some manner. The job_status will be
+        one of: incomplete, error, complete. If incomplete, the job should be requeued.
+    """
+    raise NotImplementedError
\ No newline at end of file
diff --git a/buildman/manager/enterprise.py b/buildman/manager/enterprise.py
new file mode 100644
index 000000000..59a3ac12d
--- /dev/null
+++ b/buildman/manager/enterprise.py
@@ -0,0 +1,65 @@
+import logging
+import uuid
+
+from buildman.manager.basemanager import BaseManager
+from buildman.basecomponent import BaseComponent
+from buildman.buildcomponent import BuildComponent
+
+from trollius.coroutines import From
+
+REGISTRATION_REALM = 'registration'
+logger = logging.getLogger(__name__)
+
+class DynamicRegistrationComponent(BaseComponent):
+  """ Component session that handles dynamic registration of the builder components. """
+
+  def onConnect(self):
+    self.join(REGISTRATION_REALM)
+
+  def onJoin(self, details):
+    logger.debug('Registering registration method')
+    yield From(self.register(self._worker_register, u'io.quay.buildworker.register'))
+
+  def _worker_register(self):
+    realm = self.parent_manager.add_build_component()
+    logger.debug('Registering new build component+worker with realm %s', realm)
+    return realm
+
+
+class EnterpriseManager(BaseManager):
+  """ Build manager implementation for the Enterprise Registry. """
+  build_components = []
+  shutting_down = False
+
+  def initialize(self):
+    # Add a component which is used by build workers for dynamic registration. Unlike
+    # production, build workers in enterprise are long-lived and register dynamically.
+    self.register_component(REGISTRATION_REALM, DynamicRegistrationComponent)
+  
+  def add_build_component(self):
+    # Generate a new unique realm ID for the build worker.
+    realm = str(uuid.uuid4())
+    component = self.register_component(realm, BuildComponent, token="")
+    self.build_components.append(component)
+    return realm
+
+  def schedule(self, job_item):
+    if self.shutting_down:
+      return False
+
+    for component in self.build_components:
+      if component.is_ready():
+        component.start_build(job_item)
+        return True
+
+    return False
+
+  def shutdown(self):
+    self.shutting_down = True
+
+  def job_completed(self, job_item, job_status, build_component):
+    self.job_complete_callback(job_item, job_status)
+
+  def component_disposed(self, build_component, timed_out):
+    self.build_components.remove(build_component)
+
diff --git a/buildman/server.py b/buildman/server.py
new file mode 100644
index 000000000..abeb83a14
--- /dev/null
+++ b/buildman/server.py
@@ -0,0 +1,146 @@
+import logging
+import trollius
+
+from autobahn.asyncio.wamp import RouterFactory, RouterSessionFactory
+from autobahn.asyncio.websocket import WampWebSocketServerFactory, WampWebSocketServerProtocol
+from autobahn.wamp import types
+from autobahn.wamp.exception import ApplicationError
+
+from aiowsgi import create_server as create_wsgi_server
+from flask import Flask
+from threading import Event, Lock
+from trollius.coroutines import From
+
+logger = logging.getLogger(__name__)
+
+WORK_CHECK_TIMEOUT = 30
+TIMEOUT_PERIOD_MINUTES = 20
+RESERVATION_SECONDS = (TIMEOUT_PERIOD_MINUTES + 5) * 60
+
+class BuilderServer(object):
+  """ Server which handles both HTTP and WAMP requests, managing the full state of the build
+      controller.
+  """
+  _loop = None
+  _current_status = 'starting'
+  _current_components = []
+  _job_count = 0
+
+  def __init__(self, server_hostname, queue, build_logs, user_files, lifecycle_manager_klass):
+    self._session_factory = RouterSessionFactory(RouterFactory())
+
+    self._server_hostname = server_hostname
+    self._queue = queue
+    self._build_logs = build_logs
+    self._user_files = user_files
+    self._lifecycle_manager = lifecycle_manager_klass(
+      self._register_component, self._unregister_component, self._job_complete)    
+
+    self._shutdown_event = Event()
+    self._current_status = 'running'
+
+    self._register_controller()
+
+  def _register_controller(self):
+    controller_app = Flask('controller')
+    server = self
+
+    @controller_app.route('/status')
+    def status():
+      return server._current_status
+
+    self._controller_app = controller_app
+
+  def run(self, host):
+    logging.debug('Initializing the lifecycle manager')
+    self._lifecycle_manager.initialize()
+
+    logging.debug('Initializing all members of the event loop')
+    loop = trollius.get_event_loop()
+    trollius.Task(self._initialize(loop, host))
+
+    logging.debug('Starting server on port 8080, with controller on port 8181')
+    try:
+       loop.run_forever()
+    except KeyboardInterrupt:
+       pass
+    finally:
+       loop.close()
+
+  def close(self):
+    logging.debug('Requested server shutdown')
+    self._current_status = 'shutting_down'
+    self._lifecycle_manager.shutdown()
+    self._shutdown_event.wait()
+    logging.debug('Shutting down server')
+
+  def _register_component(self, realm, component_klass, **kwargs):
+    """ Registers a component with the server. The component_klass must derive from
+        BaseComponent.
+    """
+    logging.debug('Registering component with realm %s', realm)
+
+    component = component_klass(types.ComponentConfig(realm = realm), realm=realm, **kwargs)
+    component.server = self
+    component.parent_manager = self._lifecycle_manager
+    component.build_logs = self._build_logs
+    component.user_files = self._user_files
+    component.server_hostname = self._server_hostname
+
+    self._current_components.append(component)
+    self._session_factory.add(component)
+    return component
+
+  def _unregister_component(self, component):
+    logging.debug('Unregistering component with realm %s and token %s',
+      component.builder_realm, component.expected_token)
+
+    self._current_components.remove(component)
+    self._session_factory.remove(component)
+
+  def _job_complete(self, job_item, job_status):
+    if job_status == 'incomplete':
+      self._queue.incomplete(job_item, restore_retry=True)
+    elif job_status == 'error':
+      self._queue.incomplete(job_item, restore_retry=False)
+    else:
+      self._queue.complete(job)
+
+    self._job_count = self._job_count - 1
+
+    if self._current_status == 'shutting_down' and not self._job_count:
+      self._shutdown_event.set()
+
+  @trollius.coroutine
+  def _work_checker(self):
+    while self._current_status == 'running':
+      logger.debug('Checking for more work')      
+      job_item = self._queue.get(processing_time=RESERVATION_SECONDS)
+      if job_item is None:
+        logger.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT)
+        yield From(trollius.sleep(WORK_CHECK_TIMEOUT))
+        continue
+
+      logger.debug('Build job found. Checking for an avaliable worker.')
+      if self._lifecycle_manager.schedule(job_item):
+        self._job_count = self._job_count + 1
+        logger.debug('Build job scheduled. Running: %s', self._job_count)
+      else:
+        logger.debug('All workers are busy. Requeuing.')
+        self._queue.incomplete(job_item, restore_retry=True)
+        
+      yield From(trollius.sleep(WORK_CHECK_TIMEOUT))
+
+
+  @trollius.coroutine
+  def _initialize(self, loop, host):
+    # Create the WAMP server.
+    transport_factory = WampWebSocketServerFactory(self._session_factory, debug_wamp = False)
+    transport_factory.setProtocolOptions(failByDrop = True)
+
+    # Initialize the controller server and the WAMP server
+    create_wsgi_server(self._controller_app, loop=loop, host=host, port=8181)
+    yield From(loop.create_server(transport_factory, host, 8080))
+
+    # Initialize the work queue checker.
+    yield self._work_checker()

From f93c0a46e83cebf40b9f4e62e367d62291a28445 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Wed, 12 Nov 2014 14:03:07 -0500
Subject: [PATCH 02/26] WIP: Get everything working except logging and job
 completion

---
 buildman/buildcomponent.py      | 56 +++++++++++++++----------------
 buildman/buildjob.py            | 59 +++++++++++++++++++++++++++++++++
 buildman/buildpack.py           | 27 +++++++--------
 buildman/manager/basemanager.py |  4 +--
 buildman/manager/enterprise.py  |  8 ++---
 buildman/server.py              | 18 +++++++---
 6 files changed, 120 insertions(+), 52 deletions(-)
 create mode 100644 buildman/buildjob.py

diff --git a/buildman/buildcomponent.py b/buildman/buildcomponent.py
index c67b4cb8b..b61d0d50d 100644
--- a/buildman/buildcomponent.py
+++ b/buildman/buildcomponent.py
@@ -3,8 +3,10 @@ import logging
 import json
 import trollius
 
+from autobahn.wamp.exception import ApplicationError
 from trollius.coroutines import From
 from buildman.basecomponent import BaseComponent
+from buildman.buildpack import BuildPackage, BuildPackageException
 
 HEARTBEAT_DELTA = datetime.timedelta(seconds=15)
 
@@ -41,28 +43,21 @@ class BuildComponent(BaseComponent):
   def is_ready(self):
     return self.current_phase == 'running'
 
-  def start_build(self, job_item):
+  def start_build(self, build_job):    
     if not self.is_ready():
       return False
 
-    self.job_item = job_item
+    self.current_job = build_job
     self._set_phase('building')
 
-    # Parse the build job's config.
-    logger.debug('Parsing job JSON configuration block')
-    try:
-      job_config = json.loads(job_item.body)
-    except ValueError:
-      self._build_failure('Could not parse build job configuration')
-      return False
-
     # Retrieve the job's buildpack.
-    buildpack_url = self.user_files.get_file_url(job_item.resource_key, requires_cors=False)
-    logger.debug('Retreiving build package: %s' % buildpack_url)
+    buildpack_url = self.user_files.get_file_url(build_job.repo_build().resource_key,
+                                                 requires_cors=False)
 
+    logger.debug('Retreiving build package: %s' % buildpack_url)
     buildpack = None
     try:
-      buildpack = BuildPack.from_url(buildpack_url)
+      buildpack = BuildPackage.from_url(buildpack_url)
     except BuildPackageException as bpe:
       self._build_failure('Could not retrieve build package', bpe)
       return False
@@ -71,8 +66,9 @@ class BuildComponent(BaseComponent):
     parsed_dockerfile = None
     logger.debug('Parsing dockerfile')
 
+    build_config = build_job.build_config()
     try:
-      parsed_dockerfile = buildpack.parse_dockerfile(job_config.get('build_subdir'))
+      parsed_dockerfile = buildpack.parse_dockerfile(build_config.get('build_subdir'))
     except BuildPackageException as bpe:
       self._build_failure('Could not find Dockerfile in build package', bpe)
       return False
@@ -88,12 +84,12 @@ class BuildComponent(BaseComponent):
     }
 
     # Add the pull robot information, if any.
-    if job_config.get('pull_credentials') is not None:
-      base_image_information['username'] = job_config['pull_credentials'].get('username', '')
-      base_image_information['password'] = job_config['pull_credentials'].get('password', '')
+    if build_config.get('pull_credentials') is not None:
+      base_image_information['username'] = build_config['pull_credentials'].get('username', '')
+      base_image_information['password'] = build_config['pull_credentials'].get('password', '')
 
     # Retrieve the repository's full name.
-    repo = job_config.repository
+    repo = build_job.repo_build().repository
     repository_name = repo.namespace_user.username + '/' + repo.name
 
     # Parse the build queue item into build arguments.
@@ -111,17 +107,18 @@ class BuildComponent(BaseComponent):
     #     password: The password for pulling the base image (if any).
     build_arguments = {
       'build_package': buildpack_url,
-      'sub_directory': job_config.get('build_subdir', ''),
+      'sub_directory': build_config.get('build_subdir', ''),
       'repository': repository_name,
       'registry': self.server_hostname,
-      'pull_token': job_item.access_token.code,
-      'push_token': job_item.access_token.code,
-      'tag_names': job_config.get('docker_tags', ['latest']),
-      'base_image': base_image_information
+      'pull_token': build_job.repo_build().access_token.code,
+      'push_token': build_job.repo_build().access_token.code,
+      'tag_names': build_config.get('docker_tags', ['latest']),
+      'base_image': base_image_information,
+      'cached_tag': build_job.determine_cached_tag() or ''
     }
 
-    # Invoke the build.    
-    logger.debug('Invoking build: %s', token)
+    # Invoke the build.
+    logger.debug('Invoking build: %s', self.builder_realm)
     logger.debug('With Arguments: %s', build_arguments)
 
     (self.call("io.quay.builder.build", **build_arguments)
@@ -131,7 +128,8 @@ class BuildComponent(BaseComponent):
 
   def _build_failure(self, error_message, exception=None):
     # TODO: log this message
-    print error_kind
+    print error_message
+    print exception
     self._set_phase('running')
 
   def _build_complete(self, result):
@@ -204,9 +202,9 @@ class BuildComponent(BaseComponent):
   def _dispose(self, timed_out=False):
     # If we still have a running job, then it has not completed and we need to tell the parent
     # manager.
-    if self.job_item is not None:
-      self.parent_manager.job_completed(job_item, 'incomplete', self)
-      self.job_item = None
+    if self.current_job is not None:
+      self.parent_manager.job_completed(self.current_job, 'incomplete', self)
+      self.current_job = None
 
     # Unregister the current component so that it cannot be invoked again.
     self.parent_manager.build_component_disposed(self, timed_out)
diff --git a/buildman/buildjob.py b/buildman/buildjob.py
new file mode 100644
index 000000000..ae87b26a2
--- /dev/null
+++ b/buildman/buildjob.py
@@ -0,0 +1,59 @@
+from data import model
+
+import json
+
+class BuildJobLoadException(Exception):
+  """ Exception raised if a build job could not be instantiated for some reason. """
+  pass
+
+class BuildJob(object):
+  """ Represents a single in-progress build job. """
+
+  def __init__(self, job_item):
+    self._job_item = job_item
+
+    try:
+      self._job_details = json.loads(job_item.body)
+    except ValueError:
+      raise BuildJobLoadException(
+        'Could not parse build queue item config with ID %s' % self._job_details['build_uuid'])      
+
+    try:
+      self._repo_build = model.get_repository_build(self._job_details['namespace'],
+                                                    self._job_details['repository'],
+                                                    self._job_details['build_uuid'])
+    except model.InvalidRepositoryBuildException:
+      raise BuildJobLoadException(
+        'Could not load repository build with ID %s' % self._job_details['build_uuid'])
+
+    try:
+      self._build_config = json.loads(self._repo_build.job_config)
+    except ValueError:
+      raise BuildJobLoadException(
+        'Could not parse repository build job config with ID %s' % self._job_details['build_uuid'])      
+
+  def determine_cached_tag(self):
+    """ Returns the tag to pull to prime the cache or None if none. """
+    # TODO(jschorr): Change this to use the more complicated caching rules, once we have caching
+    # be a pull of things besides the constructed tags.
+    tags = self._build_config.get('docker_tags', ['latest'])
+    existing_tags = model.list_repository_tags(self._job_details['namespace'],
+                                               self._job_details['repository'])
+
+    cached_tags = set(tags) & set([tag.name for tag in existing_tags])
+    if cached_tags:
+      return cached_tags[0]
+
+    return None
+
+  def job_item(self):
+    """ Returns the job's queue item. """
+    return self._job_item
+
+  def repo_build(self):
+    """ Returns the repository build DB row for the job. """
+    return self._repo_build
+
+  def build_config(self):
+    """ Returns the parsed repository build config for the job. """
+    return self._build_config
\ No newline at end of file
diff --git a/buildman/buildpack.py b/buildman/buildpack.py
index f3e0347d2..62dab38e5 100644
--- a/buildman/buildpack.py
+++ b/buildman/buildpack.py
@@ -1,10 +1,11 @@
 import tarfile
+import requests
+import os
 
 from tempfile import TemporaryFile, mkdtemp
 from zipfile import ZipFile
 from util.dockerfileparse import parse_dockerfile, ParsedDockerfile
 
-
 class BuildPackageException(Exception):
   """ Exception raised when retrieving or parsing a build package. """
   pass
@@ -15,16 +16,16 @@ class BuildPackage(object):
 
   def __init__(self, requests_file):  
     self._mime_processors = {
-      'application/zip': BuildPack.__prepare_zip,
-      'application/x-zip-compressed': BuildPack.__prepare_zip,
-      'text/plain': BuildPack.__prepare_dockerfile,
-      'application/octet-stream': BuildPack.__prepare_dockerfile,
-      'application/x-tar': BuildPack.__prepare_tarball,
-      'application/gzip': BuildPack.__prepare_tarball,
-      'application/x-gzip': BuildPack.__prepare_tarball,
+      'application/zip': BuildPackage.__prepare_zip,
+      'application/x-zip-compressed': BuildPackage.__prepare_zip,
+      'text/plain': BuildPackage.__prepare_dockerfile,
+      'application/octet-stream': BuildPackage.__prepare_dockerfile,
+      'application/x-tar': BuildPackage.__prepare_tarball,
+      'application/gzip': BuildPackage.__prepare_tarball,
+      'application/x-gzip': BuildPackage.__prepare_tarball,
     }
 
-    c_type = buildpack_resource.headers['content-type']
+    c_type = requests_file.headers['content-type']
     c_type = c_type.split(';')[0] if ';' in c_type else c_type
 
     if c_type not in self._mime_processors:
@@ -36,7 +37,7 @@ class BuildPackage(object):
     except Exception as ex:
       raise BuildPackageException(ex.message)
 
-  def parse_dockerfile(self, build_subdirectory):
+  def parse_dockerfile(self, subdirectory):
     dockerfile_path = os.path.join(self._package_directory, subdirectory, 'Dockerfile')
     if not os.path.exists(dockerfile_path):
       if subdirectory:
@@ -49,10 +50,10 @@ class BuildPackage(object):
     with open(dockerfile_path, 'r') as dockerfileobj:
       return parse_dockerfile(dockerfileobj.read())
 
-  @classmethod
+  @staticmethod
   def from_url(url):
-    buildpack_resource = requests.get(buildpack_url, stream=True)
-    return BuildPackage(buildpack_resource, c_type)
+    buildpack_resource = requests.get(url, stream=True)
+    return BuildPackage(buildpack_resource)
 
   @staticmethod
   def __prepare_zip(request_file):
diff --git a/buildman/manager/basemanager.py b/buildman/manager/basemanager.py
index 1a5d101f7..7b5816c9b 100644
--- a/buildman/manager/basemanager.py
+++ b/buildman/manager/basemanager.py
@@ -12,7 +12,7 @@ class BaseManager(object):
     """
     raise NotImplementedError
 
-  def schedule(self, job_item):
+  def schedule(self, build_job):
     """ Schedules a queue item to be built. Returns True if the item was properly scheduled
         and False if all workers are busy.
     """
@@ -29,7 +29,7 @@ class BaseManager(object):
     """
     raise NotImplementedError
 
-  def job_completed(self, job_item, job_status, build_component):
+  def job_completed(self, build_job, job_status, build_component):
     """ Method invoked once a job_item has completed, in some manner. The job_status will be
         one of: incomplete, error, complete. If incomplete, the job should be requeued.
     """
diff --git a/buildman/manager/enterprise.py b/buildman/manager/enterprise.py
index 59a3ac12d..e9cd1b9a3 100644
--- a/buildman/manager/enterprise.py
+++ b/buildman/manager/enterprise.py
@@ -43,13 +43,13 @@ class EnterpriseManager(BaseManager):
     self.build_components.append(component)
     return realm
 
-  def schedule(self, job_item):
+  def schedule(self, build_job, loop):
     if self.shutting_down:
       return False
 
     for component in self.build_components:
       if component.is_ready():
-        component.start_build(job_item)
+        loop.call_soon(component.start_build, build_job)
         return True
 
     return False
@@ -57,8 +57,8 @@ class EnterpriseManager(BaseManager):
   def shutdown(self):
     self.shutting_down = True
 
-  def job_completed(self, job_item, job_status, build_component):
-    self.job_complete_callback(job_item, job_status)
+  def job_completed(self, build_job, job_status, build_component):
+    self.job_complete_callback(build_job, job_status)
 
   def component_disposed(self, build_component, timed_out):
     self.build_components.remove(build_component)
diff --git a/buildman/server.py b/buildman/server.py
index abeb83a14..2252ca117 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -11,6 +11,8 @@ from flask import Flask
 from threading import Event, Lock
 from trollius.coroutines import From
 
+from buildjob import BuildJob, BuildJobLoadException
+
 logger = logging.getLogger(__name__)
 
 WORK_CHECK_TIMEOUT = 30
@@ -98,11 +100,11 @@ class BuilderServer(object):
     self._current_components.remove(component)
     self._session_factory.remove(component)
 
-  def _job_complete(self, job_item, job_status):
+  def _job_complete(self, build_job, job_status):
     if job_status == 'incomplete':
-      self._queue.incomplete(job_item, restore_retry=True)
+      self._queue.incomplete(build_job.job_item(), restore_retry=True)
     elif job_status == 'error':
-      self._queue.incomplete(job_item, restore_retry=False)
+      self._queue.incomplete(build_job.job_item(), restore_retry=False)
     else:
       self._queue.complete(job)
 
@@ -121,8 +123,14 @@ class BuilderServer(object):
         yield From(trollius.sleep(WORK_CHECK_TIMEOUT))
         continue
 
+      try:
+        build_job = BuildJob(job_item)
+      except BuildJobLoadException as irbe:
+        logger.exception(irbe)
+        self._queue.incomplete(job_item, restore_retry=False)
+
       logger.debug('Build job found. Checking for an avaliable worker.')
-      if self._lifecycle_manager.schedule(job_item):
+      if self._lifecycle_manager.schedule(build_job, self._loop):
         self._job_count = self._job_count + 1
         logger.debug('Build job scheduled. Running: %s', self._job_count)
       else:
@@ -134,6 +142,8 @@ class BuilderServer(object):
 
   @trollius.coroutine
   def _initialize(self, loop, host):
+    self._loop = loop
+    
     # Create the WAMP server.
     transport_factory = WampWebSocketServerFactory(self._session_factory, debug_wamp = False)
     transport_factory.setProtocolOptions(failByDrop = True)

From 4322b5f81ccf02b0cb294b162dda89f73cd08449 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Thu, 13 Nov 2014 19:41:17 -0500
Subject: [PATCH 03/26] Get the new build system working for enterprise

---
 buildman/buildcomponent.py     | 203 +++++++++++++++++++++++++--------
 buildman/buildjob.py           |   2 +-
 buildman/buildstatus.py        |  49 ++++++++
 buildman/manager/enterprise.py |   2 +-
 buildman/server.py             |  20 +++-
 buildman/workererror.py        |  87 ++++++++++++++
 static/css/quay.css            |   4 +
 static/js/app.js               |   7 ++
 8 files changed, 319 insertions(+), 55 deletions(-)
 create mode 100644 buildman/buildstatus.py
 create mode 100644 buildman/workererror.py

diff --git a/buildman/buildcomponent.py b/buildman/buildcomponent.py
index b61d0d50d..a4ce3c821 100644
--- a/buildman/buildcomponent.py
+++ b/buildman/buildcomponent.py
@@ -2,11 +2,18 @@ import datetime
 import logging
 import json
 import trollius
+import re
 
 from autobahn.wamp.exception import ApplicationError
 from trollius.coroutines import From
+
 from buildman.basecomponent import BaseComponent
 from buildman.buildpack import BuildPackage, BuildPackageException
+from buildman.buildstatus import StatusHandler
+from buildman.server import BUILD_JOB_RESULT
+from buildman.workererror import WorkerError
+
+from data.database import BUILD_PHASE
 
 HEARTBEAT_DELTA = datetime.timedelta(seconds=15)
 
@@ -18,10 +25,12 @@ class BuildComponent(BaseComponent):
   server_hostname = None
   expected_token = None
   builder_realm = None
-  last_heartbeat = None
 
-  current_phase = 'joining'
-  current_job = None
+  _last_heartbeat = None
+  _component_status = 'joining'
+  _current_job = None
+  _build_status = None
+  _image_info = None
 
   def __init__(self, config, realm=None, token=None, **kwargs):
     self.expected_token = token
@@ -38,17 +47,17 @@ class BuildComponent(BaseComponent):
     yield From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat'))
     yield From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage'))
 
-    self._set_phase('waiting')
+    self._set_status('waiting')
 
   def is_ready(self):
-    return self.current_phase == 'running'
+    return self._component_status == 'running'
 
   def start_build(self, build_job):    
-    if not self.is_ready():
-      return False
+    self._current_job = build_job
+    self._build_status = StatusHandler(self.build_logs, build_job.repo_build())
+    self._image_info = {}
 
-    self.current_job = build_job
-    self._set_phase('building')
+    self._set_status('building')
 
     # Retrieve the job's buildpack.
     buildpack_url = self.user_files.get_file_url(build_job.repo_build().resource_key,
@@ -60,7 +69,7 @@ class BuildComponent(BaseComponent):
       buildpack = BuildPackage.from_url(buildpack_url)
     except BuildPackageException as bpe:
       self._build_failure('Could not retrieve build package', bpe)
-      return False
+      return
 
     # Extract the base image information from the Dockerfile.
     parsed_dockerfile = None
@@ -71,18 +80,22 @@ class BuildComponent(BaseComponent):
       parsed_dockerfile = buildpack.parse_dockerfile(build_config.get('build_subdir'))
     except BuildPackageException as bpe:
       self._build_failure('Could not find Dockerfile in build package', bpe)
-      return False
+      return
 
     image_and_tag_tuple = parsed_dockerfile.get_image_and_tag()
     if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
       self._build_failure('Missing FROM line Dockerfile')
-      return False      
+      return      
 
     base_image_information = {
       'repository': image_and_tag_tuple[0],
       'tag': image_and_tag_tuple[1]
     }
 
+    # Extract the number of steps from the Dockerfile.
+    with self._build_status as status_dict:
+      status_dict['total_commands'] = len(parsed_dockerfile.commands)
+
     # Add the pull robot information, if any.
     if build_config.get('pull_credentials') is not None:
       base_image_information['username'] = build_config['pull_credentials'].get('username', '')
@@ -109,7 +122,7 @@ class BuildComponent(BaseComponent):
       'build_package': buildpack_url,
       'sub_directory': build_config.get('build_subdir', ''),
       'repository': repository_name,
-      'registry': self.server_hostname,
+      'registry': '10.0.2.2:5000' or self.server_hostname,
       'pull_token': build_job.repo_build().access_token.code,
       'push_token': build_job.repo_build().access_token.code,
       'tag_names': build_config.get('docker_tags', ['latest']),
@@ -121,31 +134,129 @@ class BuildComponent(BaseComponent):
     logger.debug('Invoking build: %s', self.builder_realm)
     logger.debug('With Arguments: %s', build_arguments)
 
-    (self.call("io.quay.builder.build", **build_arguments)
-         .add_done_callback(self._build_complete))
+    return (self.call("io.quay.builder.build", **build_arguments)
+                .add_done_callback(self._build_complete))
+
+  @staticmethod
+  def __total_completion(statuses, total_images):
+    percentage_with_sizes = float(len(statuses.values()))/total_images
+    sent_bytes = sum([status['current'] for status in statuses.values()])
+    total_bytes = sum([status['total'] for status in statuses.values()])
+    return float(sent_bytes)/total_bytes*percentage_with_sizes
+
+  @staticmethod
+  def __process_pushpull_status(status_dict, current_phase, docker_data, images):
+    if not docker_data:
+      return
+
+    num_images = 0
+    status_completion_key = ''
+
+    if current_phase == 'pushing':
+      status_completion_key = 'push_completion'
+      num_images = status_dict['total_commands']
+    elif current_phase == 'pulling':
+      status_completion_key = 'pull_completion'
+    elif current_phase == 'priming-cache':
+      status_completion_key = 'cache_completion'
+    else:
+      return
+
+    if 'progressDetail' in docker_data and 'id' in docker_data:
+      image_id = docker_data['id']
+      detail = docker_data['progressDetail']
+
+      if 'current' in detail and 'total' in detail:
+        images[image_id] = detail
+        status_dict[status_completion_key] = \
+          BuildComponent.__total_completion(images, max(len(images), num_images))
+
+  def _on_log_message(self, status, json_data):
+    # Parse any of the JSON data logged.
+    docker_data = {}
+    if json_data:
+      try:
+        docker_data = json.loads(json_data)
+      except ValueError:
+        pass
+
+    # Extra the current status message (if any).
+    fully_unwrapped = ''
+    keys_to_extract = ['error', 'status', 'stream']
+    for key in keys_to_extract:
+      if key in docker_data:
+        fully_unwrapped = docker_data[key]
+        break
+
+    # Determine if this is a step string.
+    current_step = None
+    current_status_string = str(fully_unwrapped.encode('utf-8'))
+
+    if current_status_string and status == 'building':
+      step_increment = re.search(r'Step ([0-9]+) :', current_status_string)
+      if step_increment:
+        current_step = int(step_increment.group(1))
+
+    # Parse and update the phase and the status_dict. The status dictionary contains
+    # the pull/push progress.
+    with self._build_status as status_dict:
+      self._build_status.set_phase(status)      
+      BuildComponent.__process_pushpull_status(status_dict, status, docker_data, self._image_info)
+
+      # If the current message is for a step, then update that index.
+      if current_step:
+        status_dict['current_command'] = current_step
+
+      # If the json data contains an error, then something went wrong with a push or pull.
+      if 'error' in docker_data:
+        self._build_status.set_error(docker_data['error'])
+
+    # If we are in the building phase, then write out the log.
+    if status == 'building':
+      if current_step:
+        self._build_status.set_command(current_status_string)
+      else:
+        self._build_status.append_log(current_status_string)
 
-    return True
 
   def _build_failure(self, error_message, exception=None):
-    # TODO: log this message
-    print error_message
-    print exception
-    self._set_phase('running')
+    self._build_status.set_error(error_message, {
+      'internal_error': exception.message if exception else None
+    })
+
+    build_id = self._current_job.repo_build().uuid
+    logger.warning('Build %s failed with message: %s', build_id, self._error_message)
+
+    # Mark that the build has finished (in an error state)
+    self._build_finished(BUILD_JOB_RESULT.ERROR)
 
   def _build_complete(self, result):
     try:
-      status = result.result()
-      # TODO: log the success
-      print status
+      # Retrieve the result. This will raise an ApplicationError on any error that occurred.
+      result.result()
+      self._build_finished(BUILD_JOB_RESULT.COMPLETE)
+      self._build_status.set_phase(BUILD_PHASE.COMPLETE)
     except ApplicationError as ae:
-      error_kind = ae.error
-      # TODO: log the error
-      print error_kind
-    finally:
-      self._set_phase('running')
+      worker_error = WorkerError(ae.error, ae.kwargs.get('base_error'))
+
+      # Write the error to the log.
+      self._build_status.set_error(worker_error.public_message(), worker_error.extra_data())
+
+      # Mark the build as completed.
+      if worker_error.is_internal_error():
+        self._build_finished(BUILD_JOB_RESULT.INCOMPLETE)
+      else:
+        self._build_finished(BUILD_JOB_RESULT.ERROR)
+
+  def _build_finished(self, job_status):
+    self.parent_manager.job_completed(self._current_job, job_status, self)
+    self._current_job = None
+
+    # Set the component back to a running state.
+    self._set_status('running')
 
   def _on_ready(self, token):
-    if self.current_phase != 'waiting':
+    if self._component_status != 'waiting':
       logger.warning('Build component with token %s is already connected', self.expected_token)
       return
 
@@ -153,7 +264,7 @@ class BuildComponent(BaseComponent):
       logger.warning('Builder token mismatch. Expected: %s. Found: %s', self.expected_token, token)
       return
 
-    self._set_phase('running')
+    self._set_status('running')
 
     # Start the heartbeat check.
     loop = trollius.get_event_loop()
@@ -161,15 +272,11 @@ class BuildComponent(BaseComponent):
     logger.debug('Build worker %s is connected and ready' % self.builder_realm)
     return True
 
-  def _on_log_message(self, status, json):
-    # TODO: log the message
-    print json
-
-  def _set_phase(self, phase):
-    self.current_phase = phase
+  def _set_status(self, phase):
+    self._component_status = phase
 
   def _on_heartbeat(self):
-    self.last_heartbeat = datetime.datetime.now()
+    self._last_heartbeat = datetime.datetime.now()
 
   def _start_heartbeat_check(self, loop):
     trollius.set_event_loop(loop)
@@ -178,33 +285,35 @@ class BuildComponent(BaseComponent):
   @trollius.coroutine
   def _check_heartbeat(self, loop):
     while True:
-      if self.current_phase != 'running' or self.current_phase != 'building':
+      if self._component_status != 'running' and self._component_status != 'building':
         return
 
-      logger.debug('Checking heartbeat on realm %s and build %s',
-        self.builder_realm, self.expected_token)
-
-      if not self.last_heartbeat:
+      logger.debug('Checking heartbeat on realm %s', self.builder_realm)
+      if not self._last_heartbeat:
         self._timeout()
         return
 
-      if self.last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
+      if self._last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
         self._timeout()
         return      
 
       yield From(trollius.sleep(5))
 
   def _timeout(self):
-    self._set_phase('timeout')
+    self._set_status('timeout')
     logger.warning('Build component %s timed out', self.expected_token)
     self._dispose(timed_out=True)
 
   def _dispose(self, timed_out=False):
     # If we still have a running job, then it has not completed and we need to tell the parent
     # manager.
-    if self.current_job is not None:
-      self.parent_manager.job_completed(self.current_job, 'incomplete', self)
-      self.current_job = None
+    if self._current_job is not None:
+      if timed_out:
+        self._build_status.set_error('Build worker timed out. Build has been requeued')
+
+      self.parent_manager.job_completed(self._current_job, BUILD_JOB_RESULT.INCOMPLETE, self)
+      self._build_status = None
+      self._current_job = None
 
     # Unregister the current component so that it cannot be invoked again.
     self.parent_manager.build_component_disposed(self, timed_out)
diff --git a/buildman/buildjob.py b/buildman/buildjob.py
index ae87b26a2..bfd9363ec 100644
--- a/buildman/buildjob.py
+++ b/buildman/buildjob.py
@@ -42,7 +42,7 @@ class BuildJob(object):
 
     cached_tags = set(tags) & set([tag.name for tag in existing_tags])
     if cached_tags:
-      return cached_tags[0]
+      return list(cached_tags)[0]
 
     return None
 
diff --git a/buildman/buildstatus.py b/buildman/buildstatus.py
new file mode 100644
index 000000000..b888754ad
--- /dev/null
+++ b/buildman/buildstatus.py
@@ -0,0 +1,49 @@
+from functools import partial
+from data.database import BUILD_PHASE
+
+class StatusHandler(object):
+  """ Context wrapper for writing status to build logs. """
+
+  def __init__(self, build_logs, repository_build):
+    self._current_phase = None
+    self._repository_build = repository_build
+    self._uuid = repository_build.uuid
+    self._build_logs = build_logs
+
+    self._status = {
+      'total_commands': None,
+      'current_command': None,
+      'push_completion': 0.0,
+      'pull_completion': 0.0,
+    }
+
+    # Write the initial status.
+    self.__exit__(None, None, None)
+
+  def _append_log_message(self, log_message, log_type=None, log_data=None):
+    self._build_logs.append_log_message(self._uuid, log_message, log_type, log_data)
+
+  def append_log(self, log_message, extra_data=None):
+    self._append_log_message(log_message, log_data=extra_data)
+
+  def set_command(self, command, extra_data=None):
+    self._append_log_message(command, self._build_logs.COMMAND, extra_data)
+
+  def set_error(self, error_message, extra_data=None):
+    self.set_phase(BUILD_PHASE.ERROR)
+    self._append_log_message(error_message, self._build_logs.ERROR, extra_data)
+
+  def set_phase(self, phase, extra_data=None):
+    if phase == self._current_phase:
+      return 
+      
+    self._current_phase = phase
+    self._append_log_message(phase, self._build_logs.PHASE, extra_data)
+    self._repository_build.phase = phase
+    self._repository_build.save()
+
+  def __enter__(self):
+    return self._status
+
+  def __exit__(self, exc_type, value, traceback):
+    self._build_logs.set_status(self._uuid, self._status)
\ No newline at end of file
diff --git a/buildman/manager/enterprise.py b/buildman/manager/enterprise.py
index e9cd1b9a3..1b59a07eb 100644
--- a/buildman/manager/enterprise.py
+++ b/buildman/manager/enterprise.py
@@ -60,6 +60,6 @@ class EnterpriseManager(BaseManager):
   def job_completed(self, build_job, job_status, build_component):
     self.job_complete_callback(build_job, job_status)
 
-  def component_disposed(self, build_component, timed_out):
+  def build_component_disposed(self, build_component, timed_out):
     self.build_components.remove(build_component)
 
diff --git a/buildman/server.py b/buildman/server.py
index 2252ca117..76e6eacc4 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -15,10 +15,16 @@ from buildjob import BuildJob, BuildJobLoadException
 
 logger = logging.getLogger(__name__)
 
-WORK_CHECK_TIMEOUT = 30
+WORK_CHECK_TIMEOUT = 10
 TIMEOUT_PERIOD_MINUTES = 20
 RESERVATION_SECONDS = (TIMEOUT_PERIOD_MINUTES + 5) * 60
 
+class BUILD_JOB_RESULT(object):
+  """ Build job result enum """
+  INCOMPLETE = 'incomplete'
+  COMPLETE = 'complete'
+  ERROR = 'error'
+
 class BuilderServer(object):
   """ Server which handles both HTTP and WAMP requests, managing the full state of the build
       controller.
@@ -101,18 +107,20 @@ class BuilderServer(object):
     self._session_factory.remove(component)
 
   def _job_complete(self, build_job, job_status):
-    if job_status == 'incomplete':
-      self._queue.incomplete(build_job.job_item(), restore_retry=True)
-    elif job_status == 'error':
+    if job_status == BUILD_JOB_RESULT.INCOMPLETE:
+      self._queue.incomplete(build_job.job_item(), restore_retry=True, retry_after=30)
+    elif job_status == BUILD_JOB_RESULT.ERROR:
       self._queue.incomplete(build_job.job_item(), restore_retry=False)
     else:
-      self._queue.complete(job)
+      self._queue.complete(build_job.job_item())
 
     self._job_count = self._job_count - 1
 
     if self._current_status == 'shutting_down' and not self._job_count:
       self._shutdown_event.set()
 
+    # TODO: check for work here?      
+
   @trollius.coroutine
   def _work_checker(self):
     while self._current_status == 'running':
@@ -135,7 +143,7 @@ class BuilderServer(object):
         logger.debug('Build job scheduled. Running: %s', self._job_count)
       else:
         logger.debug('All workers are busy. Requeuing.')
-        self._queue.incomplete(job_item, restore_retry=True)
+        self._queue.incomplete(job_item, restore_retry=True, retry_after=0)
         
       yield From(trollius.sleep(WORK_CHECK_TIMEOUT))
 
diff --git a/buildman/workererror.py b/buildman/workererror.py
new file mode 100644
index 000000000..023817e00
--- /dev/null
+++ b/buildman/workererror.py
@@ -0,0 +1,87 @@
+class WorkerError(object):
+  """ Helper class which represents errors raised by a build worker. """
+  def __init__(self, error_code, base_message=None):
+    self._error_code = error_code
+    self._base_message = base_message
+
+    self._error_handlers = {
+      'io.quay.builder.buildpackissue': {
+        'message': 'Could not load build package',
+        'is_internal': True
+      },
+
+      'io.quay.builder.cannotextractbuildpack': {
+        'message': 'Could not extract the contents of the build package'
+      },
+
+      'io.quay.builder.cannotpullforcache': {
+        'message': 'Could not pull cached image',
+        'is_internal': True
+      },
+
+      'io.quay.builder.cannotpullbaseimage': {
+        'message': 'Could not pull base image',
+        'show_base_error': True
+      },
+
+      'io.quay.builder.internalerror': {
+        'message': 'An internal error occurred while building. Please submit a ticket.'
+      },
+
+      'io.quay.builder.buildrunerror': {
+        'message': 'Could not start the build process',
+        'is_internal': True
+      },
+
+      'io.quay.builder.builderror': {
+        'message': 'A build step failed',
+        'show_base_error': True
+      },
+
+      'io.quay.builder.tagissue': {
+        'message': 'Could not tag built image',
+        'is_internal': True
+      },
+
+      'io.quay.builder.pushissue': {
+        'message': 'Could not push built image',
+        'show_base_error': True,
+        'is_internal': True
+      },
+
+      'io.quay.builder.dockerconnecterror': {
+        'message': 'Could not connect to Docker daemon',
+        'is_internal': True
+      },
+
+      'io.quay.builder.missingorinvalidargument': {
+        'message': 'Missing required arguments for builder',
+        'is_internal': True
+      }
+    }
+
+  def is_internal_error(self):
+    handler = self._error_handlers.get(self._error_code)
+    return handler.get('is_internal', False) if handler else True
+
+  def public_message(self):
+    handler = self._error_handlers.get(self._error_code)
+    if not handler:
+      return 'An unknown error occurred'
+
+    message = handler['message']
+    if handler.get('show_base_error', False) and self._base_message:
+      message = message + ': ' + self._base_message
+
+    if handler.get('is_internal', False):
+      message = message + '\nThe build will be retried shortly'
+
+    return message
+
+  def extra_data(self):
+    if self._base_message:
+      return {
+        'base_error': self._base_message
+      }
+
+    return {}
\ No newline at end of file
diff --git a/static/css/quay.css b/static/css/quay.css
index 25934010b..ae5913db8 100644
--- a/static/css/quay.css
+++ b/static/css/quay.css
@@ -876,6 +876,10 @@ i.toggle-icon:hover {
   background-color: #f0ad4e;
 }
 
+.phase-icon.priming-cache {
+  background-color: #ddd;
+}
+
 .phase-icon.pushing {
   background-color: #5cb85c;
 }
diff --git a/static/js/app.js b/static/js/app.js
index 7c00c5831..1f5d53574 100644
--- a/static/js/app.js
+++ b/static/js/app.js
@@ -5678,6 +5678,9 @@ quayApp.directive('buildMessage', function () {
           
           case 'building':
             return 'Building image from Dockerfile';
+
+          case 'priming-cache':
+            return 'Priming cache for build';
           
           case 'pushing':
             return 'Pushing image built from Dockerfile';
@@ -5720,6 +5723,10 @@ quayApp.directive('buildProgress', function () {
             return buildInfo.status.push_completion * 100;
             break;
 
+          case 'priming-cache':
+            return buildInfo.status.cache_completion * 100;
+            break;
+
           case 'complete':
             return 100;
             break;

From cfc6b196a491c213ff71e55c4294f32288910607 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Fri, 14 Nov 2014 14:53:35 -0500
Subject: [PATCH 04/26] - Extra the build component statuses into an enum - Add
 a ping method so the workers can verify the state of the controller - Fix a
 bug with current_step and 0 values - Rename the build status var to phase, to
 make it more distinct from the controller status

---
 buildman/buildcomponent.py | 63 ++++++++++++++++++++++----------------
 1 file changed, 37 insertions(+), 26 deletions(-)

diff --git a/buildman/buildcomponent.py b/buildman/buildcomponent.py
index a4ce3c821..83ff998bd 100644
--- a/buildman/buildcomponent.py
+++ b/buildman/buildcomponent.py
@@ -19,6 +19,13 @@ HEARTBEAT_DELTA = datetime.timedelta(seconds=15)
 
 logger = logging.getLogger(__name__)
 
+class COMPONENT_STATUS(object):
+  JOINING = 'joining'
+  WAITING = 'waiting'
+  RUNNING = 'running'
+  BUILDING = 'building'
+  TIMED_OUT = 'timeout'
+
 class BuildComponent(BaseComponent):
   """ An application session component which conducts one (or more) builds. """
 
@@ -26,8 +33,8 @@ class BuildComponent(BaseComponent):
   expected_token = None
   builder_realm = None
 
+  _component_status = COMPONENT_STATUS.JOINING
   _last_heartbeat = None
-  _component_status = 'joining'
   _current_job = None
   _build_status = None
   _image_info = None
@@ -44,20 +51,21 @@ class BuildComponent(BaseComponent):
   def onJoin(self, details):
     logger.debug('Registering methods and listeners for component %s' % self.builder_realm)
     yield From(self.register(self._on_ready, u'io.quay.buildworker.ready'))
+    yield From(self.register(self._ping, u'io.quay.buildworker.ping'))
     yield From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat'))
     yield From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage'))
 
-    self._set_status('waiting')
+    self._set_status(COMPONENT_STATUS.WAITING)
 
   def is_ready(self):
-    return self._component_status == 'running'
+    return self._component_status == COMPONENT_STATUS.RUNNING
 
   def start_build(self, build_job):    
     self._current_job = build_job
     self._build_status = StatusHandler(self.build_logs, build_job.repo_build())
     self._image_info = {}
 
-    self._set_status('building')
+    self._set_status(COMPONENT_STATUS.BUILDING)
 
     # Retrieve the job's buildpack.
     buildpack_url = self.user_files.get_file_url(build_job.repo_build().resource_key,
@@ -84,7 +92,7 @@ class BuildComponent(BaseComponent):
 
     image_and_tag_tuple = parsed_dockerfile.get_image_and_tag()
     if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
-      self._build_failure('Missing FROM line Dockerfile')
+      self._build_failure('Missing FROM line in Dockerfile')
       return      
 
     base_image_information = {
@@ -139,10 +147,10 @@ class BuildComponent(BaseComponent):
 
   @staticmethod
   def __total_completion(statuses, total_images):
-    percentage_with_sizes = float(len(statuses.values()))/total_images
+    percentage_with_sizes = float(len(statuses.values())) / total_images
     sent_bytes = sum([status['current'] for status in statuses.values()])
     total_bytes = sum([status['total'] for status in statuses.values()])
-    return float(sent_bytes)/total_bytes*percentage_with_sizes
+    return float(sent_bytes) / total_bytes * percentage_with_sizes
 
   @staticmethod
   def __process_pushpull_status(status_dict, current_phase, docker_data, images):
@@ -171,7 +179,7 @@ class BuildComponent(BaseComponent):
         status_dict[status_completion_key] = \
           BuildComponent.__total_completion(images, max(len(images), num_images))
 
-  def _on_log_message(self, status, json_data):
+  def _on_log_message(self, phase, json_data):
     # Parse any of the JSON data logged.
     docker_data = {}
     if json_data:
@@ -180,7 +188,7 @@ class BuildComponent(BaseComponent):
       except ValueError:
         pass
 
-    # Extra the current status message (if any).
+    # Extract the current status message (if any).
     fully_unwrapped = ''
     keys_to_extract = ['error', 'status', 'stream']
     for key in keys_to_extract:
@@ -192,31 +200,30 @@ class BuildComponent(BaseComponent):
     current_step = None
     current_status_string = str(fully_unwrapped.encode('utf-8'))
 
-    if current_status_string and status == 'building':
+    if current_status_string and phase == BUILD_PHASE.BUILDING:
       step_increment = re.search(r'Step ([0-9]+) :', current_status_string)
       if step_increment:
         current_step = int(step_increment.group(1))
 
     # Parse and update the phase and the status_dict. The status dictionary contains
-    # the pull/push progress.
+    # the pull/push progress, as well as the current step index.
     with self._build_status as status_dict:
-      self._build_status.set_phase(status)      
-      BuildComponent.__process_pushpull_status(status_dict, status, docker_data, self._image_info)
+      self._build_status.set_phase(phase)      
+      BuildComponent.__process_pushpull_status(status_dict, phase, docker_data, self._image_info)
 
-      # If the current message is for a step, then update that index.
-      if current_step:
+      # If the current message represents the beginning of a new step, then update the
+      # current command index.
+      if current_step is not None:
         status_dict['current_command'] = current_step
 
       # If the json data contains an error, then something went wrong with a push or pull.
       if 'error' in docker_data:
         self._build_status.set_error(docker_data['error'])
 
-    # If we are in the building phase, then write out the log.
-    if status == 'building':
-      if current_step:
-        self._build_status.set_command(current_status_string)
-      else:
-        self._build_status.append_log(current_status_string)
+    if current_step is not None:
+      self._build_status.set_command(current_status_string)
+    elif phase == BUILD_PHASE.BUILDING:
+      self._build_status.append_log(current_status_string)
 
 
   def _build_failure(self, error_message, exception=None):
@@ -234,8 +241,8 @@ class BuildComponent(BaseComponent):
     try:
       # Retrieve the result. This will raise an ApplicationError on any error that occurred.
       result.result()
-      self._build_finished(BUILD_JOB_RESULT.COMPLETE)
       self._build_status.set_phase(BUILD_PHASE.COMPLETE)
+      self._build_finished(BUILD_JOB_RESULT.COMPLETE)
     except ApplicationError as ae:
       worker_error = WorkerError(ae.error, ae.kwargs.get('base_error'))
 
@@ -253,7 +260,10 @@ class BuildComponent(BaseComponent):
     self._current_job = None
 
     # Set the component back to a running state.
-    self._set_status('running')
+    self._set_status(COMPONENT_STATUS.RUNNING)
+
+  def _ping(self):
+    return 'pong'
 
   def _on_ready(self, token):
     if self._component_status != 'waiting':
@@ -264,7 +274,7 @@ class BuildComponent(BaseComponent):
       logger.warning('Builder token mismatch. Expected: %s. Found: %s', self.expected_token, token)
       return
 
-    self._set_status('running')
+    self._set_status(COMPONENT_STATUS.RUNNING)
 
     # Start the heartbeat check.
     loop = trollius.get_event_loop()
@@ -285,7 +295,8 @@ class BuildComponent(BaseComponent):
   @trollius.coroutine
   def _check_heartbeat(self, loop):
     while True:
-      if self._component_status != 'running' and self._component_status != 'building':
+      if (self._component_status != COMPONENT_STATUS.RUNNING and 
+          self._component_status != COMPONENT_STATUS.BUILDING):
         return
 
       logger.debug('Checking heartbeat on realm %s', self.builder_realm)
@@ -300,7 +311,7 @@ class BuildComponent(BaseComponent):
       yield From(trollius.sleep(5))
 
   def _timeout(self):
-    self._set_status('timeout')
+    self._set_status(COMPONENT_STATUS.TIMED_OUT)
     logger.warning('Build component %s timed out', self.expected_token)
     self._dispose(timed_out=True)
 

From 01dc10b8fc06c2b99599d9130df204791a47fb32 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Fri, 14 Nov 2014 15:05:49 -0500
Subject: [PATCH 05/26] Remove server hostname hack

---
 buildman/buildcomponent.py | 2 +-
 buildman/buildstatus.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/buildman/buildcomponent.py b/buildman/buildcomponent.py
index 83ff998bd..dbad6540d 100644
--- a/buildman/buildcomponent.py
+++ b/buildman/buildcomponent.py
@@ -130,7 +130,7 @@ class BuildComponent(BaseComponent):
       'build_package': buildpack_url,
       'sub_directory': build_config.get('build_subdir', ''),
       'repository': repository_name,
-      'registry': '10.0.2.2:5000' or self.server_hostname,
+      'registry': self.server_hostname,
       'pull_token': build_job.repo_build().access_token.code,
       'push_token': build_job.repo_build().access_token.code,
       'tag_names': build_config.get('docker_tags', ['latest']),
diff --git a/buildman/buildstatus.py b/buildman/buildstatus.py
index b888754ad..d09876e10 100644
--- a/buildman/buildstatus.py
+++ b/buildman/buildstatus.py
@@ -35,7 +35,7 @@ class StatusHandler(object):
 
   def set_phase(self, phase, extra_data=None):
     if phase == self._current_phase:
-      return 
+      return
       
     self._current_phase = phase
     self._append_log_message(phase, self._build_logs.PHASE, extra_data)

From 043a30ee9670982e12f719163afe3b466b8b32ca Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Fri, 14 Nov 2014 15:31:02 -0500
Subject: [PATCH 06/26] Add a heartbeat to the build status, so we know if a
 manager crashed

---
 buildman/buildcomponent.py | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/buildman/buildcomponent.py b/buildman/buildcomponent.py
index dbad6540d..de63aa0b5 100644
--- a/buildman/buildcomponent.py
+++ b/buildman/buildcomponent.py
@@ -15,7 +15,8 @@ from buildman.workererror import WorkerError
 
 from data.database import BUILD_PHASE
 
-HEARTBEAT_DELTA = datetime.timedelta(seconds=15)
+HEARTBEAT_DELTA = datetime.timedelta(seconds=30)
+HEARTBEAT_TIMEOUT = 10
 
 logger = logging.getLogger(__name__)
 
@@ -276,9 +277,9 @@ class BuildComponent(BaseComponent):
 
     self._set_status(COMPONENT_STATUS.RUNNING)
 
-    # Start the heartbeat check.
+    # Start the heartbeat check and updating loop.
     loop = trollius.get_event_loop()
-    loop.create_task(self._check_heartbeat(loop))
+    loop.create_task(self._heartbeat(loop))
     logger.debug('Build worker %s is connected and ready' % self.builder_realm)
     return True
 
@@ -288,17 +289,29 @@ class BuildComponent(BaseComponent):
   def _on_heartbeat(self):
     self._last_heartbeat = datetime.datetime.now()
 
-  def _start_heartbeat_check(self, loop):
+  def _start_heartbeat(self, loop):
     trollius.set_event_loop(loop)
-    loop.run_until_complete(self._check_heartbeat())
+    loop.run_until_complete(self._heartbeat())
 
   @trollius.coroutine
-  def _check_heartbeat(self, loop):
+  def _heartbeat(self, loop):
+    """ Coroutine that runs every HEARTBEAT_TIMEOUT seconds, both checking the worker's heartbeat
+        and updating the heartbeat in the build status dictionary (if applicable). This allows
+        the build system to catch crashes from either end.
+    """
     while True:
+      # If the component is no longer running or actively building, nothing more to do.
       if (self._component_status != COMPONENT_STATUS.RUNNING and 
           self._component_status != COMPONENT_STATUS.BUILDING):
         return
 
+      # If there is an active build, write the heartbeat to its status.
+      build_status = self._build_status
+      if build_status is not None:
+        with build_status as status_dict:
+          status_dict['heartbeat'] = int(time.time())
+
+      # Check the heartbeat from the worker.
       logger.debug('Checking heartbeat on realm %s', self.builder_realm)
       if not self._last_heartbeat:
         self._timeout()
@@ -308,7 +321,7 @@ class BuildComponent(BaseComponent):
         self._timeout()
         return      
 
-      yield From(trollius.sleep(5))
+      yield From(trollius.sleep(HEARTBEAT_TIMEOUT))
 
   def _timeout(self):
     self._set_status(COMPONENT_STATUS.TIMED_OUT)

From 6df6f28edf725a63bea3292ee55848c0ecb14b8d Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Tue, 18 Nov 2014 15:45:56 -0500
Subject: [PATCH 07/26] Lint BuildManager

---
 buildman/buildcomponent.py      | 146 +++++++++++++++++---------------
 buildman/buildjob.py            |  11 +--
 buildman/buildpack.py           |  21 ++---
 buildman/buildstatus.py         |  13 ++-
 buildman/enterprise_builder.py  |   4 +-
 buildman/manager/basemanager.py |   2 +-
 buildman/manager/enterprise.py  |  10 ++-
 buildman/server.py              |  63 +++++++-------
 buildman/workererror.py         |  90 ++++++++++----------
 9 files changed, 187 insertions(+), 173 deletions(-)

diff --git a/buildman/buildcomponent.py b/buildman/buildcomponent.py
index de63aa0b5..3e2458c4a 100644
--- a/buildman/buildcomponent.py
+++ b/buildman/buildcomponent.py
@@ -10,7 +10,7 @@ from trollius.coroutines import From
 from buildman.basecomponent import BaseComponent
 from buildman.buildpack import BuildPackage, BuildPackageException
 from buildman.buildstatus import StatusHandler
-from buildman.server import BUILD_JOB_RESULT
+from buildman.server import BuildJobResult
 from buildman.workererror import WorkerError
 
 from data.database import BUILD_PHASE
@@ -18,9 +18,10 @@ from data.database import BUILD_PHASE
 HEARTBEAT_DELTA = datetime.timedelta(seconds=30)
 HEARTBEAT_TIMEOUT = 10
 
-logger = logging.getLogger(__name__)
+LOGGER = logging.getLogger(__name__)
 
-class COMPONENT_STATUS(object):
+class ComponentStatus(object):
+  """ ComponentStatus represents the possible states of a component. """
   JOINING = 'joining'
   WAITING = 'waiting'
   RUNNING = 'running'
@@ -34,7 +35,7 @@ class BuildComponent(BaseComponent):
   expected_token = None
   builder_realm = None
 
-  _component_status = COMPONENT_STATUS.JOINING
+  _component_status = ComponentStatus.JOINING
   _last_heartbeat = None
   _current_job = None
   _build_status = None
@@ -46,33 +47,35 @@ class BuildComponent(BaseComponent):
 
     BaseComponent.__init__(self, config, **kwargs)
 
-  def onConnect(self):    
+  def onConnect(self):
     self.join(self.builder_realm)
 
   def onJoin(self, details):
-    logger.debug('Registering methods and listeners for component %s' % self.builder_realm)
+    LOGGER.debug('Registering methods and listeners for component %s', self.builder_realm)
     yield From(self.register(self._on_ready, u'io.quay.buildworker.ready'))
     yield From(self.register(self._ping, u'io.quay.buildworker.ping'))
     yield From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat'))
     yield From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage'))
 
-    self._set_status(COMPONENT_STATUS.WAITING)
+    self._set_status(ComponentStatus.WAITING)
 
   def is_ready(self):
-    return self._component_status == COMPONENT_STATUS.RUNNING
+    """ Determines whether a build component is ready to begin a build. """
+    return self._component_status == ComponentStatus.RUNNING
 
-  def start_build(self, build_job):    
+  def start_build(self, build_job):
+    """ Starts a build. """
     self._current_job = build_job
     self._build_status = StatusHandler(self.build_logs, build_job.repo_build())
     self._image_info = {}
 
-    self._set_status(COMPONENT_STATUS.BUILDING)
+    self._set_status(ComponentStatus.BUILDING)
 
     # Retrieve the job's buildpack.
     buildpack_url = self.user_files.get_file_url(build_job.repo_build().resource_key,
                                                  requires_cors=False)
 
-    logger.debug('Retreiving build package: %s' % buildpack_url)
+    LOGGER.debug('Retreiving build package: %s', buildpack_url)
     buildpack = None
     try:
       buildpack = BuildPackage.from_url(buildpack_url)
@@ -82,7 +85,7 @@ class BuildComponent(BaseComponent):
 
     # Extract the base image information from the Dockerfile.
     parsed_dockerfile = None
-    logger.debug('Parsing dockerfile')
+    LOGGER.debug('Parsing dockerfile')
 
     build_config = build_job.build_config()
     try:
@@ -94,11 +97,11 @@ class BuildComponent(BaseComponent):
     image_and_tag_tuple = parsed_dockerfile.get_image_and_tag()
     if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
       self._build_failure('Missing FROM line in Dockerfile')
-      return      
+      return
 
     base_image_information = {
-      'repository': image_and_tag_tuple[0],
-      'tag': image_and_tag_tuple[1]
+        'repository': image_and_tag_tuple[0],
+        'tag': image_and_tag_tuple[1]
     }
 
     # Extract the number of steps from the Dockerfile.
@@ -115,46 +118,47 @@ class BuildComponent(BaseComponent):
     repository_name = repo.namespace_user.username + '/' + repo.name
 
     # Parse the build queue item into build arguments.
-    #   build_package: URL to the build package to download and untar/unzip.
-    #   sub_directory: The location within the build package of the Dockerfile and the build context.
-    #   repository: The repository for which this build is occurring.
-    #   registry: The registry for which this build is occuring. Example: 'quay.io', 'staging.quay.io'
-    #   pull_token: The token to use when pulling the cache for building.
-    #   push_token: The token to use to push the built image.
-    #   tag_names: The name(s) of the tag(s) for the newly built image.
-    #   base_image: The image name and credentials to use to conduct the base image pull.
-    #     repository: The repository to pull.
-    #     tag: The tag to pull.
-    #     username: The username for pulling the base image (if any).
-    #     password: The password for pulling the base image (if any).
+    #  build_package: URL to the build package to download and untar/unzip.
+    #  sub_directory: The location within the build package of the Dockerfile and the build context.
+    #  repository: The repository for which this build is occurring.
+    #  registry: The registry for which this build is occuring (e.g. 'quay.io', 'staging.quay.io').
+    #  pull_token: The token to use when pulling the cache for building.
+    #  push_token: The token to use to push the built image.
+    #  tag_names: The name(s) of the tag(s) for the newly built image.
+    #  base_image: The image name and credentials to use to conduct the base image pull.
+    #   repository: The repository to pull.
+    #   tag: The tag to pull.
+    #   username: The username for pulling the base image (if any).
+    #   password: The password for pulling the base image (if any).
     build_arguments = {
-      'build_package': buildpack_url,
-      'sub_directory': build_config.get('build_subdir', ''),
-      'repository': repository_name,
-      'registry': self.server_hostname,
-      'pull_token': build_job.repo_build().access_token.code,
-      'push_token': build_job.repo_build().access_token.code,
-      'tag_names': build_config.get('docker_tags', ['latest']),
-      'base_image': base_image_information,
-      'cached_tag': build_job.determine_cached_tag() or ''
+        'build_package': buildpack_url,
+        'sub_directory': build_config.get('build_subdir', ''),
+        'repository': repository_name,
+        'registry': self.server_hostname,
+        'pull_token': build_job.repo_build().access_token.code,
+        'push_token': build_job.repo_build().access_token.code,
+        'tag_names': build_config.get('docker_tags', ['latest']),
+        'base_image': base_image_information,
+        'cached_tag': build_job.determine_cached_tag() or ''
     }
 
     # Invoke the build.
-    logger.debug('Invoking build: %s', self.builder_realm)
-    logger.debug('With Arguments: %s', build_arguments)
+    LOGGER.debug('Invoking build: %s', self.builder_realm)
+    LOGGER.debug('With Arguments: %s', build_arguments)
 
     return (self.call("io.quay.builder.build", **build_arguments)
-                .add_done_callback(self._build_complete))
+            .add_done_callback(self._build_complete))
 
   @staticmethod
-  def __total_completion(statuses, total_images):
+  def _total_completion(statuses, total_images):
+    """ Returns the current amount completion relative to the total completion of a build. """
     percentage_with_sizes = float(len(statuses.values())) / total_images
     sent_bytes = sum([status['current'] for status in statuses.values()])
     total_bytes = sum([status['total'] for status in statuses.values()])
     return float(sent_bytes) / total_bytes * percentage_with_sizes
 
   @staticmethod
-  def __process_pushpull_status(status_dict, current_phase, docker_data, images):
+  def _process_pushpull_status(status_dict, current_phase, docker_data, images):
     if not docker_data:
       return
 
@@ -178,7 +182,7 @@ class BuildComponent(BaseComponent):
       if 'current' in detail and 'total' in detail:
         images[image_id] = detail
         status_dict[status_completion_key] = \
-          BuildComponent.__total_completion(images, max(len(images), num_images))
+          BuildComponent._total_completion(images, max(len(images), num_images))
 
   def _on_log_message(self, phase, json_data):
     # Parse any of the JSON data logged.
@@ -209,8 +213,8 @@ class BuildComponent(BaseComponent):
     # Parse and update the phase and the status_dict. The status dictionary contains
     # the pull/push progress, as well as the current step index.
     with self._build_status as status_dict:
-      self._build_status.set_phase(phase)      
-      BuildComponent.__process_pushpull_status(status_dict, phase, docker_data, self._image_info)
+      self._build_status.set_phase(phase)
+      BuildComponent._process_pushpull_status(status_dict, phase, docker_data, self._image_info)
 
       # If the current message represents the beginning of a new step, then update the
       # current command index.
@@ -228,104 +232,110 @@ class BuildComponent(BaseComponent):
 
 
   def _build_failure(self, error_message, exception=None):
+    """ Handles and logs a failed build. """
     self._build_status.set_error(error_message, {
-      'internal_error': exception.message if exception else None
+        'internal_error': exception.message if exception else None
     })
 
     build_id = self._current_job.repo_build().uuid
-    logger.warning('Build %s failed with message: %s', build_id, self._error_message)
+    LOGGER.warning('Build %s failed with message: %s', build_id, error_message)
 
     # Mark that the build has finished (in an error state)
-    self._build_finished(BUILD_JOB_RESULT.ERROR)
+    self._build_finished(BuildJobResult.ERROR)
 
   def _build_complete(self, result):
+    """ Wraps up a completed build. Handles any errors and calls self._build_finished. """
     try:
       # Retrieve the result. This will raise an ApplicationError on any error that occurred.
       result.result()
       self._build_status.set_phase(BUILD_PHASE.COMPLETE)
-      self._build_finished(BUILD_JOB_RESULT.COMPLETE)
-    except ApplicationError as ae:
-      worker_error = WorkerError(ae.error, ae.kwargs.get('base_error'))
+      self._build_finished(BuildJobResult.COMPLETE)
+    except ApplicationError as aex:
+      worker_error = WorkerError(aex.error, aex.kwargs.get('base_error'))
 
       # Write the error to the log.
       self._build_status.set_error(worker_error.public_message(), worker_error.extra_data())
 
       # Mark the build as completed.
       if worker_error.is_internal_error():
-        self._build_finished(BUILD_JOB_RESULT.INCOMPLETE)
+        self._build_finished(BuildJobResult.INCOMPLETE)
       else:
-        self._build_finished(BUILD_JOB_RESULT.ERROR)
+        self._build_finished(BuildJobResult.ERROR)
 
   def _build_finished(self, job_status):
+    """ Alerts the parent that a build has completed and sets the status back to running. """
     self.parent_manager.job_completed(self._current_job, job_status, self)
     self._current_job = None
 
     # Set the component back to a running state.
-    self._set_status(COMPONENT_STATUS.RUNNING)
+    self._set_status(ComponentStatus.RUNNING)
 
-  def _ping(self):
+  def _ping():
+    """ Ping pong. """
     return 'pong'
 
   def _on_ready(self, token):
     if self._component_status != 'waiting':
-      logger.warning('Build component with token %s is already connected', self.expected_token)
+      LOGGER.warning('Build component with token %s is already connected', self.expected_token)
       return
 
     if token != self.expected_token:
-      logger.warning('Builder token mismatch. Expected: %s. Found: %s', self.expected_token, token)
+      LOGGER.warning('Builder token mismatch. Expected: %s. Found: %s', self.expected_token, token)
       return
 
-    self._set_status(COMPONENT_STATUS.RUNNING)
+    self._set_status(ComponentStatus.RUNNING)
 
     # Start the heartbeat check and updating loop.
     loop = trollius.get_event_loop()
-    loop.create_task(self._heartbeat(loop))
-    logger.debug('Build worker %s is connected and ready' % self.builder_realm)
+    loop.create_task(self._heartbeat())
+    LOGGER.debug('Build worker %s is connected and ready', self.builder_realm)
     return True
 
   def _set_status(self, phase):
     self._component_status = phase
 
   def _on_heartbeat(self):
+    """ Updates the last known heartbeat. """
     self._last_heartbeat = datetime.datetime.now()
 
   def _start_heartbeat(self, loop):
+    """ Begins an async loop to keep a heartbeat going with a client. """
     trollius.set_event_loop(loop)
     loop.run_until_complete(self._heartbeat())
 
   @trollius.coroutine
-  def _heartbeat(self, loop):
+  def _heartbeat(self):
     """ Coroutine that runs every HEARTBEAT_TIMEOUT seconds, both checking the worker's heartbeat
         and updating the heartbeat in the build status dictionary (if applicable). This allows
         the build system to catch crashes from either end.
     """
     while True:
       # If the component is no longer running or actively building, nothing more to do.
-      if (self._component_status != COMPONENT_STATUS.RUNNING and 
-          self._component_status != COMPONENT_STATUS.BUILDING):
+      if (self._component_status != ComponentStatus.RUNNING and
+          self._component_status != ComponentStatus.BUILDING):
         return
 
       # If there is an active build, write the heartbeat to its status.
       build_status = self._build_status
       if build_status is not None:
         with build_status as status_dict:
-          status_dict['heartbeat'] = int(time.time())
+          status_dict['heartbeat'] = int(datetime.time())
 
       # Check the heartbeat from the worker.
-      logger.debug('Checking heartbeat on realm %s', self.builder_realm)
+      LOGGER.debug('Checking heartbeat on realm %s', self.builder_realm)
       if not self._last_heartbeat:
         self._timeout()
         return
 
       if self._last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
         self._timeout()
-        return      
+        return
 
       yield From(trollius.sleep(HEARTBEAT_TIMEOUT))
 
   def _timeout(self):
-    self._set_status(COMPONENT_STATUS.TIMED_OUT)
-    logger.warning('Build component %s timed out', self.expected_token)
+    self._set_status(ComponentStatus.TIMED_OUT)
+    LOGGER.warning('Build component %s timed out', self.expected_token)
     self._dispose(timed_out=True)
 
   def _dispose(self, timed_out=False):
@@ -335,7 +345,7 @@ class BuildComponent(BaseComponent):
       if timed_out:
         self._build_status.set_error('Build worker timed out. Build has been requeued')
 
-      self.parent_manager.job_completed(self._current_job, BUILD_JOB_RESULT.INCOMPLETE, self)
+      self.parent_manager.job_completed(self._current_job, BuildJobResult.INCOMPLETE, self)
       self._build_status = None
       self._current_job = None
 
diff --git a/buildman/buildjob.py b/buildman/buildjob.py
index bfd9363ec..2b9dbf35c 100644
--- a/buildman/buildjob.py
+++ b/buildman/buildjob.py
@@ -8,7 +8,6 @@ class BuildJobLoadException(Exception):
 
 class BuildJob(object):
   """ Represents a single in-progress build job. """
-
   def __init__(self, job_item):
     self._job_item = job_item
 
@@ -16,7 +15,8 @@ class BuildJob(object):
       self._job_details = json.loads(job_item.body)
     except ValueError:
       raise BuildJobLoadException(
-        'Could not parse build queue item config with ID %s' % self._job_details['build_uuid'])      
+          'Could not parse build queue item config with ID %s' % self._job_details['build_uuid']
+      )
 
     try:
       self._repo_build = model.get_repository_build(self._job_details['namespace'],
@@ -24,13 +24,14 @@ class BuildJob(object):
                                                     self._job_details['build_uuid'])
     except model.InvalidRepositoryBuildException:
       raise BuildJobLoadException(
-        'Could not load repository build with ID %s' % self._job_details['build_uuid'])
+          'Could not load repository build with ID %s' % self._job_details['build_uuid'])
 
     try:
       self._build_config = json.loads(self._repo_build.job_config)
     except ValueError:
       raise BuildJobLoadException(
-        'Could not parse repository build job config with ID %s' % self._job_details['build_uuid'])      
+          'Could not parse repository build job config with ID %s' % self._job_details['build_uuid']
+      )
 
   def determine_cached_tag(self):
     """ Returns the tag to pull to prime the cache or None if none. """
@@ -56,4 +57,4 @@ class BuildJob(object):
 
   def build_config(self):
     """ Returns the parsed repository build config for the job. """
-    return self._build_config
\ No newline at end of file
+    return self._build_config
diff --git a/buildman/buildpack.py b/buildman/buildpack.py
index 62dab38e5..cdc4d4b07 100644
--- a/buildman/buildpack.py
+++ b/buildman/buildpack.py
@@ -4,7 +4,8 @@ import os
 
 from tempfile import TemporaryFile, mkdtemp
 from zipfile import ZipFile
-from util.dockerfileparse import parse_dockerfile, ParsedDockerfile
+from util.dockerfileparse import parse_dockerfile
+from util.safetar import safe_extractall
 
 class BuildPackageException(Exception):
   """ Exception raised when retrieving or parsing a build package. """
@@ -14,15 +15,15 @@ class BuildPackageException(Exception):
 class BuildPackage(object):
   """ Helper class for easy reading and updating of a Dockerfile build pack. """
 
-  def __init__(self, requests_file):  
+  def __init__(self, requests_file):
     self._mime_processors = {
-      'application/zip': BuildPackage.__prepare_zip,
-      'application/x-zip-compressed': BuildPackage.__prepare_zip,
-      'text/plain': BuildPackage.__prepare_dockerfile,
-      'application/octet-stream': BuildPackage.__prepare_dockerfile,
-      'application/x-tar': BuildPackage.__prepare_tarball,
-      'application/gzip': BuildPackage.__prepare_tarball,
-      'application/x-gzip': BuildPackage.__prepare_tarball,
+        'application/zip': BuildPackage.__prepare_zip,
+        'application/x-zip-compressed': BuildPackage.__prepare_zip,
+        'text/plain': BuildPackage.__prepare_dockerfile,
+        'application/octet-stream': BuildPackage.__prepare_dockerfile,
+        'application/x-tar': BuildPackage.__prepare_tarball,
+        'application/gzip': BuildPackage.__prepare_tarball,
+        'application/x-gzip': BuildPackage.__prepare_tarball,
     }
 
     c_type = requests_file.headers['content-type']
@@ -84,4 +85,4 @@ class BuildPackage(object):
     with tarfile.open(mode='r|*', fileobj=request_file.raw) as tar_stream:
       safe_extractall(tar_stream, build_dir)
 
-    return build_dir
\ No newline at end of file
+    return build_dir
diff --git a/buildman/buildstatus.py b/buildman/buildstatus.py
index d09876e10..17a7c9d1e 100644
--- a/buildman/buildstatus.py
+++ b/buildman/buildstatus.py
@@ -1,4 +1,3 @@
-from functools import partial
 from data.database import BUILD_PHASE
 
 class StatusHandler(object):
@@ -11,10 +10,10 @@ class StatusHandler(object):
     self._build_logs = build_logs
 
     self._status = {
-      'total_commands': None,
-      'current_command': None,
-      'push_completion': 0.0,
-      'pull_completion': 0.0,
+        'total_commands': None,
+        'current_command': None,
+        'push_completion': 0.0,
+        'pull_completion': 0.0,
     }
 
     # Write the initial status.
@@ -36,7 +35,7 @@ class StatusHandler(object):
   def set_phase(self, phase, extra_data=None):
     if phase == self._current_phase:
       return
-      
+
     self._current_phase = phase
     self._append_log_message(phase, self._build_logs.PHASE, extra_data)
     self._repository_build.phase = phase
@@ -46,4 +45,4 @@ class StatusHandler(object):
     return self._status
 
   def __exit__(self, exc_type, value, traceback):
-    self._build_logs.set_status(self._uuid, self._status)
\ No newline at end of file
+    self._build_logs.set_status(self._uuid, self._status)
diff --git a/buildman/enterprise_builder.py b/buildman/enterprise_builder.py
index 62b67a14c..cc9ce8432 100644
--- a/buildman/enterprise_builder.py
+++ b/buildman/enterprise_builder.py
@@ -6,7 +6,7 @@ from app import app, userfiles as user_files, build_logs, dockerfile_build_queue
 from buildman.manager.enterprise import EnterpriseManager
 from buildman.server import BuilderServer
 
-logger = logging.getLogger(__name__)
+LOGGER = logging.getLogger(__name__)
 
 if __name__ == '__main__':
   logging.basicConfig(level=logging.DEBUG)
@@ -17,4 +17,4 @@ if __name__ == '__main__':
 
   server = BuilderServer(app.config['SERVER_HOSTNAME'], dockerfile_build_queue, build_logs,
                          user_files, EnterpriseManager)
-  server.run(args.host)  
+  server.run(args.host)
diff --git a/buildman/manager/basemanager.py b/buildman/manager/basemanager.py
index 7b5816c9b..c856159b5 100644
--- a/buildman/manager/basemanager.py
+++ b/buildman/manager/basemanager.py
@@ -33,4 +33,4 @@ class BaseManager(object):
     """ Method invoked once a job_item has completed, in some manner. The job_status will be
         one of: incomplete, error, complete. If incomplete, the job should be requeued.
     """
-    raise NotImplementedError
\ No newline at end of file
+    raise NotImplementedError
diff --git a/buildman/manager/enterprise.py b/buildman/manager/enterprise.py
index 1b59a07eb..584a6daf4 100644
--- a/buildman/manager/enterprise.py
+++ b/buildman/manager/enterprise.py
@@ -8,7 +8,7 @@ from buildman.buildcomponent import BuildComponent
 from trollius.coroutines import From
 
 REGISTRATION_REALM = 'registration'
-logger = logging.getLogger(__name__)
+LOGGER = logging.getLogger(__name__)
 
 class DynamicRegistrationComponent(BaseComponent):
   """ Component session that handles dynamic registration of the builder components. """
@@ -17,12 +17,12 @@ class DynamicRegistrationComponent(BaseComponent):
     self.join(REGISTRATION_REALM)
 
   def onJoin(self, details):
-    logger.debug('Registering registration method')
+    LOGGER.debug('Registering registration method')
     yield From(self.register(self._worker_register, u'io.quay.buildworker.register'))
 
   def _worker_register(self):
     realm = self.parent_manager.add_build_component()
-    logger.debug('Registering new build component+worker with realm %s', realm)
+    LOGGER.debug('Registering new build component+worker with realm %s', realm)
     return realm
 
 
@@ -35,8 +35,9 @@ class EnterpriseManager(BaseManager):
     # Add a component which is used by build workers for dynamic registration. Unlike
     # production, build workers in enterprise are long-lived and register dynamically.
     self.register_component(REGISTRATION_REALM, DynamicRegistrationComponent)
-  
+
   def add_build_component(self):
+    """ Adds a new build component for an Enterprise Registry. """
     # Generate a new unique realm ID for the build worker.
     realm = str(uuid.uuid4())
     component = self.register_component(realm, BuildComponent, token="")
@@ -44,6 +45,7 @@ class EnterpriseManager(BaseManager):
     return realm
 
   def schedule(self, build_job, loop):
+    """ Schedules a build for an Enterprise Registry. """
     if self.shutting_down:
       return False
 
diff --git a/buildman/server.py b/buildman/server.py
index 76e6eacc4..e8fa237c7 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -2,24 +2,23 @@ import logging
 import trollius
 
 from autobahn.asyncio.wamp import RouterFactory, RouterSessionFactory
-from autobahn.asyncio.websocket import WampWebSocketServerFactory, WampWebSocketServerProtocol
+from autobahn.asyncio.websocket import WampWebSocketServerFactory
 from autobahn.wamp import types
-from autobahn.wamp.exception import ApplicationError
 
 from aiowsgi import create_server as create_wsgi_server
 from flask import Flask
-from threading import Event, Lock
+from threading import Event
 from trollius.coroutines import From
 
-from buildjob import BuildJob, BuildJobLoadException
+from buildman.buildjob import BuildJob, BuildJobLoadException
 
-logger = logging.getLogger(__name__)
+LOGGER = logging.getLogger(__name__)
 
 WORK_CHECK_TIMEOUT = 10
 TIMEOUT_PERIOD_MINUTES = 20
 RESERVATION_SECONDS = (TIMEOUT_PERIOD_MINUTES + 5) * 60
 
-class BUILD_JOB_RESULT(object):
+class BuildJobResult(object):
   """ Build job result enum """
   INCOMPLETE = 'incomplete'
   COMPLETE = 'complete'
@@ -29,20 +28,22 @@ class BuilderServer(object):
   """ Server which handles both HTTP and WAMP requests, managing the full state of the build
       controller.
   """
-  _loop = None
-  _current_status = 'starting'
-  _current_components = []
-  _job_count = 0
-
   def __init__(self, server_hostname, queue, build_logs, user_files, lifecycle_manager_klass):
-    self._session_factory = RouterSessionFactory(RouterFactory())
+    self._loop = None
+    self._current_status = 'starting'
+    self._current_components = []
+    self._job_count = 0
 
+    self._session_factory = RouterSessionFactory(RouterFactory())
     self._server_hostname = server_hostname
     self._queue = queue
     self._build_logs = build_logs
     self._user_files = user_files
     self._lifecycle_manager = lifecycle_manager_klass(
-      self._register_component, self._unregister_component, self._job_complete)    
+        self._register_component,
+        self._unregister_component,
+        self._job_complete
+    )
 
     self._shutdown_event = Event()
     self._current_status = 'running'
@@ -69,11 +70,11 @@ class BuilderServer(object):
 
     logging.debug('Starting server on port 8080, with controller on port 8181')
     try:
-       loop.run_forever()
+      loop.run_forever()
     except KeyboardInterrupt:
-       pass
+      pass
     finally:
-       loop.close()
+      loop.close()
 
   def close(self):
     logging.debug('Requested server shutdown')
@@ -88,7 +89,7 @@ class BuilderServer(object):
     """
     logging.debug('Registering component with realm %s', realm)
 
-    component = component_klass(types.ComponentConfig(realm = realm), realm=realm, **kwargs)
+    component = component_klass(types.ComponentConfig(realm=realm), realm=realm, **kwargs)
     component.server = self
     component.parent_manager = self._lifecycle_manager
     component.build_logs = self._build_logs
@@ -101,15 +102,15 @@ class BuilderServer(object):
 
   def _unregister_component(self, component):
     logging.debug('Unregistering component with realm %s and token %s',
-      component.builder_realm, component.expected_token)
+                  component.builder_realm, component.expected_token)
 
     self._current_components.remove(component)
     self._session_factory.remove(component)
 
   def _job_complete(self, build_job, job_status):
-    if job_status == BUILD_JOB_RESULT.INCOMPLETE:
+    if job_status == BuildJobResult.INCOMPLETE:
       self._queue.incomplete(build_job.job_item(), restore_retry=True, retry_after=30)
-    elif job_status == BUILD_JOB_RESULT.ERROR:
+    elif job_status == BuildJobResult.ERROR:
       self._queue.incomplete(build_job.job_item(), restore_retry=False)
     else:
       self._queue.complete(build_job.job_item())
@@ -119,42 +120,42 @@ class BuilderServer(object):
     if self._current_status == 'shutting_down' and not self._job_count:
       self._shutdown_event.set()
 
-    # TODO: check for work here?      
+    # TODO:(jschorr) check for work here?
 
   @trollius.coroutine
   def _work_checker(self):
     while self._current_status == 'running':
-      logger.debug('Checking for more work')      
+      LOGGER.debug('Checking for more work')
       job_item = self._queue.get(processing_time=RESERVATION_SECONDS)
       if job_item is None:
-        logger.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT)
+        LOGGER.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT)
         yield From(trollius.sleep(WORK_CHECK_TIMEOUT))
         continue
 
       try:
         build_job = BuildJob(job_item)
       except BuildJobLoadException as irbe:
-        logger.exception(irbe)
+        LOGGER.exception(irbe)
         self._queue.incomplete(job_item, restore_retry=False)
 
-      logger.debug('Build job found. Checking for an avaliable worker.')
+      LOGGER.debug('Build job found. Checking for an avaliable worker.')
       if self._lifecycle_manager.schedule(build_job, self._loop):
         self._job_count = self._job_count + 1
-        logger.debug('Build job scheduled. Running: %s', self._job_count)
+        LOGGER.debug('Build job scheduled. Running: %s', self._job_count)
       else:
-        logger.debug('All workers are busy. Requeuing.')
+        LOGGER.debug('All workers are busy. Requeuing.')
         self._queue.incomplete(job_item, restore_retry=True, retry_after=0)
-        
+
       yield From(trollius.sleep(WORK_CHECK_TIMEOUT))
 
 
   @trollius.coroutine
   def _initialize(self, loop, host):
     self._loop = loop
-    
+
     # Create the WAMP server.
-    transport_factory = WampWebSocketServerFactory(self._session_factory, debug_wamp = False)
-    transport_factory.setProtocolOptions(failByDrop = True)
+    transport_factory = WampWebSocketServerFactory(self._session_factory, debug_wamp=False)
+    transport_factory.setProtocolOptions(failByDrop=True)
 
     # Initialize the controller server and the WAMP server
     create_wsgi_server(self._controller_app, loop=loop, host=host, port=8181)
diff --git a/buildman/workererror.py b/buildman/workererror.py
index 023817e00..c5a5adbd0 100644
--- a/buildman/workererror.py
+++ b/buildman/workererror.py
@@ -5,59 +5,59 @@ class WorkerError(object):
     self._base_message = base_message
 
     self._error_handlers = {
-      'io.quay.builder.buildpackissue': {
-        'message': 'Could not load build package',
-        'is_internal': True
-      },
+        'io.quay.builder.buildpackissue': {
+            'message': 'Could not load build package',
+            'is_internal': True
+        },
 
-      'io.quay.builder.cannotextractbuildpack': {
-        'message': 'Could not extract the contents of the build package'
-      },
+        'io.quay.builder.cannotextractbuildpack': {
+            'message': 'Could not extract the contents of the build package'
+        },
 
-      'io.quay.builder.cannotpullforcache': {
-        'message': 'Could not pull cached image',
-        'is_internal': True
-      },
+        'io.quay.builder.cannotpullforcache': {
+            'message': 'Could not pull cached image',
+            'is_internal': True
+        },
 
-      'io.quay.builder.cannotpullbaseimage': {
-        'message': 'Could not pull base image',
-        'show_base_error': True
-      },
+        'io.quay.builder.cannotpullbaseimage': {
+            'message': 'Could not pull base image',
+            'show_base_error': True
+        },
 
-      'io.quay.builder.internalerror': {
-        'message': 'An internal error occurred while building. Please submit a ticket.'
-      },
+        'io.quay.builder.internalerror': {
+            'message': 'An internal error occurred while building. Please submit a ticket.'
+        },
 
-      'io.quay.builder.buildrunerror': {
-        'message': 'Could not start the build process',
-        'is_internal': True
-      },
+        'io.quay.builder.buildrunerror': {
+            'message': 'Could not start the build process',
+            'is_internal': True
+        },
 
-      'io.quay.builder.builderror': {
-        'message': 'A build step failed',
-        'show_base_error': True
-      },
+        'io.quay.builder.builderror': {
+            'message': 'A build step failed',
+            'show_base_error': True
+        },
 
-      'io.quay.builder.tagissue': {
-        'message': 'Could not tag built image',
-        'is_internal': True
-      },
+        'io.quay.builder.tagissue': {
+            'message': 'Could not tag built image',
+            'is_internal': True
+        },
 
-      'io.quay.builder.pushissue': {
-        'message': 'Could not push built image',
-        'show_base_error': True,
-        'is_internal': True
-      },
+        'io.quay.builder.pushissue': {
+            'message': 'Could not push built image',
+            'show_base_error': True,
+            'is_internal': True
+        },
 
-      'io.quay.builder.dockerconnecterror': {
-        'message': 'Could not connect to Docker daemon',
-        'is_internal': True
-      },
+        'io.quay.builder.dockerconnecterror': {
+            'message': 'Could not connect to Docker daemon',
+            'is_internal': True
+        },
 
-      'io.quay.builder.missingorinvalidargument': {
-        'message': 'Missing required arguments for builder',
-        'is_internal': True
-      }
+        'io.quay.builder.missingorinvalidargument': {
+            'message': 'Missing required arguments for builder',
+            'is_internal': True
+        }
     }
 
   def is_internal_error(self):
@@ -81,7 +81,7 @@ class WorkerError(object):
   def extra_data(self):
     if self._base_message:
       return {
-        'base_error': self._base_message
+          'base_error': self._base_message
       }
 
-    return {}
\ No newline at end of file
+    return {}

From 63f2e7794f555a511577ac46f376446e7fa123d2 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Tue, 18 Nov 2014 16:34:09 -0500
Subject: [PATCH 08/26] Various small fixes

---
 buildman/buildcomponent.py | 30 ++++++++++++++++--------------
 buildman/buildstatus.py    |  3 ++-
 buildman/server.py         |  4 ++--
 3 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/buildman/buildcomponent.py b/buildman/buildcomponent.py
index 3e2458c4a..4daa93156 100644
--- a/buildman/buildcomponent.py
+++ b/buildman/buildcomponent.py
@@ -1,4 +1,5 @@
 import datetime
+import time
 import logging
 import json
 import trollius
@@ -31,20 +32,19 @@ class ComponentStatus(object):
 class BuildComponent(BaseComponent):
   """ An application session component which conducts one (or more) builds. """
 
-  server_hostname = None
-  expected_token = None
-  builder_realm = None
-
-  _component_status = ComponentStatus.JOINING
-  _last_heartbeat = None
-  _current_job = None
-  _build_status = None
-  _image_info = None
-
   def __init__(self, config, realm=None, token=None, **kwargs):
     self.expected_token = token
     self.builder_realm = realm
 
+    self.parent_manager = None
+    self.server_hostname = None
+
+    self._component_status = ComponentStatus.JOINING
+    self._last_heartbeat = None
+    self._current_job = None
+    self._build_status = None
+    self._image_info = None
+
     BaseComponent.__init__(self, config, **kwargs)
 
   def onConnect(self):
@@ -147,7 +147,7 @@ class BuildComponent(BaseComponent):
     LOGGER.debug('With Arguments: %s', build_arguments)
 
     return (self.call("io.quay.builder.build", **build_arguments)
-            .add_done_callback(self._build_complete))
+                .add_done_callback(self._build_complete))
 
   @staticmethod
   def _total_completion(statuses, total_images):
@@ -213,7 +213,9 @@ class BuildComponent(BaseComponent):
     # Parse and update the phase and the status_dict. The status dictionary contains
     # the pull/push progress, as well as the current step index.
     with self._build_status as status_dict:
-      self._build_status.set_phase(phase)
+      if self._build_status.set_phase(phase):
+        LOGGER.debug('Build %s has entered a new phase: %s', self.builder_realm, phase)
+
       BuildComponent._process_pushpull_status(status_dict, phase, docker_data, self._image_info)
 
       # If the current message represents the beginning of a new step, then update the
@@ -234,7 +236,7 @@ class BuildComponent(BaseComponent):
   def _build_failure(self, error_message, exception=None):
     """ Handles and logs a failed build. """
     self._build_status.set_error(error_message, {
-        'internal_error': exception.message if exception else None
+      'internal_error': exception.message if exception else None
     })
 
     build_id = self._current_job.repo_build().uuid
@@ -319,7 +321,7 @@ class BuildComponent(BaseComponent):
       build_status = self._build_status
       if build_status is not None:
         with build_status as status_dict:
-          status_dict['heartbeat'] = int(datetime.time())
+          status_dict['heartbeat'] = int(time.time())
 
       # Check the heartbeat from the worker.
       LOGGER.debug('Checking heartbeat on realm %s', self.builder_realm)
diff --git a/buildman/buildstatus.py b/buildman/buildstatus.py
index 17a7c9d1e..0ce509343 100644
--- a/buildman/buildstatus.py
+++ b/buildman/buildstatus.py
@@ -34,12 +34,13 @@ class StatusHandler(object):
 
   def set_phase(self, phase, extra_data=None):
     if phase == self._current_phase:
-      return
+      return False
 
     self._current_phase = phase
     self._append_log_message(phase, self._build_logs.PHASE, extra_data)
     self._repository_build.phase = phase
     self._repository_build.save()
+    return True
 
   def __enter__(self):
     return self._status
diff --git a/buildman/server.py b/buildman/server.py
index e8fa237c7..ba14fac43 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -154,7 +154,7 @@ class BuilderServer(object):
     self._loop = loop
 
     # Create the WAMP server.
-    transport_factory = WampWebSocketServerFactory(self._session_factory, debug_wamp=False)
+    transport_factory = WampWebSocketServerFactory(self._session_factory, debug_wamp=True)
     transport_factory.setProtocolOptions(failByDrop=True)
 
     # Initialize the controller server and the WAMP server
@@ -162,4 +162,4 @@ class BuilderServer(object):
     yield From(loop.create_server(transport_factory, host, 8080))
 
     # Initialize the work queue checker.
-    yield self._work_checker()
+    yield From(self._work_checker())

From a9fd516dadc5c7373c05b8125cd178e5167e09af Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Tue, 18 Nov 2014 16:35:03 -0500
Subject: [PATCH 09/26] Disable WAMP debug

---
 buildman/server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildman/server.py b/buildman/server.py
index ba14fac43..96a793e01 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -154,7 +154,7 @@ class BuilderServer(object):
     self._loop = loop
 
     # Create the WAMP server.
-    transport_factory = WampWebSocketServerFactory(self._session_factory, debug_wamp=True)
+    transport_factory = WampWebSocketServerFactory(self._session_factory, debug_wamp=False)
     transport_factory.setProtocolOptions(failByDrop=True)
 
     # Initialize the controller server and the WAMP server

From 0763f0d999b725ae5df39baa55196b0917a94543 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Wed, 19 Nov 2014 13:17:53 -0500
Subject: [PATCH 10/26] Initialize BaseComponent members in constructor

---
 buildman/basecomponent.py  | 8 ++++----
 buildman/buildcomponent.py | 5 +++--
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/buildman/basecomponent.py b/buildman/basecomponent.py
index 3cf5bc26b..5a1d04536 100644
--- a/buildman/basecomponent.py
+++ b/buildman/basecomponent.py
@@ -2,10 +2,10 @@ from autobahn.asyncio.wamp import ApplicationSession
 
 class BaseComponent(ApplicationSession):
   """ Base class for all registered component sessions in the server. """
-  server = None
-  parent_manager = None
-  build_logs = None
-  user_files = None
 
   def __init__(self, config, **kwargs):
     ApplicationSession.__init__(self, config)
+    self.server = None
+    self.parent_manager = None
+    self.build_logs = None
+    self.user_files = None
diff --git a/buildman/buildcomponent.py b/buildman/buildcomponent.py
index 4daa93156..3c57e912f 100644
--- a/buildman/buildcomponent.py
+++ b/buildman/buildcomponent.py
@@ -146,8 +146,9 @@ class BuildComponent(BaseComponent):
     LOGGER.debug('Invoking build: %s', self.builder_realm)
     LOGGER.debug('With Arguments: %s', build_arguments)
 
-    return (self.call("io.quay.builder.build", **build_arguments)
-                .add_done_callback(self._build_complete))
+    return (self
+            .call("io.quay.builder.build", **build_arguments)
+            .add_done_callback(self._build_complete))
 
   @staticmethod
   def _total_completion(statuses, total_images):

From d0763862b14dd9eae00ae8aacede4dd17025ed5a Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Thu, 20 Nov 2014 14:36:22 -0500
Subject: [PATCH 11/26] Simple code review changes.

I sneakily also added local-test.sh and renamed run-local to
local-run.sh.
---
 buildman/basecomponent.py       |  1 -
 buildman/buildcomponent.py      |  9 +++++----
 buildman/buildpack.py           | 20 ++++++++++----------
 buildman/enterprise_builder.py  |  2 +-
 buildman/manager/basemanager.py |  2 +-
 buildman/server.py              | 16 ++++++++--------
 run-local.sh => local-run.sh    |  0
 local-test.sh                   |  2 ++
 8 files changed, 27 insertions(+), 25 deletions(-)
 rename run-local.sh => local-run.sh (100%)
 create mode 100755 local-test.sh

diff --git a/buildman/basecomponent.py b/buildman/basecomponent.py
index 5a1d04536..47781dff5 100644
--- a/buildman/basecomponent.py
+++ b/buildman/basecomponent.py
@@ -2,7 +2,6 @@ from autobahn.asyncio.wamp import ApplicationSession
 
 class BaseComponent(ApplicationSession):
   """ Base class for all registered component sessions in the server. """
-
   def __init__(self, config, **kwargs):
     ApplicationSession.__init__(self, config)
     self.server = None
diff --git a/buildman/buildcomponent.py b/buildman/buildcomponent.py
index 3c57e912f..59c07042c 100644
--- a/buildman/buildcomponent.py
+++ b/buildman/buildcomponent.py
@@ -31,7 +31,6 @@ class ComponentStatus(object):
 
 class BuildComponent(BaseComponent):
   """ An application session component which conducts one (or more) builds. """
-
   def __init__(self, config, realm=None, token=None, **kwargs):
     self.expected_token = token
     self.builder_realm = realm
@@ -113,7 +112,7 @@ class BuildComponent(BaseComponent):
       base_image_information['username'] = build_config['pull_credentials'].get('username', '')
       base_image_information['password'] = build_config['pull_credentials'].get('password', '')
 
-    # Retrieve the repository's full name.
+    # Retrieve the repository's fully qualified name.
     repo = build_job.repo_build().repository
     repository_name = repo.namespace_user.username + '/' + repo.name
 
@@ -160,6 +159,7 @@ class BuildComponent(BaseComponent):
 
   @staticmethod
   def _process_pushpull_status(status_dict, current_phase, docker_data, images):
+    """ Processes the status of a push or pull by updating the provided status_dict and images. """
     if not docker_data:
       return
 
@@ -186,6 +186,7 @@ class BuildComponent(BaseComponent):
           BuildComponent._total_completion(images, max(len(images), num_images))
 
   def _on_log_message(self, phase, json_data):
+    """ Tails log messages and updates the build status. """
     # Parse any of the JSON data logged.
     docker_data = {}
     if json_data:
@@ -237,7 +238,7 @@ class BuildComponent(BaseComponent):
   def _build_failure(self, error_message, exception=None):
     """ Handles and logs a failed build. """
     self._build_status.set_error(error_message, {
-      'internal_error': exception.message if exception else None
+        'internal_error': exception.message if exception else None
     })
 
     build_id = self._current_job.repo_build().uuid
@@ -346,7 +347,7 @@ class BuildComponent(BaseComponent):
     # manager.
     if self._current_job is not None:
       if timed_out:
-        self._build_status.set_error('Build worker timed out. Build has been requeued')
+        self._build_status.set_error('Build worker timed out. Build has been requeued.')
 
       self.parent_manager.job_completed(self._current_job, BuildJobResult.INCOMPLETE, self)
       self._build_status = None
diff --git a/buildman/buildpack.py b/buildman/buildpack.py
index cdc4d4b07..9892c65d3 100644
--- a/buildman/buildpack.py
+++ b/buildman/buildpack.py
@@ -17,13 +17,13 @@ class BuildPackage(object):
 
   def __init__(self, requests_file):
     self._mime_processors = {
-        'application/zip': BuildPackage.__prepare_zip,
-        'application/x-zip-compressed': BuildPackage.__prepare_zip,
-        'text/plain': BuildPackage.__prepare_dockerfile,
-        'application/octet-stream': BuildPackage.__prepare_dockerfile,
-        'application/x-tar': BuildPackage.__prepare_tarball,
-        'application/gzip': BuildPackage.__prepare_tarball,
-        'application/x-gzip': BuildPackage.__prepare_tarball,
+        'application/zip': BuildPackage._prepare_zip,
+        'application/x-zip-compressed': BuildPackage._prepare_zip,
+        'text/plain': BuildPackage._prepare_dockerfile,
+        'application/octet-stream': BuildPackage._prepare_dockerfile,
+        'application/x-tar': BuildPackage._prepare_tarball,
+        'application/gzip': BuildPackage._prepare_tarball,
+        'application/x-gzip': BuildPackage._prepare_tarball,
     }
 
     c_type = requests_file.headers['content-type']
@@ -57,7 +57,7 @@ class BuildPackage(object):
     return BuildPackage(buildpack_resource)
 
   @staticmethod
-  def __prepare_zip(request_file):
+  def _prepare_zip(request_file):
     build_dir = mkdtemp(prefix='docker-build-')
 
     # Save the zip file to temp somewhere
@@ -69,7 +69,7 @@ class BuildPackage(object):
     return build_dir
 
   @staticmethod
-  def __prepare_dockerfile(request_file):
+  def _prepare_dockerfile(request_file):
     build_dir = mkdtemp(prefix='docker-build-')
     dockerfile_path = os.path.join(build_dir, "Dockerfile")
     with open(dockerfile_path, 'w') as dockerfile:
@@ -78,7 +78,7 @@ class BuildPackage(object):
     return build_dir
 
   @staticmethod
-  def __prepare_tarball(request_file):
+  def _prepare_tarball(request_file):
     build_dir = mkdtemp(prefix='docker-build-')
 
     # Save the zip file to temp somewhere
diff --git a/buildman/enterprise_builder.py b/buildman/enterprise_builder.py
index cc9ce8432..88ea62b2d 100644
--- a/buildman/enterprise_builder.py
+++ b/buildman/enterprise_builder.py
@@ -12,7 +12,7 @@ if __name__ == '__main__':
   logging.basicConfig(level=logging.DEBUG)
 
   parser = argparse.ArgumentParser()
-  parser.add_argument("--host", type = str, default = "127.0.0.1", help = 'Host IP.')
+  parser.add_argument('--host', type=str, default='127.0.0.1', help='Host IP.')
   args = parser.parse_args()
 
   server = BuilderServer(app.config['SERVER_HOSTNAME'], dockerfile_build_queue, build_logs,
diff --git a/buildman/manager/basemanager.py b/buildman/manager/basemanager.py
index c856159b5..1de6e24df 100644
--- a/buildman/manager/basemanager.py
+++ b/buildman/manager/basemanager.py
@@ -12,7 +12,7 @@ class BaseManager(object):
     """
     raise NotImplementedError
 
-  def schedule(self, build_job):
+  def schedule(self, build_job, loop):
     """ Schedules a queue item to be built. Returns True if the item was properly scheduled
         and False if all workers are busy.
     """
diff --git a/buildman/server.py b/buildman/server.py
index 96a793e01..0e2d7e050 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -61,14 +61,14 @@ class BuilderServer(object):
     self._controller_app = controller_app
 
   def run(self, host):
-    logging.debug('Initializing the lifecycle manager')
+    LOGGER.debug('Initializing the lifecycle manager')
     self._lifecycle_manager.initialize()
 
-    logging.debug('Initializing all members of the event loop')
+    LOGGER.debug('Initializing all members of the event loop')
     loop = trollius.get_event_loop()
     trollius.Task(self._initialize(loop, host))
 
-    logging.debug('Starting server on port 8080, with controller on port 8181')
+    LOGGER.debug('Starting server on port 8080, with controller on port 8181')
     try:
       loop.run_forever()
     except KeyboardInterrupt:
@@ -77,17 +77,17 @@ class BuilderServer(object):
       loop.close()
 
   def close(self):
-    logging.debug('Requested server shutdown')
+    LOGGER.debug('Requested server shutdown')
     self._current_status = 'shutting_down'
     self._lifecycle_manager.shutdown()
     self._shutdown_event.wait()
-    logging.debug('Shutting down server')
+    LOGGER.debug('Shutting down server')
 
   def _register_component(self, realm, component_klass, **kwargs):
     """ Registers a component with the server. The component_klass must derive from
         BaseComponent.
     """
-    logging.debug('Registering component with realm %s', realm)
+    LOGGER.debug('Registering component with realm %s', realm)
 
     component = component_klass(types.ComponentConfig(realm=realm), realm=realm, **kwargs)
     component.server = self
@@ -101,7 +101,7 @@ class BuilderServer(object):
     return component
 
   def _unregister_component(self, component):
-    logging.debug('Unregistering component with realm %s and token %s',
+    LOGGER.debug('Unregistering component with realm %s and token %s',
                   component.builder_realm, component.expected_token)
 
     self._current_components.remove(component)
@@ -120,7 +120,7 @@ class BuilderServer(object):
     if self._current_status == 'shutting_down' and not self._job_count:
       self._shutdown_event.set()
 
-    # TODO:(jschorr) check for work here?
+    # TODO(jschorr): check for work here?
 
   @trollius.coroutine
   def _work_checker(self):
diff --git a/run-local.sh b/local-run.sh
similarity index 100%
rename from run-local.sh
rename to local-run.sh
diff --git a/local-test.sh b/local-test.sh
new file mode 100755
index 000000000..c5cb9e283
--- /dev/null
+++ b/local-test.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+TEST=true python -m unittest discover

From 290c8abeb551532a8da1e02580cd2afbd7ce2eae Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Thu, 20 Nov 2014 15:22:34 -0500
Subject: [PATCH 12/26] Make empty token more readable in logs.

Enterprises use "" for tokens. This was confusing to read in the logs
without making things more clear by adding quotes around the value.
---
 buildman/buildcomponent.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/buildman/buildcomponent.py b/buildman/buildcomponent.py
index 59c07042c..6f362fde2 100644
--- a/buildman/buildcomponent.py
+++ b/buildman/buildcomponent.py
@@ -280,11 +280,11 @@ class BuildComponent(BaseComponent):
 
   def _on_ready(self, token):
     if self._component_status != 'waiting':
-      LOGGER.warning('Build component with token %s is already connected', self.expected_token)
+      LOGGER.warning('Build component (token "%s") is already connected', self.expected_token)
       return
 
     if token != self.expected_token:
-      LOGGER.warning('Builder token mismatch. Expected: %s. Found: %s', self.expected_token, token)
+      LOGGER.warning('Builder token mismatch. Expected: "%s". Found: "%s"', self.expected_token, token)
       return
 
     self._set_status(ComponentStatus.RUNNING)
@@ -339,7 +339,7 @@ class BuildComponent(BaseComponent):
 
   def _timeout(self):
     self._set_status(ComponentStatus.TIMED_OUT)
-    LOGGER.warning('Build component %s timed out', self.expected_token)
+    LOGGER.warning('Build component (token "%s") timed out', self.expected_token)
     self._dispose(timed_out=True)
 
   def _dispose(self, timed_out=False):

From d2be37dbdeb8b156eb94897bb35351317bb57218 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Thu, 20 Nov 2014 15:25:13 -0500
Subject: [PATCH 13/26] remove shebang from local scripts

---
 local-run.sh  | 2 +-
 local-test.sh | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/local-run.sh b/local-run.sh
index 628873fd7..d606624a7 100755
--- a/local-run.sh
+++ b/local-run.sh
@@ -1 +1 @@
-gunicorn -c conf/gunicorn_local.py application:application
\ No newline at end of file
+gunicorn -c conf/gunicorn_local.py application:application
\ No newline at end of file
diff --git a/local-test.sh b/local-test.sh
index c5cb9e283..a54491969 100755
--- a/local-test.sh
+++ b/local-test.sh
@@ -1,2 +1 @@
-#!/bin/sh
 TEST=true python -m unittest discover

From 872c135205b6b99a8c50459c6cc9ba27c6019347 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Thu, 20 Nov 2014 16:06:23 -0500
Subject: [PATCH 14/26] Make ping method static.

Without being static or passing a self parameter, the worker will
receive a runtime WAMP error when they attempt to ping during a
health check, this marks them unhealthy every single time you
attempt a health check.
---
 buildman/buildcomponent.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/buildman/buildcomponent.py b/buildman/buildcomponent.py
index 6f362fde2..f47de4022 100644
--- a/buildman/buildcomponent.py
+++ b/buildman/buildcomponent.py
@@ -274,6 +274,7 @@ class BuildComponent(BaseComponent):
     # Set the component back to a running state.
     self._set_status(ComponentStatus.RUNNING)
 
+  @staticmethod
   def _ping():
     """ Ping pong. """
     return 'pong'

From 300473e7ade3bb7bcc7f2d92d82061bfbbd8a0c3 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Thu, 20 Nov 2014 17:30:57 -0500
Subject: [PATCH 15/26] Partial fix for high CPU on build status pages

---
 static/css/quay.css | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/static/css/quay.css b/static/css/quay.css
index ae5913db8..1065fa586 100644
--- a/static/css/quay.css
+++ b/static/css/quay.css
@@ -4875,4 +4875,12 @@ i.slack-icon {
 #startTriggerDialog #runForm .field-title {
   width: 120px;
   padding-right: 10px;
+}
+
+.progress.active .progress-bar {
+  /* Note: There is a bug in Chrome which results in high CPU usage for active progress-bars
+     due to their animation. This enables the GPU for the rendering, which cuts CPU usage in
+      half (although it is still not great)
+   */
+  transform: translateZ(0);
 }
\ No newline at end of file

From b8e873b00b4cbeec4f864d11ea1c0bcf5dab29b4 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Fri, 21 Nov 2014 14:27:06 -0500
Subject: [PATCH 16/26] Add support to the build system for tracking if/when
 the build manager crashes and make sure builds are restarted within a few
 minutes

---
 buildman/buildcomponent.py             | 11 +++++++++--
 buildman/buildstatus.py                |  7 +++++--
 buildman/manager/basemanager.py        | 15 ++++++++++++++-
 buildman/manager/enterprise.py         |  5 +++++
 buildman/server.py                     | 12 ++++++++++--
 buildman/workererror.py                |  3 ---
 data/database.py                       |  1 +
 data/queue.py                          |  8 ++++++--
 endpoints/api/build.py                 | 11 ++++++++++-
 static/css/quay.css                    |  4 ++++
 static/directives/build-log-error.html |  1 -
 static/js/app.js                       |  8 ++++++++
 static/js/controllers.js               |  2 +-
 13 files changed, 73 insertions(+), 15 deletions(-)

diff --git a/buildman/buildcomponent.py b/buildman/buildcomponent.py
index f47de4022..9b29ad5b6 100644
--- a/buildman/buildcomponent.py
+++ b/buildman/buildcomponent.py
@@ -258,7 +258,8 @@ class BuildComponent(BaseComponent):
       worker_error = WorkerError(aex.error, aex.kwargs.get('base_error'))
 
       # Write the error to the log.
-      self._build_status.set_error(worker_error.public_message(), worker_error.extra_data())
+      self._build_status.set_error(worker_error.public_message(), worker_error.extra_data(),
+                                   internal_error=worker_error.is_internal_error())
 
       # Mark the build as completed.
       if worker_error.is_internal_error():
@@ -326,6 +327,12 @@ class BuildComponent(BaseComponent):
         with build_status as status_dict:
           status_dict['heartbeat'] = int(time.time())
 
+
+      # Mark the build item.
+      current_job = self._current_job
+      if current_job is not None:
+        self.parent_manager.job_heartbeat(current_job)
+
       # Check the heartbeat from the worker.
       LOGGER.debug('Checking heartbeat on realm %s', self.builder_realm)
       if not self._last_heartbeat:
@@ -348,7 +355,7 @@ class BuildComponent(BaseComponent):
     # manager.
     if self._current_job is not None:
       if timed_out:
-        self._build_status.set_error('Build worker timed out. Build has been requeued.')
+        self._build_status.set_error('Build worker timed out', internal_error=True)
 
       self.parent_manager.job_completed(self._current_job, BuildJobResult.INCOMPLETE, self)
       self._build_status = None
diff --git a/buildman/buildstatus.py b/buildman/buildstatus.py
index 0ce509343..68b8cd5e3 100644
--- a/buildman/buildstatus.py
+++ b/buildman/buildstatus.py
@@ -28,8 +28,11 @@ class StatusHandler(object):
   def set_command(self, command, extra_data=None):
     self._append_log_message(command, self._build_logs.COMMAND, extra_data)
 
-  def set_error(self, error_message, extra_data=None):
-    self.set_phase(BUILD_PHASE.ERROR)
+  def set_error(self, error_message, extra_data=None, internal_error=False):
+    self.set_phase(BUILD_PHASE.INTERNAL_ERROR if internal_error else BUILD_PHASE.ERROR)
+
+    extra_data = extra_data or {}
+    extra_data['internal_error'] = internal_error
     self._append_log_message(error_message, self._build_logs.ERROR, extra_data)
 
   def set_phase(self, phase, extra_data=None):
diff --git a/buildman/manager/basemanager.py b/buildman/manager/basemanager.py
index 1de6e24df..f66054c45 100644
--- a/buildman/manager/basemanager.py
+++ b/buildman/manager/basemanager.py
@@ -1,10 +1,23 @@
 class BaseManager(object):
   """ Base for all worker managers. """
-  def __init__(self, register_component, unregister_component, job_complete_callback):
+  def __init__(self, register_component, unregister_component, job_heartbeat_callback,
+               job_complete_callback):
     self.register_component = register_component
     self.unregister_component = unregister_component
+    self.job_heartbeat_callback = job_heartbeat_callback
     self.job_complete_callback = job_complete_callback
 
+  def job_heartbeat(self, build_job):
+    """ Method invoked to tell the manager that a job is still running. This method will be called
+        every few minutes. """
+    self.job_heartbeat_callback(build_job)
+
+  def setup_time(self):
+    """ Returns the number of seconds that the build system should wait before allowing the job
+        to be picked up again after called 'schedule'.
+    """
+    raise NotImplementedError
+
   def shutdown(self):
     """ Indicates that the build controller server is in a shutdown state and that no new jobs
         or workers should be performed. Existing workers should be cleaned up once their jobs
diff --git a/buildman/manager/enterprise.py b/buildman/manager/enterprise.py
index 584a6daf4..9abe5f3c0 100644
--- a/buildman/manager/enterprise.py
+++ b/buildman/manager/enterprise.py
@@ -36,6 +36,11 @@ class EnterpriseManager(BaseManager):
     # production, build workers in enterprise are long-lived and register dynamically.
     self.register_component(REGISTRATION_REALM, DynamicRegistrationComponent)
 
+  def setup_time(self):
+    # Builders are already registered, so the setup time should be essentially instant. We therefore
+    # only return a minute here.
+    return 60
+
   def add_build_component(self):
     """ Adds a new build component for an Enterprise Registry. """
     # Generate a new unique realm ID for the build worker.
diff --git a/buildman/server.py b/buildman/server.py
index 0e2d7e050..fd5d4f409 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -9,14 +9,17 @@ from aiowsgi import create_server as create_wsgi_server
 from flask import Flask
 from threading import Event
 from trollius.coroutines import From
+from datetime import datetime, timedelta
 
 from buildman.buildjob import BuildJob, BuildJobLoadException
+from data.queue import WorkQueue
 
 LOGGER = logging.getLogger(__name__)
 
 WORK_CHECK_TIMEOUT = 10
 TIMEOUT_PERIOD_MINUTES = 20
-RESERVATION_SECONDS = (TIMEOUT_PERIOD_MINUTES + 5) * 60
+JOB_TIMEOUT_SECONDS = 300
+MINIMUM_JOB_EXTENSION = timedelta(minutes=2)
 
 class BuildJobResult(object):
   """ Build job result enum """
@@ -42,6 +45,7 @@ class BuilderServer(object):
     self._lifecycle_manager = lifecycle_manager_klass(
         self._register_component,
         self._unregister_component,
+        self._job_heartbeat,
         self._job_complete
     )
 
@@ -107,6 +111,10 @@ class BuilderServer(object):
     self._current_components.remove(component)
     self._session_factory.remove(component)
 
+  def _job_heartbeat(self, build_job):
+    WorkQueue.extend_processing(build_job.job_item(), seconds_from_now=JOB_TIMEOUT_SECONDS,
+                                retry_count=1, minimum_extension=MINIMUM_JOB_EXTENSION)
+
   def _job_complete(self, build_job, job_status):
     if job_status == BuildJobResult.INCOMPLETE:
       self._queue.incomplete(build_job.job_item(), restore_retry=True, retry_after=30)
@@ -126,7 +134,7 @@ class BuilderServer(object):
   def _work_checker(self):
     while self._current_status == 'running':
       LOGGER.debug('Checking for more work')
-      job_item = self._queue.get(processing_time=RESERVATION_SECONDS)
+      job_item = self._queue.get(processing_time=self._lifecycle_manager.setup_time())
       if job_item is None:
         LOGGER.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT)
         yield From(trollius.sleep(WORK_CHECK_TIMEOUT))
diff --git a/buildman/workererror.py b/buildman/workererror.py
index c5a5adbd0..8271976e4 100644
--- a/buildman/workererror.py
+++ b/buildman/workererror.py
@@ -73,9 +73,6 @@ class WorkerError(object):
     if handler.get('show_base_error', False) and self._base_message:
       message = message + ': ' + self._base_message
 
-    if handler.get('is_internal', False):
-      message = message + '\nThe build will be retried shortly'
-
     return message
 
   def extra_data(self):
diff --git a/data/database.py b/data/database.py
index 6562e1f63..e8874f692 100644
--- a/data/database.py
+++ b/data/database.py
@@ -370,6 +370,7 @@ class RepositoryTag(BaseModel):
 class BUILD_PHASE(object):
   """ Build phases enum """
   ERROR = 'error'
+  INTERNAL_ERROR = 'internalerror'
   UNPACKING = 'unpacking'
   PULLING = 'pulling'
   BUILDING = 'building'
diff --git a/data/queue.py b/data/queue.py
index 548ee380c..203d31eeb 100644
--- a/data/queue.py
+++ b/data/queue.py
@@ -120,10 +120,14 @@ class WorkQueue(object):
       self._currently_processing = False
 
   @staticmethod
-  def extend_processing(queue_item, seconds_from_now):
+  def extend_processing(queue_item, seconds_from_now, retry_count=None,
+                        minimum_extension=MINIMUM_EXTENSION):
     new_expiration = datetime.utcnow() + timedelta(seconds=seconds_from_now)
 
     # Only actually write the new expiration to the db if it moves the expiration some minimum
-    if new_expiration - queue_item.processing_expires > MINIMUM_EXTENSION:
+    if new_expiration - queue_item.processing_expires > minimum_extension:
+      if retry_count is not None:
+        queue_item.retries_remaining = retry_count
+
       queue_item.processing_expires = new_expiration
       queue_item.save()
diff --git a/endpoints/api/build.py b/endpoints/api/build.py
index adf6f43ec..2af1e0c8b 100644
--- a/endpoints/api/build.py
+++ b/endpoints/api/build.py
@@ -1,5 +1,7 @@
 import logging
 import json
+import time
+import datetime
 
 from flask import request, redirect
 
@@ -9,7 +11,7 @@ from endpoints.api import (RepositoryParamResource, parse_args, query_param, nic
                            ApiResource, internal_only, format_date, api, Unauthorized, NotFound)
 from endpoints.common import start_build
 from endpoints.trigger import BuildTrigger
-from data import model
+from data import model, database
 from auth.auth_context import get_authenticated_user
 from auth.permissions import ModifyRepositoryPermission, AdministerOrganizationPermission
 from data.buildlogs import BuildStatusRetrievalError
@@ -65,6 +67,13 @@ def build_status_view(build_obj, can_write=False):
     status = {}
     phase = 'cannot_load'
 
+  # If the status contains a heartbeat, then check to see if has been written in the last few
+  # minutes. If not, then the build timed out.
+  if status is not None and 'heartbeat' in status and status['heartbeat']:
+    heartbeat = datetime.datetime.fromtimestamp(status['heartbeat'])
+    if datetime.datetime.now() - heartbeat > datetime.timedelta(minutes=1):
+      phase = database.BUILD_PHASE.INTERNAL_ERROR
+
   logger.debug('Can write: %s job_config: %s', can_write, build_obj.job_config)
   resp = {
     'id': build_obj.uuid,
diff --git a/static/css/quay.css b/static/css/quay.css
index 1065fa586..2a2f80136 100644
--- a/static/css/quay.css
+++ b/static/css/quay.css
@@ -864,6 +864,10 @@ i.toggle-icon:hover {
   background-color: red;
 }
 
+.phase-icon.internalerror {
+  background-color: #DFFF00;
+}
+
 .phase-icon.waiting, .phase-icon.unpacking, .phase-icon.starting, .phase-icon.initializing {
   background-color: #ddd;
 }
diff --git a/static/directives/build-log-error.html b/static/directives/build-log-error.html
index 13b399bb9..c012a623f 100644
--- a/static/directives/build-log-error.html
+++ b/static/directives/build-log-error.html
@@ -9,7 +9,6 @@
     </span>
   </span>
 
-  
   <div class="alert alert-danger" ng-if="error.message == 'HTTP code: 403' && getLocalPullInfo().isLocal">
     <div ng-if="getLocalPullInfo().login">
       Note: The credentials <b>{{ getLocalPullInfo().login.username }}</b> for registry <b>{{ getLocalPullInfo().login.registry }}</b> cannot
diff --git a/static/js/app.js b/static/js/app.js
index 1f5d53574..3440468d4 100644
--- a/static/js/app.js
+++ b/static/js/app.js
@@ -4746,6 +4746,11 @@ quayApp.directive('buildLogError', function () {
       'entries': '=entries'
     },
     controller: function($scope, $element, Config) {
+      $scope.isInternalError = function() {
+        var entry = $scope.entries[$scope.entries.length - 1];
+        return entry && entry.data && entry.data['internal_error'];
+      };
+
       $scope.getLocalPullInfo = function() {
         if ($scope.entries.__localpull !== undefined) {
           return $scope.entries.__localpull;
@@ -5690,6 +5695,9 @@ quayApp.directive('buildMessage', function () {
           
           case 'error':
             return 'Dockerfile build failed';
+
+          case 'internalerror':
+            return 'A system error occurred while building - Will be retried shortly';
         }
       };
     }
diff --git a/static/js/controllers.js b/static/js/controllers.js
index 07041ae7a..6a4051e50 100644
--- a/static/js/controllers.js
+++ b/static/js/controllers.js
@@ -1238,7 +1238,7 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope
 
     ApiService.getRepoBuildStatus(null, params, true).then(function(resp) {
       if (build != $scope.currentBuild) { callback(false); return; }
-      
+
       // Note: We use extend here rather than replacing as Angular is depending on the
       // root build object to remain the same object.
       var matchingBuilds = $.grep($scope.builds, function(elem) {

From c48559ee3d4ed579c3480441ea49de550e076c72 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Sun, 23 Nov 2014 15:45:55 -0500
Subject: [PATCH 17/26] Better status message for internal error.

This also trims whitespace in app.js.
---
 static/js/app.js | 308 +++++++++++++++++++++++------------------------
 1 file changed, 154 insertions(+), 154 deletions(-)

diff --git a/static/js/app.js b/static/js/app.js
index 3440468d4..2ffde83b5 100644
--- a/static/js/app.js
+++ b/static/js/app.js
@@ -1,7 +1,7 @@
 var TEAM_PATTERN = '^[a-zA-Z][a-zA-Z0-9]+$';
 var ROBOT_PATTERN = '^[a-zA-Z][a-zA-Z0-9]{3,29}$';
 
-$.fn.clipboardCopy = function() { 
+$.fn.clipboardCopy = function() {
   if (zeroClipboardSupported) {
     (new ZeroClipboard($(this)));
     return true;
@@ -17,7 +17,7 @@ ZeroClipboard.config({
 });
 
 ZeroClipboard.on("error", function(e) {
-  zeroClipboardSupported = false;  
+  zeroClipboardSupported = false;
 });
 
 ZeroClipboard.on('aftercopy', function(e) {
@@ -67,7 +67,7 @@ function clickElement(el){
 
 function getFirstTextLine(commentString) {
   if (!commentString) { return ''; }
-      
+
   var lines = commentString.split('\n');
   var MARKDOWN_CHARS = {
     '#': true,
@@ -113,7 +113,7 @@ function createOrganizationTeam(ApiService, orgname, teamname, callback) {
     'orgname': orgname,
     'teamname': teamname
   };
-  
+
   ApiService.updateOrganizationTeam(data, params)
     .then(callback, ApiService.errorDisplay('Cannot create team'));
 }
@@ -206,7 +206,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
           that.showAdditionalEntries_();
         }, 10);
       };
-      
+
       _ViewArray.prototype.stopTimer_ = function() {
         if (this.timerRef_) {
           $interval.cancel(this.timerRef_);
@@ -286,7 +286,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
           that.call_();
         }, this.sleeptime_)
       };
-    
+
       var service = {
         'create': function(scope, requester, opt_sleeptime) {
           return new _PollChannel(scope, requester, opt_sleeptime);
@@ -307,7 +307,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
       dataFileService.tryAsZip_ = function(buf, success, failure) {
         var zip = null;
         var zipFiles = null;
-        try {      
+        try {
           var zip = new JSZip(buf);
           zipFiles = zip.files;
         } catch (e) {
@@ -391,7 +391,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
                 success(processed);
                 break;
           }
-        });        
+        });
       };
 
       dataFileService.blobToString = function(blob, callback) {
@@ -462,7 +462,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
 
     $provide.factory('UIService', [function() {
       var uiService = {};
-      
+
       uiService.hidePopover = function(elem) {
         var popover = $(elem).data('bs.popover');
         if (popover) {
@@ -482,7 +482,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
           popover.show();
         }, 500);
       };
-      
+
       uiService.showFormError = function(elem, result) {
         var message =  result.data['message'] || result.data['error_description'] || '';
         if (message) {
@@ -491,7 +491,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
           uiService.hidePopover(elem);
         }
       };
-      
+
       return uiService;
     }]);
 
@@ -510,14 +510,14 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
           .replace(/>/g, "&gt;")
           .replace(/"/g, "&quot;")
           .replace(/'/g, "&#039;");
-        
+
         return adjusted;
       };
-      
+
       utilService.textToSafeHtml = function(text) {
         return $sanitize(utilService.escapeHtmlString(text));
       };
-      
+
       return utilService;
     }]);
 
@@ -578,7 +578,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
           }, 1000);
         }
       };
-      
+
       var pingUrlInternal = function($scope, url, callback) {
         var path = url + '?cb=' + (Math.random() * 100);
 	var start = new Date();
@@ -605,7 +605,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
 
       pingService.pingUrl = function($scope, url, callback) {
         if (pingCache[url]) {
-          invokeCallback($scope, pingCache[url]['pings'], callback);          
+          invokeCallback($scope, pingCache[url]['pings'], callback);
           return;
         }
 
@@ -615,12 +615,12 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
           pingUrlInternal($scope, url, callback);
         }, 1000);
       };
-      
+
       return pingService;
     }]);
 
 
-    $provide.factory('TriggerService', ['UtilService', '$sanitize', 'KeyService', 
+    $provide.factory('TriggerService', ['UtilService', '$sanitize', 'KeyService',
         function(UtilService, $sanitize, KeyService) {
       var triggerService = {};
 
@@ -643,13 +643,13 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
           ],
 
           'get_redirect_url': function(namespace, repository) {
-            var redirect_uri = KeyService['githubRedirectUri'] + '/trigger/' + 
+            var redirect_uri = KeyService['githubRedirectUri'] + '/trigger/' +
               namespace + '/' + repository;
 
             var authorize_url = KeyService['githubTriggerAuthorizeUrl'];
             var client_id = KeyService['githubTriggerClientId'];
 
-            return authorize_url + 'client_id=' + client_id + 
+            return authorize_url + 'client_id=' + client_id +
               '&scope=repo,user:email&redirect_uri=' + redirect_uri;
           }
         }
@@ -758,7 +758,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
         if (typeof description != 'string') {
           description = description(metadata);
         }
-          
+
         for (var key in metadata) {
           if (metadata.hasOwnProperty(key)) {
             var value = metadata[key] != null ? metadata[key] : '(Unknown)';
@@ -807,7 +807,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
           if (command.length > 2 && command[0] == '/bin/sh' && command[1] == '-c') {
             return command[2];
           }
-          
+
           return command.join(' ');
         };
 
@@ -838,7 +838,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
       features.hasFeature = function(name) {
         return !!features.getFeature(name);
       };
-      
+
       features.matchesFeatures = function(list) {
         for (var i = 0; i < list.length; ++i) {
           var value = features.getFeature(list[i]);
@@ -982,7 +982,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
         if (userRelatedResource) {
           var operations = userRelatedResource['operations'];
           for (var i = 0; i < operations.length; ++i) {
-            var operation = operations[i];          
+            var operation = operations[i];
             if (operation['method'].toLowerCase() == method) {
               return operation['nickname'];
             }
@@ -996,7 +996,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
         var name = endpointResource['name'];
         var operations = endpointResource['operations'];
         for (var i = 0; i < operations.length; ++i) {
-          var operation = operations[i];          
+          var operation = operations[i];
           buildMethodsForOperation(operation, endpointResource, resourceMap);
         }
       };
@@ -1004,7 +1004,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
       var freshLoginFailCheck = function(opName, opArgs) {
         return function(resp) {
           var deferred = $q.defer();
-          
+
           // If the error is a fresh login required, show the dialog.
           if (resp.status == 401 && resp.data['error_type'] == 'fresh_login_required') {
             var verifyNow = function() {
@@ -1028,12 +1028,12 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
                 deferred.reject({'data': {'message': 'Invalid verification credentials'}});
               });
             };
-           
+
             var box = bootbox.dialog({
               "message": 'It has been more than a few minutes since you last logged in, ' +
-                'so please verify your password to perform this sensitive operation:' + 
+                'so please verify your password to perform this sensitive operation:' +
                 '<form style="margin-top: 10px" action="javascript:void(0)">' +
-                '<input id="freshPassword" class="form-control" type="password" placeholder="Current Password">' + 
+                '<input id="freshPassword" class="form-control" type="password" placeholder="Current Password">' +
                 '</form>',
               "title": 'Please Verify',
               "buttons": {
@@ -1056,7 +1056,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
               box.find("input").focus();
               box.find("form").submit(function() {
                 if (!$('#freshPassword').val()) { return; }
-                
+
                 box.modal('hide');
                 verifyNow();
               });
@@ -1085,7 +1085,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
               'ignoreLoadingBar': true
             });
           }
-          
+
           var opObj = one['custom' + method.toUpperCase()](opt_options);
 
           // If the operation requires_fresh_login, then add a specialized error handler that
@@ -1176,7 +1176,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
 
       return apiService;
     }]);
-    
+
     $provide.factory('CookieService', ['$cookies', '$cookieStore', function($cookies, $cookieStore) {
       var cookieService = {};
       cookieService.putPermanent = function(name, value) {
@@ -1219,7 +1219,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
         scope.$watch(function () { return userService.currentUser(); }, function (currentUser) {
           scope.user = currentUser;
           if (opt_callback) {
-            opt_callback(currentUser); 
+            opt_callback(currentUser);
           }
         }, true);
       };
@@ -1273,7 +1273,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
         ApiService.getLoggedInUser().then(function(loadedUser) {
           handleUserResponse(loadedUser);
         }, function() {
-          handleUserResponse({'anonymous': true});        
+          handleUserResponse({'anonymous': true});
         });
       };
 
@@ -1310,7 +1310,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
         var org = userService.getOrganization(namespace);
         return !!org;
       };
-        
+
       userService.currentUser = function() {
         return userResponse;
       };
@@ -1337,22 +1337,22 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
 
     if (Features.BUILD_SUPPORT) {
       var buildEvents = [
-        { 
+        {
           'id': 'build_queued',
           'title': 'Dockerfile Build Queued',
           'icon': 'fa-tasks'
         },
-        { 
+        {
           'id': 'build_start',
           'title': 'Dockerfile Build Started',
           'icon': 'fa-circle-o-notch'
         },
-        { 
+        {
           'id': 'build_success',
           'title': 'Dockerfile Build Successfully Completed',
           'icon': 'fa-check-circle-o'
         },
-        { 
+        {
           'id': 'build_failure',
           'title': 'Dockerfile Build Failed',
           'icon': 'fa-times-circle-o'
@@ -1376,7 +1376,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
           }
         ]
       },
-      { 
+      {
         'id': 'email',
         'title': 'E-mail',
         'icon': 'fa-envelope',
@@ -1612,7 +1612,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
         var params = {
           'uuid': notification.id
         };
-        
+
         ApiService.updateUserNotification(notification, params, function() {
           notificationService.update();
         }, ApiService.errorDisplay('Could not update notification'));
@@ -1762,7 +1762,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
 
       return keyService;
     }]);
-  
+
     $provide.factory('PlanService', ['KeyService', 'UserService', 'CookieService', 'ApiService', 'Features', 'Config',
                                      function(KeyService, UserService, CookieService, ApiService, Features, Config) {
       var plans = null;
@@ -1842,7 +1842,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
         CookieService.clear('quay.notedplan');
         return planId;
       };
-      
+
       planService.handleCardError = function(resp) {
         if (!planService.isCardError(resp)) { return; }
 
@@ -1902,14 +1902,14 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
 
       planService.getPlanIncludingDeprecated = function(planId, callback) {
         planService.verifyLoaded(function() {
-          if (planDict[planId]) {            
+          if (planDict[planId]) {
             callback(planDict[planId]);
           }
         });
       };
 
       planService.getMinimumPlan = function(privateCount, isBusiness, callback) {
-        planService.getPlans(function(plans) {          
+        planService.getPlans(function(plans) {
           for (var i = 0; i < plans.length; i++) {
             var plan = plans[i];
             if (plan.privateRepos >= privateCount) {
@@ -1975,7 +1975,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
               planService.showSubscribeDialog($scope, orgname, planId, callbacks, title, /* async */true);
               return;
             }
-        
+
             previousSubscribeFailure = false;
 
             planService.setSubscription(orgname, planId, callbacks['success'], function(resp) {
@@ -1998,11 +1998,11 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
         var submitToken = function(token) {
           if (submitted) { return; }
           submitted = true;
-          $scope.$apply(function() {            
+          $scope.$apply(function() {
             if (callbacks['started']) {
               callbacks['started']();
             }
-            
+
             var cardInfo = {
               'token': token.id
             };
@@ -2069,7 +2069,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
                 "className": "btn-default"
               }
             }
-          });          
+          });
           return;
         }
 
@@ -2120,7 +2120,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
     return {
       require: 'ngModel',
       link: function(scope, elem, attrs, ctrl) {
-        scope.$watch(function() {        
+        scope.$watch(function() {
           return $parse(attrs.match)(scope) === ctrl.$modelValue;
         }, function(currentValue) {
           ctrl.$setValidity('mismatch', currentValue);
@@ -2215,7 +2215,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
 
   if (window.__config && window.__config.MIXPANEL_KEY) {
     quayApp.config(['$analyticsProvider', function($analyticsProvider) {
-      $analyticsProvider.virtualPageviews(true);      
+      $analyticsProvider.virtualPageviews(true);
     }]);
   }
 
@@ -2331,7 +2331,7 @@ quayApp.directive('quayClasses', function(Features, Config) {
     priority: 580,
     restrict: 'A',
     link: function($scope, $element, $attr, ctrl, $transclude) {
-      
+
       // Borrowed from ngClass.
       function flattenClasses(classVal) {
         if(angular.isArray(classVal)) {
@@ -2345,10 +2345,10 @@ quayApp.directive('quayClasses', function(Features, Config) {
           });
           return classes.join(' ');
         }
-        
+
         return classVal;
       }
-      
+
       function removeClass(classVal) {
         $attr.$removeClass(flattenClasses(classVal));
       }
@@ -2416,7 +2416,7 @@ quayApp.directive('quayInclude', function($compile, $templateCache, $http, Featu
       if (!templatePath) {
         return;
       }
-      
+
       var promise = getTemplate(templatePath).success(function(html) {
         $element.html(html);
       }).then(function (response) {
@@ -2495,7 +2495,7 @@ quayApp.directive('applicationInfo', function () {
       'application': '=application'
     },
     controller: function($scope, $element, ApiService) {
-      
+
     }
   };
   return directiveDefinitionObject;
@@ -2572,7 +2572,7 @@ quayApp.directive('repoBreadcrumb', function () {
       'subsection': '=subsection',
       'subsectionIcon': '=subsectionIcon'
     },
-    controller: function($scope, $element) {     
+    controller: function($scope, $element) {
     }
   };
   return directiveDefinitionObject;
@@ -2586,7 +2586,7 @@ quayApp.directive('focusablePopoverContent', ['$timeout', '$popover', function (
       var hide = function() {
         $body.off('click');
 
-        if (!scope) { return; }        
+        if (!scope) { return; }
         scope.$apply(function() {
           if (!scope) { return; }
           scope.$hide();
@@ -2594,7 +2594,7 @@ quayApp.directive('focusablePopoverContent', ['$timeout', '$popover', function (
       };
 
       scope.$on('$destroy', function() {
-        $body.off('click');        
+        $body.off('click');
       });
 
       $timeout(function() {
@@ -2646,7 +2646,7 @@ quayApp.directive('copyBox', function () {
       var number = $rootScope.__copyBoxIdCounter || 0;
       $rootScope.__copyBoxIdCounter = number + 1;
       $scope.inputId = "copy-box-input-" + number;
-      
+
       var button = $($element).find('.copy-icon');
       var input = $($element).find('input');
 
@@ -2727,7 +2727,7 @@ quayApp.directive('externalLoginButton', function () {
         $scope.signInStarted({'service': service});
 
         var url = KeyService.getExternalLoginUrl(service, $scope.action || 'login');
-        
+
         // Save the redirect URL in a cookie so that we can redirect back after the service returns to us.
         var redirectURL = $scope.redirectUrl || window.location.toString();
         CookieService.putPermanent('quay.redirectAfterLoad', redirectURL);
@@ -2825,7 +2825,7 @@ quayApp.directive('signinForm', function () {
             $scope.cancelInterval();
 
             $scope.tryAgainSoon = result.headers('Retry-After');
-            $scope.tryAgainInterval = $interval(function() {              
+            $scope.tryAgainInterval = $interval(function() {
               $scope.tryAgainSoon--;
               if ($scope.tryAgainSoon <= 0) {
                 $scope.cancelInterval();
@@ -2855,10 +2855,10 @@ quayApp.directive('signupForm', function () {
 
       'userRegistered': '&userRegistered'
     },
-    controller: function($scope, $location, $timeout, ApiService, KeyService, UserService, Config, UIService) {   
+    controller: function($scope, $location, $timeout, ApiService, KeyService, UserService, Config, UIService) {
       $('.form-signup').popover();
 
-      $scope.awaitingConfirmation = false;      
+      $scope.awaitingConfirmation = false;
       $scope.registering = false;
 
       $scope.register = function() {
@@ -2872,7 +2872,7 @@ quayApp.directive('signupForm', function () {
         ApiService.createNewUser($scope.newUser).then(function(resp) {
           $scope.registering  = false;
           $scope.awaitingConfirmation = !!resp['awaiting_verification'];
-          
+
           if (Config.MIXPANEL_KEY) {
             mixpanel.alias($scope.newUser.username);
           }
@@ -3016,7 +3016,7 @@ quayApp.directive('dockerAuthDialog', function (Config) {
           $('#dockerauthmodal').modal('hide');
           return;
         }
-         
+
         $('#copyClipboard').clipboardCopy();
         $('#dockerauthmodal').modal({});
       };
@@ -3047,7 +3047,7 @@ quayApp.filter('regex', function() {
       if (m && m[0].length == input[i].length) {
         out.push(input[i]);
       }
-    }      
+    }
     return out;
   };
 });
@@ -3076,7 +3076,7 @@ quayApp.filter('visibleLogFilter', function () {
     if (!allowed) {
       return logs;
     }
-    
+
     var filtered = [];
     angular.forEach(logs, function (log) {
       if (allowed[log.kind]) {
@@ -3117,7 +3117,7 @@ quayApp.directive('billingInvoices', function () {
         if (!$scope.makevisible || !isValid) {
           return;
         }
-        
+
         $scope.loading = true;
 
         ApiService.listInvoices($scope.organization).then(function(resp) {
@@ -3158,7 +3158,7 @@ quayApp.directive('logsView', function () {
       $scope.kindsAllowed = null;
       $scope.chartVisible = true;
       $scope.logsPath = '';
-      
+
       var datetime = new Date();
       $scope.logStartDate = new Date(datetime.getUTCFullYear(), datetime.getUTCMonth(), datetime.getUTCDate() - 7);
       $scope.logEndDate = new Date(datetime.getUTCFullYear(), datetime.getUTCMonth(), datetime.getUTCDate());
@@ -3269,7 +3269,7 @@ quayApp.directive('logsView', function () {
               return 'Create default permission: {role} for {delegate_team}' + defaultPermSuffix(metadata);
             }
           },
-          'modify_prototype_permission': function(metadata) {              
+          'modify_prototype_permission': function(metadata) {
             if (metadata.delegate_user) {
               return 'Modify default permission: {role} (from {original_role}) for {delegate_user}' + defaultPermSuffix(metadata);
             } else if (metadata.delegate_team) {
@@ -3442,7 +3442,7 @@ quayApp.directive('logsView', function () {
         return $scope.chart.getColor(kind);
       };
 
-      $scope.getDescription = function(log) {       
+      $scope.getDescription = function(log) {
         log.metadata['_ip'] = log.ip ? log.ip : null;
         return StringBuilderService.buildString(logDescriptions[log.kind] || log.kind, log.metadata);
       };
@@ -3472,10 +3472,10 @@ quayApp.directive('applicationManager', function () {
       'organization': '=organization',
       'makevisible': '=makevisible'
     },
-    controller: function($scope, $element, ApiService) {      
+    controller: function($scope, $element, ApiService) {
       $scope.loading = false;
       $scope.applications = [];
-     
+
       $scope.createApplication = function(appName) {
         if (!appName) { return; }
 
@@ -3552,7 +3552,7 @@ quayApp.directive('robotsManager', function () {
         $scope.shownRobot = info;
         $scope.showRobotCounter++;
       };
-      
+
       $scope.findRobotIndexByName = function(name) {
         for (var i = 0; i < $scope.robots.length; ++i) {
           if ($scope.robots[i].name == name) {
@@ -3942,13 +3942,13 @@ quayApp.directive('repoSearch', function () {
       $scope.$watch( function () { return UserService.currentUser(); }, function (currentUser) {
         ++searchToken;
       }, true);
-        
+
       var repoHound = new Bloodhound({
         name: 'repositories',
         remote: {
           url: '/api/v1/find/repository?query=%QUERY',
           replace: function (url, uriEncodedQuery) {
-            url = url.replace('%QUERY', uriEncodedQuery);              
+            url = url.replace('%QUERY', uriEncodedQuery);
             url += '&cb=' + searchToken;
             return url;
           },
@@ -3965,8 +3965,8 @@ quayApp.directive('repoSearch', function () {
             return datums;
           }
         },
-        datumTokenizer: function(d) { 
-          return Bloodhound.tokenizers.whitespace(d.val); 
+        datumTokenizer: function(d) {
+          return Bloodhound.tokenizers.whitespace(d.val);
         },
         queryTokenizer: Bloodhound.tokenizers.whitespace
       });
@@ -3974,7 +3974,7 @@ quayApp.directive('repoSearch', function () {
 
       var element = $($element[0].childNodes[0]);
       element.typeahead({ 'highlight': true }, {
-        source: repoHound.ttAdapter(),       
+        source: repoHound.ttAdapter(),
         templates: {
           'suggestion': function (datum) {
             template = '<div class="repo-mini-listing">';
@@ -4014,23 +4014,23 @@ quayApp.directive('headerBar', function () {
     },
     controller: function($scope, $element, $location, UserService, PlanService, ApiService, NotificationService) {
       $scope.notificationService = NotificationService;
-     
+
       // Monitor any user changes and place the current user into the scope.
       UserService.updateUserIn($scope);
-     
+
       $scope.signout = function() {
         ApiService.logout().then(function() {
           UserService.load();
           $location.path('/');
         });
       };
-        
+
       $scope.appLinkTarget = function() {
         if ($("div[ng-view]").length === 0) {
             return "_self";
         }
         return "";
-      };     
+      };
     }
   };
   return directiveDefinitionObject;
@@ -4100,7 +4100,7 @@ quayApp.directive('entitySearch', function () {
         // Reset the cached teams and robots.
         $scope.teams = null;
         $scope.robots = null;
-        
+
         // Load the organization's teams (if applicable).
         if ($scope.isOrganization && isSupported('team')) {
           // Note: We load the org here again so that we always have the fully up-to-date
@@ -4128,7 +4128,7 @@ quayApp.directive('entitySearch', function () {
 
         bootbox.prompt('Enter the name of the new team', function(teamname) {
           if (!teamname) { return; }
-            
+
           var regex = new RegExp(TEAM_PATTERN);
           if (!regex.test(teamname)) {
             bootbox.alert('Invalid team name');
@@ -4138,7 +4138,7 @@ quayApp.directive('entitySearch', function () {
           createOrganizationTeam(ApiService, $scope.namespace, teamname, function(created) {
             $scope.setEntity(created.name, 'team', false);
             $scope.teams[teamname] = created;
-          });          
+          });
         });
       };
 
@@ -4157,7 +4157,7 @@ quayApp.directive('entitySearch', function () {
           createRobotAccount(ApiService, $scope.isOrganization, $scope.namespace, robotname, function(created) {
             $scope.setEntity(created.name, 'user', true);
             $scope.robots.push(created);
-          });          
+          });
         });
       };
 
@@ -4251,8 +4251,8 @@ quayApp.directive('entitySearch', function () {
               return datums;
             }
           },
-          datumTokenizer: function(d) { 
-            return Bloodhound.tokenizers.whitespace(d.val); 
+          datumTokenizer: function(d) {
+            return Bloodhound.tokenizers.whitespace(d.val);
           },
           queryTokenizer: Bloodhound.tokenizers.whitespace
         });
@@ -4293,7 +4293,7 @@ quayApp.directive('entitySearch', function () {
 
                 var class_string = '';
                 for (var i = 0; i < classes.length; ++i) {
-                  if (i > 0) {                   
+                  if (i > 0) {
                     if (i == classes.length - 1) {
                       class_string += ' or ';
                     } else {
@@ -4308,17 +4308,17 @@ quayApp.directive('entitySearch', function () {
               }
 
               return null;
-            }, 
+            },
             'suggestion': function (datum) {
               template = '<div class="entity-mini-listing">';
               if (datum.entity.kind == 'user' && !datum.entity.is_robot) {
                 template += '<i class="fa fa-user fa-lg"></i>';
               } else if (datum.entity.kind == 'user' && datum.entity.is_robot) {
-                template += '<i class="fa fa-wrench fa-lg"></i>';            
+                template += '<i class="fa fa-wrench fa-lg"></i>';
               } else if (datum.entity.kind == 'team') {
                 template += '<i class="fa fa-group fa-lg"></i>';
               } else if (datum.entity.kind == 'org') {
-                template += '<i class="fa"><img src="//www.gravatar.com/avatar/' + 
+                template += '<i class="fa"><img src="//www.gravatar.com/avatar/' +
                   datum.entity.gravatar + '?s=16&amp;d=identicon"></i>';
               }
 
@@ -4360,7 +4360,7 @@ quayApp.directive('entitySearch', function () {
         if (!input) { return; }
 
         $(input).typeahead('val', '');
-        $scope.clearEntityInternal();        
+        $scope.clearEntityInternal();
       });
 
       $scope.$watch('placeholder', function(title) {
@@ -4410,7 +4410,7 @@ quayApp.directive('roleGroup', function () {
     controller: function($scope, $element) {
       $scope.setRole = function(role) {
         if ($scope.currentRole == role) { return; }
-        if ($scope.roleChanged) {          
+        if ($scope.roleChanged) {
           $scope.roleChanged({'role': role});
         } else {
           $scope.currentRole = role;
@@ -4489,10 +4489,10 @@ quayApp.directive('billingOptions', function () {
           'diners club': 'diners',
           'discover': 'discover',
           'jcb': 'jcb',
-          'mastercard': 'mastercard',       
+          'mastercard': 'mastercard',
           'visa': 'visa'
         };
-        
+
         kind = supported[kind] || 'credit';
         return kind + '.png';
       };
@@ -4524,7 +4524,7 @@ quayApp.directive('billingOptions', function () {
           save();
         }
       };
-      
+
       $scope.$watch('invoice_email', checkSave);
       $scope.$watch('organization', update);
       $scope.$watch('user', update);
@@ -4533,7 +4533,7 @@ quayApp.directive('billingOptions', function () {
   return directiveDefinitionObject;
 });
 
-      
+
 quayApp.directive('planManager', function () {
   var directiveDefinitionObject = {
     priority: 0,
@@ -4549,7 +4549,7 @@ quayApp.directive('planManager', function () {
     },
     controller: function($scope, $element, PlanService, ApiService) {
       $scope.isExistingCustomer = false;
-        
+
       $scope.parseDate = function(timestamp) {
         return new Date(timestamp * 1000);
       };
@@ -4575,7 +4575,7 @@ quayApp.directive('planManager', function () {
           'opened': function() { $scope.planChanging = true; },
           'closed': function() { $scope.planChanging = false; },
           'success': subscribedToPlan,
-          'failure': function(resp) {            
+          'failure': function(resp) {
              $scope.planChanging = false;
           }
         };
@@ -4594,7 +4594,7 @@ quayApp.directive('planManager', function () {
         PlanService.getPlanIncludingDeprecated(sub.plan, function(subscribedPlan) {
           $scope.subscribedPlan = subscribedPlan;
           $scope.planUsagePercent = sub.usedPrivateRepos * 100 / $scope.subscribedPlan.privateRepos;
-          
+
           if ($scope.planChanged) {
             $scope.planChanged({ 'plan': subscribedPlan });
           }
@@ -4622,7 +4622,7 @@ quayApp.directive('planManager', function () {
         PlanService.verifyLoaded(function(plans) {
           $scope.plans = plans;
           update();
-            
+
           if ($scope.readyForPlan) {
             var planRequested = $scope.readyForPlan();
             if (planRequested && planRequested != PlanService.getFreePlan()) {
@@ -4688,7 +4688,7 @@ quayApp.directive('namespaceSelector', function () {
           namespaceObj = $scope.namespaces[$scope.user.username];
         }
 
-        var newNamespace = namespaceObj.name || namespaceObj.username;       
+        var newNamespace = namespaceObj.name || namespaceObj.username;
         $scope.namespaceObj = namespaceObj;
         $scope.namespace = newNamespace;
 
@@ -4778,7 +4778,7 @@ quayApp.directive('buildLogError', function () {
                 var repo = repo_and_tag.substring(0, tagIndex);
 
                 localInfo['repo_url'] = repo_url;
-                localInfo['repo'] = repo;                
+                localInfo['repo'] = repo;
 
                 localInfo['isLocal'] = repo_url.indexOf(Config.SERVER_HOSTNAME + '/') == 0;
               }
@@ -4957,7 +4957,7 @@ quayApp.directive('dropdownSelect', function ($compile) {
 
       // Setup lookahead.
       var input = $($element).find('.lookahead-input');
-          
+
       $scope.$watch('clearValue', function(cv) {
         if (cv) {
           $scope.selectedItem = null;
@@ -5001,7 +5001,7 @@ quayApp.directive('dropdownSelect', function ($compile) {
           datumTokenizer: function(d) {
             return Bloodhound.tokenizers.whitespace(d.val || d.value || '');
           },
-          queryTokenizer: Bloodhound.tokenizers.whitespace          
+          queryTokenizer: Bloodhound.tokenizers.whitespace
         });
         dropdownHound.initialize();
 
@@ -5044,7 +5044,7 @@ quayApp.directive('dropdownSelect', function ($compile) {
 
       var iconContainer = element.find('div.dropdown-select-icon-transclude');
       var menuContainer = element.find('div.dropdown-select-menu-transclude');
-      
+
       angular.forEach(transcludedElements, function(elem) {
         if (angular.element(elem).hasClass('dropdown-select-icon')) {
           iconContainer.append(elem);
@@ -5053,7 +5053,7 @@ quayApp.directive('dropdownSelect', function ($compile) {
         }
       });
 
-      transcludedBlock.remove();      
+      transcludedBlock.remove();
     }
   };
   return directiveDefinitionObject;
@@ -5217,7 +5217,7 @@ quayApp.directive('setupTriggerDialog', function () {
           $scope.currentView = 'analyzed';
           return;
         }
-        
+
         var params = {
           'repository': $scope.repository.namespace + '/' + $scope.repository.name,
           'trigger_uuid': $scope.trigger.id
@@ -5336,7 +5336,7 @@ quayApp.directive('triggerSetupGithub', function () {
         if ($scope.isMatching(kind, name, $scope.state.branchTagFilter)) {
           return;
         }
-        
+
         var newFilter = kind + '/' + name;
         var existing = $scope.state.branchTagFilter;
         if (existing) {
@@ -5380,7 +5380,7 @@ quayApp.directive('triggerSetupGithub', function () {
             if (ref.kind == 'branch') {
               $scope.branchNames.push(ref.name);
             } else {
-              $scope.tagNames.push(ref.name);              
+              $scope.tagNames.push(ref.name);
             }
           }
 
@@ -5414,9 +5414,9 @@ quayApp.directive('triggerSetupGithub', function () {
           }
 
           callback();
-        }, ApiService.errorDisplay('Cannot load locations'));        
+        }, ApiService.errorDisplay('Cannot load locations'));
       }
-           
+
       $scope.handleLocationInput = function(location) {
         $scope.state.isInvalidLocation = $scope.locations.indexOf(location) < 0;
         $scope.trigger['config']['subdir'] = location || '';
@@ -5433,7 +5433,7 @@ quayApp.directive('triggerSetupGithub', function () {
         $scope.trigger['config']['subdir'] = location || '';
         $scope.trigger.$ready = true;
       };
- 
+
       $scope.selectRepo = function(repo, org) {
         $scope.state.currentRepo = {
           'repo': repo,
@@ -5459,7 +5459,7 @@ quayApp.directive('triggerSetupGithub', function () {
         };
       };
 
-      var setupTypeahead = function() {        
+      var setupTypeahead = function() {
         var repos = [];
         for (var i = 0; i < $scope.orgs.length; ++i) {
           var org = $scope.orgs[i];
@@ -5487,7 +5487,7 @@ quayApp.directive('triggerSetupGithub', function () {
       };
 
       $scope.$watch('state.currentRepo', function(repo) {
-        if (repo) {          
+        if (repo) {
           $scope.selectRepoInternal(repo);
         }
       });
@@ -5523,7 +5523,7 @@ quayApp.directive('buildLogCommand', function () {
         if (colon <= 0) {
           return '';
         }
-        
+
         return $.trim(fullTitle.substring(colon + 1));
       };
     }
@@ -5547,7 +5547,7 @@ quayApp.directive('dockerfileCommand', function () {
         'quay.io': function(pieces) {
           var rnamespace =  pieces[pieces.length - 2];
           var rname = pieces[pieces.length - 1].split(':')[0];
-          return '/repository/' + rnamespace + '/' + rname + '/'; 
+          return '/repository/' + rnamespace + '/' + rname + '/';
         },
 
         '': function(pieces) {
@@ -5566,11 +5566,11 @@ quayApp.directive('dockerfileCommand', function () {
           if (!registryHandlers[registry]) {
             return title;
           }
-          
+
           return '<i class="fa fa-hdd-o"></i> <a href="' + registryHandlers[registry](pieces) + '" target="_blank">' + title + '</a>';
         }
       };
-     
+
       $scope.getCommandKind = function(title) {
         var space = title.indexOf(' ');
         return title.substring(0, space);
@@ -5581,17 +5581,17 @@ quayApp.directive('dockerfileCommand', function () {
         if (space <= 0) {
           return UtilService.textToSafeHtml(title);
         }
-        
+
         var kind = $scope.getCommandKind(title);
         var sanitized = UtilService.textToSafeHtml(title.substring(space + 1));
-        
+
         var handler = kindHandlers[kind || ''];
         if (handler) {
           return handler(sanitized);
         } else {
           return sanitized;
         }
-      };      
+      };
     }
   };
   return directiveDefinitionObject;
@@ -5671,7 +5671,7 @@ quayApp.directive('buildMessage', function () {
           case 'starting':
           case 'initializing':
             return 'Starting Dockerfile build';
-          
+
           case 'waiting':
             return 'Waiting for available build worker';
 
@@ -5680,24 +5680,24 @@ quayApp.directive('buildMessage', function () {
 
           case 'pulling':
             return 'Pulling base image';
-          
+
           case 'building':
             return 'Building image from Dockerfile';
 
           case 'priming-cache':
             return 'Priming cache for build';
-          
+
           case 'pushing':
             return 'Pushing image built from Dockerfile';
-            
+
           case 'complete':
             return 'Dockerfile build completed and pushed';
-          
+
           case 'error':
             return 'Dockerfile build failed';
 
           case 'internalerror':
-            return 'A system error occurred while building - Will be retried shortly';
+            return 'An internal system error occurred while building; the build will be retried in the next few minutes.';
         }
       };
     }
@@ -5747,7 +5747,7 @@ quayApp.directive('buildProgress', function () {
             return 0;
             break;
         }
-        
+
         return -1;
       };
     }
@@ -5796,7 +5796,7 @@ quayApp.directive('externalNotificationView', function () {
                 "className": "btn-primary"
               }
             }
-          });          
+          });
         });
       };
 
@@ -5835,7 +5835,7 @@ quayApp.directive('createExternalNotificationDialog', function () {
 
       $scope.events = ExternalNotificationData.getSupportedEvents();
       $scope.methods = ExternalNotificationData.getSupportedMethods();
-   
+
       $scope.setEvent = function(event) {
         $scope.currentEvent = event;
       };
@@ -5892,7 +5892,7 @@ quayApp.directive('createExternalNotificationDialog', function () {
       $scope.handleEmailCheck = function(isAuthorized) {
         if (isAuthorized) {
           $scope.performCreateNotification();
-          return; 
+          return;
         }
 
         if ($scope.status == 'authorizing-email-sent') {
@@ -5943,7 +5943,7 @@ quayApp.directive('createExternalNotificationDialog', function () {
           $scope.unauthorizedEmail = false;
           $('#createNotificationModal').modal({});
         }
-      });      
+      });
     }
   };
   return directiveDefinitionObject;
@@ -6074,7 +6074,7 @@ quayApp.directive('notificationView', function () {
           } else {
             var parts = url.split('?')
             $location.path(parts[0]);
-            
+
             if (parts.length > 1) {
               $location.search(parts[1]);
             }
@@ -6137,7 +6137,7 @@ quayApp.directive('dockerfileBuildDialog', function () {
         $scope.errorMessage = null;
         $scope.startCounter++;
       };
-      
+
       $scope.$watch('showNow', function(sn) {
         if (sn && $scope.repository) {
           $('#dockerfilebuildModal').modal({});
@@ -6295,11 +6295,11 @@ quayApp.directive('dockerfileBuildForm', function () {
         };
         request.send(file);
       };
-      
+
       var startFileUpload = function(repo) {
         $scope.uploading = true;
         $scope.uploading_progress = 0;
-        
+
         var uploader = $('#file-drop')[0];
         if (uploader.files.length == 0) {
           handleMissingFile();
@@ -6309,18 +6309,18 @@ quayApp.directive('dockerfileBuildForm', function () {
 
         var file = uploader.files[0];
         $scope.upload_file = file.name;
-        
+
         var mimeType = file.type || 'application/octet-stream';
         var data = {
           'mimeType': mimeType
         };
-      
+
         var getUploadUrl = ApiService.getFiledropUrl(data).then(function(resp) {
           conductUpload(file, resp.url, resp.file_id, mimeType);
         }, function() {
           handleUploadFailed('Could not retrieve upload URL');
         });
-      };      
+      };
 
       $scope.$watch('internal.hasDockerfile', function(d) {
         $scope.hasDockerfile = d;
@@ -6465,12 +6465,12 @@ quayApp.directive('tagSpecificImagesView', function () {
         if (image.ancestors.length > 1) {
           classes += 'child ';
         }
- 
+
         var currentTag = $scope.repository.tags[$scope.tag];
         if (image.id == currentTag.image_id) {
           classes += 'tag-image ';
         }
-        
+
         return classes;
       };
 
@@ -6489,7 +6489,7 @@ quayApp.directive('tagSpecificImagesView', function () {
         if (!tag_image) {
           return;
         }
-        
+
         callback(tag_image);
 
         var ancestors = tag_image.ancestors.split('/').reverse();
@@ -6499,7 +6499,7 @@ quayApp.directive('tagSpecificImagesView', function () {
             if (image == opt_cutoff) {
               return;
             }
-            
+
             callback(image);
           }
         }
@@ -6516,7 +6516,7 @@ quayApp.directive('tagSpecificImagesView', function () {
           $scope.tagSpecificImages = [];
           return;
         }
-        
+
         var getIdsForTag = function(currentTag) {
           var ids = {};
           forAllTagImages(currentTag, function(image) {
@@ -6544,7 +6544,7 @@ quayApp.directive('tagSpecificImagesView', function () {
             images.push(image);
           }
         }
-        
+
         images.sort(function(a, b) {
           var result = new Date(b.created) - new Date(a.created);
           if (result != 0) {
@@ -6614,7 +6614,7 @@ quayApp.directive('ngVisible', function () {
 quayApp.config( [
     '$compileProvider',
     function( $compileProvider )
-    {   
+    {
         $compileProvider.aHrefSanitizationWhitelist(/^\s*(https?|ftp|mailto|tel|irc):/);
     }
 ]);
@@ -6695,11 +6695,11 @@ quayApp.run(['$location', '$rootScope', 'Restangular', 'UserService', 'PlanServi
     if (!description) {
       description = 'Hosted private docker repositories. Includes full user management and history. Free for public repositories.';
     }
-    
+
     // Note: We set the content of the description tag manually here rather than using Angular binding
     // because we need the <meta> tag to have a default description that is not of the form "{{ description }}",
     // we read by tools that do not properly invoke the Angular code.
-    $('#descriptionTag').attr('content', description);    
+    $('#descriptionTag').attr('content', description);
   });
 
   $rootScope.$on('$routeUpdate', function(){
@@ -6752,8 +6752,8 @@ quayApp.run(['$location', '$rootScope', 'Restangular', 'UserService', 'PlanServi
 
         $location.search(newSearch);
       });
-      
-      e.preventDefault();        
+
+      e.preventDefault();
     });
 
     if (activeTab) {

From 660a640de6b666857e7f2e0c6dab6bb03d2d7fb1 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Tue, 25 Nov 2014 16:14:44 -0500
Subject: [PATCH 18/26] Better organize the source file structure of the build
 manager and change it to choose a lifecycle manager based on the config

---
 .../{enterprise_builder.py => builder.py}     | 25 +++++++++++++------
 buildman/component/__init__.py                |  0
 buildman/{ => component}/basecomponent.py     |  0
 buildman/{ => component}/buildcomponent.py    |  8 +++---
 buildman/jobutil/__init__.py                  |  0
 buildman/{ => jobutil}/buildjob.py            |  0
 buildman/{ => jobutil}/buildpack.py           |  0
 buildman/{ => jobutil}/buildstatus.py         |  0
 buildman/{ => jobutil}/workererror.py         |  0
 buildman/manager/enterprise.py                |  4 +--
 buildman/server.py                            |  9 ++++---
 config.py                                     |  8 +++---
 12 files changed, 34 insertions(+), 20 deletions(-)
 rename buildman/{enterprise_builder.py => builder.py} (51%)
 create mode 100644 buildman/component/__init__.py
 rename buildman/{ => component}/basecomponent.py (100%)
 rename buildman/{ => component}/buildcomponent.py (98%)
 create mode 100644 buildman/jobutil/__init__.py
 rename buildman/{ => jobutil}/buildjob.py (100%)
 rename buildman/{ => jobutil}/buildpack.py (100%)
 rename buildman/{ => jobutil}/buildstatus.py (100%)
 rename buildman/{ => jobutil}/workererror.py (100%)

diff --git a/buildman/enterprise_builder.py b/buildman/builder.py
similarity index 51%
rename from buildman/enterprise_builder.py
rename to buildman/builder.py
index 88ea62b2d..efd7b10cc 100644
--- a/buildman/enterprise_builder.py
+++ b/buildman/builder.py
@@ -1,4 +1,3 @@
-import argparse
 import logging
 
 from app import app, userfiles as user_files, build_logs, dockerfile_build_queue
@@ -8,13 +7,23 @@ from buildman.server import BuilderServer
 
 LOGGER = logging.getLogger(__name__)
 
-if __name__ == '__main__':
-  logging.basicConfig(level=logging.DEBUG)
+BUILD_MANAGERS = {
+  'enterprise': EnterpriseManager
+}
 
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--host', type=str, default='127.0.0.1', help='Host IP.')
-  args = parser.parse_args()
+def run_build_manager():
+  build_manager_config = app.config.get('BUILD_MANAGER')
+  if build_manager_config is None:
+    return
+
+  manager_klass = BUILD_MANAGERS.get(build_manager_config[0])
+  if manager_klass is None:
+    return
 
   server = BuilderServer(app.config['SERVER_HOSTNAME'], dockerfile_build_queue, build_logs,
-                         user_files, EnterpriseManager)
-  server.run(args.host)
+                         user_files, manager_klass)
+  server.run('0.0.0.0')
+
+if __name__ == '__main__':
+  logging.basicConfig(level=logging.DEBUG)
+  run_build_manager()
diff --git a/buildman/component/__init__.py b/buildman/component/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/buildman/basecomponent.py b/buildman/component/basecomponent.py
similarity index 100%
rename from buildman/basecomponent.py
rename to buildman/component/basecomponent.py
diff --git a/buildman/buildcomponent.py b/buildman/component/buildcomponent.py
similarity index 98%
rename from buildman/buildcomponent.py
rename to buildman/component/buildcomponent.py
index 9b29ad5b6..12e6b280b 100644
--- a/buildman/buildcomponent.py
+++ b/buildman/component/buildcomponent.py
@@ -8,11 +8,11 @@ import re
 from autobahn.wamp.exception import ApplicationError
 from trollius.coroutines import From
 
-from buildman.basecomponent import BaseComponent
-from buildman.buildpack import BuildPackage, BuildPackageException
-from buildman.buildstatus import StatusHandler
 from buildman.server import BuildJobResult
-from buildman.workererror import WorkerError
+from buildman.component.basecomponent import BaseComponent
+from buildman.jobutil.buildpack import BuildPackage, BuildPackageException
+from buildman.jobutil.buildstatus import StatusHandler
+from buildman.jobutil.workererror import WorkerError
 
 from data.database import BUILD_PHASE
 
diff --git a/buildman/jobutil/__init__.py b/buildman/jobutil/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/buildman/buildjob.py b/buildman/jobutil/buildjob.py
similarity index 100%
rename from buildman/buildjob.py
rename to buildman/jobutil/buildjob.py
diff --git a/buildman/buildpack.py b/buildman/jobutil/buildpack.py
similarity index 100%
rename from buildman/buildpack.py
rename to buildman/jobutil/buildpack.py
diff --git a/buildman/buildstatus.py b/buildman/jobutil/buildstatus.py
similarity index 100%
rename from buildman/buildstatus.py
rename to buildman/jobutil/buildstatus.py
diff --git a/buildman/workererror.py b/buildman/jobutil/workererror.py
similarity index 100%
rename from buildman/workererror.py
rename to buildman/jobutil/workererror.py
diff --git a/buildman/manager/enterprise.py b/buildman/manager/enterprise.py
index 9abe5f3c0..741781078 100644
--- a/buildman/manager/enterprise.py
+++ b/buildman/manager/enterprise.py
@@ -1,9 +1,9 @@
 import logging
 import uuid
 
+from buildman.component.basecomponent import BaseComponent
+from buildman.component.buildcomponent import BuildComponent
 from buildman.manager.basemanager import BaseManager
-from buildman.basecomponent import BaseComponent
-from buildman.buildcomponent import BuildComponent
 
 from trollius.coroutines import From
 
diff --git a/buildman/server.py b/buildman/server.py
index fd5d4f409..f83d71a81 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -11,7 +11,7 @@ from threading import Event
 from trollius.coroutines import From
 from datetime import datetime, timedelta
 
-from buildman.buildjob import BuildJob, BuildJobLoadException
+from buildman.jobutil.buildjob import BuildJob, BuildJobLoadException
 from data.queue import WorkQueue
 
 LOGGER = logging.getLogger(__name__)
@@ -21,6 +21,9 @@ TIMEOUT_PERIOD_MINUTES = 20
 JOB_TIMEOUT_SECONDS = 300
 MINIMUM_JOB_EXTENSION = timedelta(minutes=2)
 
+WEBSOCKET_PORT = 8080
+CONTROLLER_PORT = 8181
+
 class BuildJobResult(object):
   """ Build job result enum """
   INCOMPLETE = 'incomplete'
@@ -166,8 +169,8 @@ class BuilderServer(object):
     transport_factory.setProtocolOptions(failByDrop=True)
 
     # Initialize the controller server and the WAMP server
-    create_wsgi_server(self._controller_app, loop=loop, host=host, port=8181)
-    yield From(loop.create_server(transport_factory, host, 8080))
+    create_wsgi_server(self._controller_app, loop=loop, host=host, port=CONTROLLER_PORT)
+    yield From(loop.create_server(transport_factory, host, WEBSOCKET_PORT))
 
     # Initialize the work queue checker.
     yield From(self._work_checker())
diff --git a/config.py b/config.py
index ddb6b54f7..b7e1ca531 100644
--- a/config.py
+++ b/config.py
@@ -49,9 +49,9 @@ class DefaultConfig(object):
   REGISTRY_TITLE = 'Quay.io'
   REGISTRY_TITLE_SHORT = 'Quay.io'
   CONTACT_INFO = [
-    'mailto:support@quay.io', 
-    'irc://chat.freenode.net:6665/quayio', 
-    'tel:+1-888-930-3475', 
+    'mailto:support@quay.io',
+    'irc://chat.freenode.net:6665/quayio',
+    'tel:+1-888-930-3475',
     'https://twitter.com/quayio',
   ]
 
@@ -160,6 +160,8 @@ class DefaultConfig(object):
   # Feature Flag: Whether users can be created (by non-super users).
   FEATURE_USER_CREATION = True
 
+  BUILD_MANAGER = ('enterprise', {})
+
   DISTRIBUTED_STORAGE_CONFIG = {
     'local_eu': ['LocalStorage', {'storage_path': 'test/data/registry/eu'}],
     'local_us': ['LocalStorage', {'storage_path': 'test/data/registry/us'}],

From 04fc6d82a561a5eab3138c0b0546f4bed0e27e7e Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Tue, 25 Nov 2014 16:36:21 -0500
Subject: [PATCH 19/26] Add support for SSL if the certificate is found in the
 config directory

---
 buildman/builder.py | 13 ++++++++++++-
 buildman/server.py  | 10 +++++-----
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/buildman/builder.py b/buildman/builder.py
index efd7b10cc..4411b54ff 100644
--- a/buildman/builder.py
+++ b/buildman/builder.py
@@ -1,10 +1,13 @@
 import logging
+import os
 
 from app import app, userfiles as user_files, build_logs, dockerfile_build_queue
 
 from buildman.manager.enterprise import EnterpriseManager
 from buildman.server import BuilderServer
 
+from trollius import SSLContext
+
 LOGGER = logging.getLogger(__name__)
 
 BUILD_MANAGERS = {
@@ -16,13 +19,21 @@ def run_build_manager():
   if build_manager_config is None:
     return
 
+  LOGGER.debug('Asking to start build manager with lifecycle "%s"', build_manager_config[0])
   manager_klass = BUILD_MANAGERS.get(build_manager_config[0])
   if manager_klass is None:
     return
 
+  LOGGER.debug('Starting build manager with lifecycle "%s"', build_manager_config[0])
+  ssl_context = None
+  if os.path.exists('conf/stack/ssl.cert'):
+    LOGGER.debug('Loading SSL cert and key')
+    ssl_context = SSLContext()
+    ssl_context.load_cert_chain('conf/stack/ssl.cert', 'conf/stack/ssl.key')
+
   server = BuilderServer(app.config['SERVER_HOSTNAME'], dockerfile_build_queue, build_logs,
                          user_files, manager_klass)
-  server.run('0.0.0.0')
+  server.run('0.0.0.0', ssl=ssl_context)
 
 if __name__ == '__main__':
   logging.basicConfig(level=logging.DEBUG)
diff --git a/buildman/server.py b/buildman/server.py
index f83d71a81..876d77786 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -67,13 +67,13 @@ class BuilderServer(object):
 
     self._controller_app = controller_app
 
-  def run(self, host):
+  def run(self, host, ssl=None):
     LOGGER.debug('Initializing the lifecycle manager')
     self._lifecycle_manager.initialize()
 
     LOGGER.debug('Initializing all members of the event loop')
     loop = trollius.get_event_loop()
-    trollius.Task(self._initialize(loop, host))
+    trollius.Task(self._initialize(loop, host, ssl))
 
     LOGGER.debug('Starting server on port 8080, with controller on port 8181')
     try:
@@ -161,7 +161,7 @@ class BuilderServer(object):
 
 
   @trollius.coroutine
-  def _initialize(self, loop, host):
+  def _initialize(self, loop, host, ssl=None):
     self._loop = loop
 
     # Create the WAMP server.
@@ -169,8 +169,8 @@ class BuilderServer(object):
     transport_factory.setProtocolOptions(failByDrop=True)
 
     # Initialize the controller server and the WAMP server
-    create_wsgi_server(self._controller_app, loop=loop, host=host, port=CONTROLLER_PORT)
-    yield From(loop.create_server(transport_factory, host, WEBSOCKET_PORT))
+    create_wsgi_server(self._controller_app, loop=loop, host=host, port=CONTROLLER_PORT, ssl=ssl)
+    yield From(loop.create_server(transport_factory, host, WEBSOCKET_PORT, ssl=ssl))
 
     # Initialize the work queue checker.
     yield From(self._work_checker())

From 9d675b51ed1b918531b4d2bdeddfe76b65fff329 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Tue, 25 Nov 2014 18:08:18 -0500
Subject: [PATCH 20/26] - Change SSL to only be enabled via an environment
 variable. Nginx will be terminating SSL for the ER. - Add the missing
 dependencies to the requirements.txt - Change the builder ports to
 non-standard locations - Add the /b1/socket and /b1/controller endpoints in
 nginx, to map to the build manager - Have the build manager start
 automatically.

---
 Dockerfile.web                 |  1 +
 buildman/builder.py            |  5 +++--
 buildman/server.py             |  7 ++++---
 conf/http-base.conf            |  8 ++++++++
 conf/init/buildmanager/log/run |  2 ++
 conf/init/buildmanager/run     |  8 ++++++++
 conf/server-base.conf          | 14 +++++++++++++-
 requirements-nover.txt         |  3 +++
 requirements.txt               |  3 +++
 9 files changed, 45 insertions(+), 6 deletions(-)
 create mode 100755 conf/init/buildmanager/log/run
 create mode 100755 conf/init/buildmanager/run

diff --git a/Dockerfile.web b/Dockerfile.web
index a5ea9d487..62f2419e9 100644
--- a/Dockerfile.web
+++ b/Dockerfile.web
@@ -48,6 +48,7 @@ ADD conf/init/nginx /etc/service/nginx
 ADD conf/init/diffsworker /etc/service/diffsworker
 ADD conf/init/notificationworker /etc/service/notificationworker
 ADD conf/init/buildlogsarchiver /etc/service/buildlogsarchiver
+ADD conf/init/buildmanager /etc/service/buildmanager
 
 # Download any external libs.
 RUN mkdir static/fonts static/ldn
diff --git a/buildman/builder.py b/buildman/builder.py
index 4411b54ff..8734de99e 100644
--- a/buildman/builder.py
+++ b/buildman/builder.py
@@ -26,10 +26,11 @@ def run_build_manager():
 
   LOGGER.debug('Starting build manager with lifecycle "%s"', build_manager_config[0])
   ssl_context = None
-  if os.path.exists('conf/stack/ssl.cert'):
+  if os.environ.get('SSL_CONFIG'):
     LOGGER.debug('Loading SSL cert and key')
     ssl_context = SSLContext()
-    ssl_context.load_cert_chain('conf/stack/ssl.cert', 'conf/stack/ssl.key')
+    ssl_context.load_cert_chain(os.environ.get('SSL_CONFIG') + '/ssl.cert',
+                                os.environ.get('SSL_CONFIG') + '/ssl.key')
 
   server = BuilderServer(app.config['SERVER_HOSTNAME'], dockerfile_build_queue, build_logs,
                          user_files, manager_klass)
diff --git a/buildman/server.py b/buildman/server.py
index 876d77786..fccaeacfb 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -21,8 +21,8 @@ TIMEOUT_PERIOD_MINUTES = 20
 JOB_TIMEOUT_SECONDS = 300
 MINIMUM_JOB_EXTENSION = timedelta(minutes=2)
 
-WEBSOCKET_PORT = 8080
-CONTROLLER_PORT = 8181
+WEBSOCKET_PORT = 8787
+CONTROLLER_PORT = 8686
 
 class BuildJobResult(object):
   """ Build job result enum """
@@ -75,7 +75,8 @@ class BuilderServer(object):
     loop = trollius.get_event_loop()
     trollius.Task(self._initialize(loop, host, ssl))
 
-    LOGGER.debug('Starting server on port 8080, with controller on port 8181')
+    LOGGER.debug('Starting server on port %s, with controller on port %s', WEBSOCKET_PORT,
+                 CONTROLLER_PORT)
     try:
       loop.run_forever()
     except KeyboardInterrupt:
diff --git a/conf/http-base.conf b/conf/http-base.conf
index ad3d9f178..1eb0b6170 100644
--- a/conf/http-base.conf
+++ b/conf/http-base.conf
@@ -23,3 +23,11 @@ upstream verbs_app_server {
 upstream registry_app_server {
     server unix:/tmp/gunicorn_registry.sock fail_timeout=0;
 }
+
+upstream build_manager_controller_server {
+    server localhost:8686;
+}
+
+upstream build_manager_websocket_server {
+    server localhost:8787;
+}
\ No newline at end of file
diff --git a/conf/init/buildmanager/log/run b/conf/init/buildmanager/log/run
new file mode 100755
index 000000000..1dd4c3fef
--- /dev/null
+++ b/conf/init/buildmanager/log/run
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec svlogd /var/log/buildmanager/
\ No newline at end of file
diff --git a/conf/init/buildmanager/run b/conf/init/buildmanager/run
new file mode 100755
index 000000000..d0bc6564f
--- /dev/null
+++ b/conf/init/buildmanager/run
@@ -0,0 +1,8 @@
+#! /bin/bash
+
+echo 'Starting internal build manager'
+
+cd /
+venv/bin/python -m buildman.builder 2>&1
+
+echo 'Internal build manager exited'
\ No newline at end of file
diff --git a/conf/server-base.conf b/conf/server-base.conf
index 75c6594bb..419c08b44 100644
--- a/conf/server-base.conf
+++ b/conf/server-base.conf
@@ -24,7 +24,7 @@ location / {
 location /realtime {
     proxy_pass   http://web_app_server;
     proxy_buffering off;
-    proxy_request_buffering off;    
+    proxy_request_buffering off;
 }
 
 location /v1/ {
@@ -59,4 +59,16 @@ location /v1/_ping {
     add_header X-Docker-Registry-Version 0.6.0;
     add_header X-Docker-Registry-Standalone 0;
     return 200 'true';
+}
+
+location ~ ^/b1/controller(/?)(.*) {
+    proxy_pass http://build_manager_controller_server/$2;
+    proxy_read_timeout 2000;
+}
+
+location ~ ^/b1/socket(/?)(.*) {
+    proxy_pass http://build_manager_websocket_server/$2;
+    proxy_http_version 1.1;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "upgrade";
 }
\ No newline at end of file
diff --git a/requirements-nover.txt b/requirements-nover.txt
index e5c81cfd3..1289fedf9 100644
--- a/requirements-nover.txt
+++ b/requirements-nover.txt
@@ -1,3 +1,6 @@
+autobahn
+aiowsgi
+trollius
 peewee
 flask
 py-bcrypt
diff --git a/requirements.txt b/requirements.txt
index be726ece7..6b2efb90f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,7 +17,9 @@ SQLAlchemy==0.9.7
 Werkzeug==0.9.6
 git+https://github.com/DevTable/aniso8601-fake.git
 git+https://github.com/DevTable/anunidecode.git
+aiowsgi==0.3
 alembic==0.6.7
+autobahn==0.9.3-3
 backports.ssl-match-hostname==3.4.0.2
 beautifulsoup4==4.3.2
 blinker==1.3
@@ -51,6 +53,7 @@ reportlab==2.7
 requests==2.4.3
 six==1.8.0
 stripe==1.19.0
+trollius==1.0.3
 tzlocal==1.1.1
 websocket-client==0.18.0
 wsgiref==0.1.2

From d91829dc3c728cf7b87a6cff1a2212ca94c89d41 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Wed, 26 Nov 2014 11:28:29 -0500
Subject: [PATCH 21/26] Only start the build manager if building is enabled

---
 buildman/builder.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/buildman/builder.py b/buildman/builder.py
index 8734de99e..86097c8a4 100644
--- a/buildman/builder.py
+++ b/buildman/builder.py
@@ -1,5 +1,6 @@
 import logging
 import os
+import features
 
 from app import app, userfiles as user_files, build_logs, dockerfile_build_queue
 
@@ -15,6 +16,10 @@ BUILD_MANAGERS = {
 }
 
 def run_build_manager():
+  if not features.BUILD_SUPPORT:
+    LOGGER.debug('Building is disabled. Please enable the feature flag')
+    return
+
   build_manager_config = app.config.get('BUILD_MANAGER')
   if build_manager_config is None:
     return

From b7a489813a8375a61165a20faeb5873907bbbc5f Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Wed, 26 Nov 2014 12:37:20 -0500
Subject: [PATCH 22/26] Fix build system to work with Github Enterprise

---
 endpoints/trigger.py | 14 ++++++++------
 static/js/app.js     |  2 +-
 util/oauth.py        |  3 +++
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/endpoints/trigger.py b/endpoints/trigger.py
index ec4434858..3a9813428 100644
--- a/endpoints/trigger.py
+++ b/endpoints/trigger.py
@@ -100,7 +100,7 @@ class BuildTrigger(object):
     raise NotImplementedError
 
   def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
-    """ 
+    """
     Activates the trigger for the service, with the given new configuration.
     Returns new configuration that should be stored if successful.
     """
@@ -150,8 +150,10 @@ def raise_unsupported():
 class GithubBuildTrigger(BuildTrigger):
   @staticmethod
   def _get_client(auth_token):
-    return Github(auth_token, client_id=github_trigger.client_id(),
-                              client_secret=github_trigger.client_secret())
+    return Github(auth_token,
+                  base_url=github_trigger.api_endpoint(),
+                  client_id=github_trigger.client_id(),
+                  client_secret=github_trigger.client_secret())
 
   @classmethod
   def service_name(cls):
@@ -246,7 +248,7 @@ class GithubBuildTrigger(BuildTrigger):
     gh_client = self._get_client(auth_token)
     source = config['build_source']
 
-    try:      
+    try:
       repo = gh_client.get_repo(source)
 
       # Find the first matching branch.
@@ -381,7 +383,7 @@ class GithubBuildTrigger(BuildTrigger):
         raise SkipRequestException()
 
     if should_skip_commit(commit_message):
-      raise SkipRequestException() 
+      raise SkipRequestException()
 
     short_sha = GithubBuildTrigger.get_display_name(commit_sha)
 
@@ -419,7 +421,7 @@ class GithubBuildTrigger(BuildTrigger):
       branches = self.list_field_values(auth_token, config, 'branch_name')
       tags = self.list_field_values(auth_token, config, 'tag_name')
 
-      return ([{'kind': 'branch', 'name': b} for b in branches] + 
+      return ([{'kind': 'branch', 'name': b} for b in branches] +
               [{'kind': 'tag', 'name': tag} for tag in tags])
 
     if field_name == 'tag_name':
diff --git a/static/js/app.js b/static/js/app.js
index 2ffde83b5..3d1e86727 100644
--- a/static/js/app.js
+++ b/static/js/app.js
@@ -1729,7 +1729,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
 
       keyService['githubEndpoint'] = oauth['GITHUB_LOGIN_CONFIG']['GITHUB_ENDPOINT'];
 
-      keyService['githubTriggerAuthorizeUrl'] = oauth['GITHUB_LOGIN_CONFIG']['AUTHORIZE_ENDPOINT'];
+      keyService['githubTriggerAuthorizeUrl'] = oauth['GITHUB_TRIGGER_CONFIG']['AUTHORIZE_ENDPOINT'];
 
       keyService['githubLoginScope'] = 'user:email';
       keyService['googleLoginScope'] = 'openid email';
diff --git a/util/oauth.py b/util/oauth.py
index 5349e5435..e0d38d395 100644
--- a/util/oauth.py
+++ b/util/oauth.py
@@ -52,6 +52,9 @@ class GithubOAuthConfig(OAuthConfig):
   def _api_endpoint(self):
     return self.config.get('API_ENDPOINT', self._get_url(self._endpoint(), '/api/v3/'))
 
+  def api_endpoint(self):
+    return self._api_endpoint()[0:-1]
+
   def user_endpoint(self):
     api_endpoint = self._api_endpoint()
     return self._get_url(api_endpoint, 'user')

From 52b7896835209c922b0217db5780e427db2fd23d Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Wed, 26 Nov 2014 12:54:02 -0500
Subject: [PATCH 23/26] Further fixes to make everything work nicely with
 Github Enterprise

---
 static/directives/trigger-description.html |  5 +-
 static/js/app.js                           | 16 +++-
 static/js/controllers.js                   | 99 +++++++++++-----------
 static/partials/repo-admin.html            | 33 ++++----
 4 files changed, 84 insertions(+), 69 deletions(-)

diff --git a/static/directives/trigger-description.html b/static/directives/trigger-description.html
index 91000dd1e..b39771a95 100644
--- a/static/directives/trigger-description.html
+++ b/static/directives/trigger-description.html
@@ -1,7 +1,10 @@
 <span class="trigger-description-element" ng-switch on="trigger.service">
   <span ng-switch-when="github">
     <i class="fa fa-github fa-lg" style="margin-right: 6px" data-title="GitHub" bs-tooltip="tooltip.title"></i>
-    Push to GitHub repository <a href="https://github.com/{{ trigger.config.build_source }}" target="_new">{{ trigger.config.build_source }}</a>
+    Push to GitHub <span ng-if="KeyService.isEnterprise('github-trigger')">Enterprise</span> repository
+    <a href="{{ KeyService['githubTriggerEndpoint'] }}{{ trigger.config.build_source }}" target="_new">
+      {{ trigger.config.build_source }}
+    </a>
     <div style="margin-top: 4px; margin-left: 26px; font-size: 12px; color: gray;" ng-if="!short">
       <div>
         <span class="trigger-description-subtitle">Branches/Tags:</span>
diff --git a/static/js/app.js b/static/js/app.js
index 3d1e86727..f422dc1f4 100644
--- a/static/js/app.js
+++ b/static/js/app.js
@@ -1729,14 +1729,22 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
 
       keyService['githubEndpoint'] = oauth['GITHUB_LOGIN_CONFIG']['GITHUB_ENDPOINT'];
 
+      keyService['githubTriggerEndpoint'] = oauth['GITHUB_TRIGGER_CONFIG']['GITHUB_ENDPOINT'];
       keyService['githubTriggerAuthorizeUrl'] = oauth['GITHUB_TRIGGER_CONFIG']['AUTHORIZE_ENDPOINT'];
 
       keyService['githubLoginScope'] = 'user:email';
       keyService['googleLoginScope'] = 'openid email';
 
       keyService.isEnterprise = function(service) {
-        var isGithubEnterprise = keyService['githubLoginUrl'].indexOf('https://github.com/') < 0;
-        return service == 'github' && isGithubEnterprise;
+        switch (service) {
+          case 'github':
+            return keyService['githubLoginUrl'].indexOf('https://github.com/') < 0;
+
+          case 'github-trigger':
+            return keyService['githubTriggerAuthorizeUrl'].indexOf('https://github.com/') < 0;
+        }
+
+        return false;
       };
 
       keyService.getExternalLoginUrl = function(service, action) {
@@ -4806,7 +4814,9 @@ quayApp.directive('triggerDescription', function () {
       'trigger': '=trigger',
       'short': '=short'
     },
-    controller: function($scope, $element) {
+    controller: function($scope, $element, KeyService, TriggerService) {
+      $scope.KeyService = KeyService;
+      $scope.TriggerService = TriggerService;
     }
   };
   return directiveDefinitionObject;
diff --git a/static/js/controllers.js b/static/js/controllers.js
index 6a4051e50..4f7515ff6 100644
--- a/static/js/controllers.js
+++ b/static/js/controllers.js
@@ -59,13 +59,13 @@ function PlansCtrl($scope, $location, UserService, PlanService, $routeParams) {
 
   $scope.signedIn = function() {
     $('#signinModal').modal('hide');
-    PlanService.handleNotedPlan();    
+    PlanService.handleNotedPlan();
   };
-    
+
   $scope.buyNow = function(plan) {
     PlanService.notePlan(plan);
     if ($scope.user && !$scope.user.anonymous) {
-      PlanService.handleNotedPlan();    
+      PlanService.handleNotedPlan();
     } else {
       $('#signinModal').modal({});
     }
@@ -77,7 +77,7 @@ function PlansCtrl($scope, $location, UserService, PlanService, $routeParams) {
 
     if ($scope && $routeParams['trial-plan']) {
       $scope.buyNow($routeParams['trial-plan']);
-    }    
+    }
   }, /* include the personal plan */ true);
 }
 
@@ -94,7 +94,7 @@ function TutorialCtrl($scope, AngularTour, AngularTourSignals, UserService, Conf
     'steps': [
       {
         'title': 'Welcome to the ' + Config.REGISTRY_TITLE_SHORT  + ' tutorial!',
-        'templateUrl': '/static/tutorial/welcome.html'        
+        'templateUrl': '/static/tutorial/welcome.html'
       },
       {
         'title': 'Sign in to get started',
@@ -235,7 +235,7 @@ function RepoListCtrl($scope, $sanitize, Restangular, UserService, ApiService) {
   $scope.namespace = null;
   $scope.page = 1;
   $scope.publicPageCount = null;
- 
+
   // Monitor changes in the user.
   UserService.updateUserIn($scope, function() {
     loadMyRepos($scope.namespace);
@@ -269,7 +269,7 @@ function RepoListCtrl($scope, $sanitize, Restangular, UserService, ApiService) {
     }
 
     var options = {'public': false, 'sort': true, 'namespace': namespace};
-    
+
     $scope.user_repositories = ApiService.listReposAsResource().withOptions(options).get(function(resp) {
       return resp.repositories;
     });
@@ -318,7 +318,7 @@ function LandingCtrl($scope, UserService, ApiService, Features, Config) {
     if (namespace == $scope.user.username) {
       return true;
     }
-    
+
     if ($scope.user.organizations) {
       for (var i = 0; i < $scope.user.organizations.length; ++i) {
         var org = $scope.user.organizations[i];
@@ -483,7 +483,7 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
 
     $scope.currentPullCommand = $scope.pullCommands[0];
   };
-  
+
   $scope.showNewBuildDialog = function() {
     $scope.buildDialogShowCounter++;
   };
@@ -655,7 +655,7 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
 
   $scope.setImage = function(imageId, opt_updateURL) {
     if (!$scope.images) { return; }
-    
+
     var image = null;
     for (var i = 0; i < $scope.images.length; ++i) {
       var currentImage = $scope.images[i];
@@ -691,7 +691,7 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
       // We must find a good default.
       for (tagName in repo.tags) {
         if (!proposedTag || tagName == 'latest') {
-          proposedTag = repo.tags[tagName];          
+          proposedTag = repo.tags[tagName];
         }
       }
     }
@@ -702,7 +702,7 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
 
       if ($scope.tree) {
         $scope.tree.setTag(proposedTag.name);
-      } 
+      }
 
       if (opt_updateURL) {
         $location.search('image', null);
@@ -871,7 +871,7 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
   };
 
   var listImages = function() {
-    var params = {'repository': namespace + '/' + name};    
+    var params = {'repository': namespace + '/' + name};
     $scope.imageHistory = ApiService.listRepositoryImagesAsResource(params).get(function(resp) {
       $scope.images = resp.images;
       $scope.specificImages = [];
@@ -971,7 +971,7 @@ function BuildPackageCtrl($scope, Restangular, ApiService, DataFileService, $rou
     return dockerfilePath;
   };
 
-  var processBuildPack = function(uint8array) {    
+  var processBuildPack = function(uint8array) {
     var archiveread = function(files) {
       var getpath = function(file) {
         return file.path;
@@ -1056,7 +1056,7 @@ function BuildPackageCtrl($scope, Restangular, ApiService, DataFileService, $rou
         $scope.accessDenied = true;
         return;
       }
-      
+
       $rootScope.title = 'Repository Build Pack - ' + resp['display_name'];
       $scope.repobuild = resp;
       $scope.repo = {
@@ -1115,7 +1115,7 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope
     $('#confirmRestartBuildModal').modal('hide');
 
     var subdirectory = '';
-    if (build['job_config']) { 
+    if (build['job_config']) {
       subdirectory = build['job_config']['build_subdir'] || '';
     }
 
@@ -1159,7 +1159,7 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope
 
   $scope.processANSI = function(message, container) {
     var filter = container.logs._filter = (container.logs._filter || ansi2html.create());
-    
+
     // Note: order is important here.
     var setup = filter.getSetupHtml();
     var stream = filter.addInputToStream(message);
@@ -1194,7 +1194,7 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope
     if ($scope.pollChannel) {
       $scope.pollChannel.stop();
     }
-    
+
     // Create a new channel for polling the build status and logs.
     var conductStatusAndLogRequest = function(callback) {
       getBuildStatusAndLogs(build, callback);
@@ -1230,7 +1230,7 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope
     return endIndex;
   };
 
-  var getBuildStatusAndLogs = function(build, callback) {    
+  var getBuildStatusAndLogs = function(build, callback) {
     var params = {
       'repository': namespace + '/' + name,
       'build_uuid': build.id
@@ -1258,9 +1258,9 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope
         'start': $scope.logStartIndex
       };
 
-      ApiService.getRepoBuildLogsAsResource(params, true).withOptions(options).get(function(resp) {        
+      ApiService.getRepoBuildLogsAsResource(params, true).withOptions(options).get(function(resp) {
         if (build != $scope.currentBuild) { callback(false); return; }
-        
+
         // Process the logs we've received.
         $scope.logStartIndex = processLogs(resp['logs'], resp['start'], resp['total']);
 
@@ -1323,7 +1323,7 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope
   fetchRepository();
 }
 
-function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, TriggerService, $routeParams, 
+function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, TriggerService, $routeParams,
   $rootScope, $location, UserService, Config, Features, ExternalNotificationData) {
 
   var namespace = $routeParams.namespace;
@@ -1331,11 +1331,12 @@ function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, TriggerServi
 
   $scope.Features = Features;
   $scope.TriggerService = TriggerService;
+  $scope.KeyService = KeyService;
 
   $scope.permissions = {'team': [], 'user': [], 'loading': 2};
   $scope.logsShown = 0;
   $scope.deleting = false;
-  
+
   $scope.permissionCache = {};
   $scope.showTriggerSetupCounter = 0;
 
@@ -1436,7 +1437,7 @@ function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, TriggerServi
     var permission = $scope.permissions[kind][entityName];
     var currentRole = permission.role;
     permission.role = role;
-   
+
     var permissionPut = Restangular.one(getRestUrl('repository', namespace, name, 'permissions', kind, entityName));
     permissionPut.customPUT(permission).then(function() {}, function(resp) {
       $scope.permissions[kind][entityName] = {'role': currentRole};
@@ -1534,7 +1535,7 @@ function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, TriggerServi
     $scope.deleting = true;
     ApiService.deleteRepository(null, params).then(function() {
       $scope.repo = null;
-      
+
       setTimeout(function() {
         document.location = '/repository/';
       }, 1000);
@@ -1545,7 +1546,7 @@ function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, TriggerServi
   };
 
   $scope.showNewNotificationCounter = 0;
-  
+
   $scope.showNewNotificationDialog = function() {
     $scope.showNewNotificationCounter++;
   };
@@ -1629,7 +1630,7 @@ function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, TriggerServi
   };
 
   $scope.showManualBuildDialog = 0;
-  
+
   $scope.startTrigger = function(trigger, opt_custom) {
     var parameters = TriggerService.getRunParameters(trigger.service);
     if (parameters.length && !opt_custom) {
@@ -1765,7 +1766,7 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use
 
   $scope.logsShown = 0;
   $scope.invoicesShown = 0;
-  
+
   $scope.loadAuthedApps = function() {
     if ($scope.authorizedApps) { return; }
 
@@ -1808,7 +1809,7 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use
         $scope.orgPlans = plans;
       });
     }
-      
+
     $scope.convertStep = 1;
   };
 
@@ -1883,7 +1884,7 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use
       UserService.load();
     }, function(result) {
       $scope.updatingUser = false;
-      UIService.showFormError('#changePasswordForm', result);      
+      UIService.showFormError('#changePasswordForm', result);
     });
   };
 
@@ -1900,7 +1901,7 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use
   };
 }
 
-function ImageViewCtrl($scope, $routeParams, $rootScope, $timeout, ApiService, ImageMetadataService) {  
+function ImageViewCtrl($scope, $routeParams, $rootScope, $timeout, ApiService, ImageMetadataService) {
   var namespace = $routeParams.namespace;
   var name = $routeParams.name;
   var imageid = $routeParams.image;
@@ -1924,7 +1925,7 @@ function ImageViewCtrl($scope, $routeParams, $rootScope, $timeout, ApiService, I
     if (index < 0) {
       return '';
     }
-    
+
     return filepath.substr(0, index).split('/');
   };
 
@@ -1947,10 +1948,10 @@ function ImageViewCtrl($scope, $routeParams, $rootScope, $timeout, ApiService, I
     $scope.search['$'] = filter;
     document.getElementById('change-filter').value = filter;
   };
-  
+
   $scope.initializeTree = function() {
     if ($scope.tree) { return; }
-    
+
     $scope.tree = new ImageFileChangeTree($scope.image, $scope.combinedChanges);
     $timeout(function() {
       $scope.tree.draw('changes-tree-container');
@@ -1970,7 +1971,7 @@ function ImageViewCtrl($scope, $routeParams, $rootScope, $timeout, ApiService, I
   var fetchImage = function() {
     var params = {
       'repository': namespace + '/' + name,
-      'image_id': imageid  
+      'image_id': imageid
     };
 
     $scope.image = ApiService.getImageAsResource(params).get(function(image) {
@@ -1995,14 +1996,14 @@ function ImageViewCtrl($scope, $routeParams, $rootScope, $timeout, ApiService, I
   var fetchChanges = function() {
     var params = {
       'repository': namespace + '/' + name,
-      'image_id': imageid  
+      'image_id': imageid
     };
 
     ApiService.getImageChanges(null, params).then(function(changes) {
       var combinedChanges = [];
       var addCombinedChanges = function(c, kind) {
         for (var i = 0;  i < c.length; ++i) {
-          combinedChanges.push({ 
+          combinedChanges.push({
             'kind': kind,
             'file': c[i]
           });
@@ -2046,7 +2047,7 @@ function NewRepoCtrl($scope, $location, $http, $timeout, UserService, ApiService
   $scope.$watch('repo.namespace', function(namespace) {
     // Note: Can initially be undefined.
     if (!namespace) { return; }
-    
+
     var isUserNamespace = (namespace == $scope.user.username);
 
     $scope.planRequired = null;
@@ -2083,7 +2084,7 @@ function NewRepoCtrl($scope, $location, $http, $timeout, UserService, ApiService
         }
       }
     });
-    
+
     return true;
   };
 
@@ -2156,7 +2157,7 @@ function NewRepoCtrl($scope, $location, $http, $timeout, UserService, ApiService
     var isUserNamespace = $scope.isUserNamespace;
     ApiService.getPrivateAllowed(isUserNamespace ? null : $scope.repo.namespace).then(function(resp) {
       $scope.checkingPlan = false;
- 
+
       if (resp['privateAllowed']) {
         $scope.planRequired = null;
         return;
@@ -2198,8 +2199,8 @@ function OrgViewCtrl($rootScope, $scope, ApiService, $routeParams) {
       { 'id': 'creator', 'title': 'Creator', 'kind': 'success' },
       { 'id': 'admin', 'title': 'Admin', 'kind': 'primary' }
   ];
-  
-  $scope.setRole = function(role, teamname) {    
+
+  $scope.setRole = function(role, teamname) {
     var previousRole = $scope.organization.teams[teamname].role;
     $scope.organization.teams[teamname].role = role;
 
@@ -2211,7 +2212,7 @@ function OrgViewCtrl($rootScope, $scope, ApiService, $routeParams) {
     var data = $scope.organization.teams[teamname];
 
     ApiService.updateOrganizationTeam(data, params).then(function(resp) {
-    }, function(resp) {     
+    }, function(resp) {
       $scope.organization.teams[teamname].role = previousRole;
       $scope.roleError = resp.data || '';
       $('#cannotChangeTeamModal').modal({});
@@ -2286,7 +2287,7 @@ function OrgAdminCtrl($rootScope, $scope, $timeout, Restangular, $routeParams, U
     PlanService.getPlans(function(plans) {
       $scope.plans = plans;
       $scope.plan_map = {};
-      
+
       for (var i = 0; i < plans.length; ++i) {
         $scope.plan_map[plans[i].stripeId] = plans[i];
       }
@@ -2301,7 +2302,7 @@ function OrgAdminCtrl($rootScope, $scope, $timeout, Restangular, $routeParams, U
   $scope.invoicesShown = 0;
   $scope.applicationsShown = 0;
   $scope.changingOrganization = false;
-  
+
   $scope.loadLogs = function() {
     $scope.logsShown++;
   };
@@ -2424,7 +2425,7 @@ function TeamViewCtrl($rootScope, $scope, $timeout, Features, Restangular, ApiSe
 
   $scope.addNewMember = function(member) {
     if (!member || $scope.memberMap[member.name]) { return; }
-          
+
     var params = {
       'orgname': orgname,
       'teamname': teamname,
@@ -2519,7 +2520,7 @@ function TeamViewCtrl($rootScope, $scope, $timeout, Features, Restangular, ApiSe
     $scope.membersResource = ApiService.getOrganizationTeamMembersAsResource(params).get(function(resp) {
       $scope.members = resp.members;
       $scope.canEditMembers = resp.can_edit;
-        
+
       $('.info-icon').popover({
         'trigger': 'hover',
         'html': true
@@ -2667,7 +2668,7 @@ function OrgMemberLogsCtrl($scope, $routeParams, $rootScope, $timeout, Restangul
       });
 
       return resp.member;
-    });   
+    });
   };
 
   // Load the org info and the member info.
@@ -2763,7 +2764,7 @@ function ManageApplicationCtrl($scope, $routeParams, $rootScope, $location, $tim
          ' under organization ' + $scope.orgname;
 
       return resp;
-    });   
+    });
   };
 
 
diff --git a/static/partials/repo-admin.html b/static/partials/repo-admin.html
index 27e4c8d64..78add8642 100644
--- a/static/partials/repo-admin.html
+++ b/static/partials/repo-admin.html
@@ -12,7 +12,7 @@
             subsection="'Admin'"></span>
     </h3>
   </div>
-  
+
   <div class="row">
     <!-- Side tabs -->
     <div class="col-md-2">
@@ -30,8 +30,8 @@
 
     <!-- Content -->
     <div class="col-md-10">
-      
-      <div class="tab-content"> 
+
+      <div class="tab-content">
         <!-- Logs tab -->
         <div id="logs" class="tab-pane">
           <div class="logs-view" repository="repo" makevisible="logsShown"></div>
@@ -48,7 +48,7 @@
                 Note: This repository is currently <b>private</b>. Publishing this badge will reveal the status information of your repository (and links may
                 not work for unregistered users).
               </div>
-              
+
               <!-- Status Image -->
               <a ng-href="/repository/{{ repo.namespace }}/{{ repo.name }}" ng-if="repo && repo.name">
                 <img ng-src="/repository/{{ repo.namespace }}/{{ repo.name }}/status?token={{ repo.status_token }}" data-title="Docker Repository on Quay.io">
@@ -77,12 +77,12 @@
                   <td>
                     <div class="copy-box" hovering-message="true" value="getBadgeFormat('asciidoc', repo)"></div>
                   </td>
-                </tr>                
+                </tr>
               </table>
             </div>
           </div>
         </div>
-        
+
         <!-- Permissions tab -->
         <div id="permissions" class="tab-pane active">
           <!-- User Access Permissions -->
@@ -102,7 +102,7 @@
                     <td style="width: 95px;"></td>
                   </tr>
                 </thead>
-                
+
                 <!-- Team Permissions -->
                 <tr ng-repeat="(name, permission) in permissions['team']">
                   <td class="team entity">
@@ -117,7 +117,7 @@
                     <span class="delete-ui" delete-title="'Delete Permission'" perform-delete="deleteRole(name, 'team')"></span>
                   </td>
                 </tr>
-                
+
                 <!-- User Permissions -->
                 <tr ng-repeat="(name, permission) in permissions['user']">
                   <td class="{{ 'user entity ' + (permission.is_org_member ? '' : 'outside') }}">
@@ -164,10 +164,10 @@
                       <td></td>
                     </tr>
                   </thead>
-                  
+
                   <tr ng-repeat="(code, token) in tokens">
                     <td class="user token">
-                      <i class="fa fa-key"></i> 
+                      <i class="fa fa-key"></i>
                       <a ng-click="showToken(token.code)">{{ token.friendlyName }}</a>
                     </td>
                     <td class="user-permissions">
@@ -245,7 +245,7 @@
                       <td style="width: 104px;"></td>
                     </tr>
                   </thead>
-                  
+
                   <tbody>
                     <tr ng-repeat="trigger in triggers">
                       <td>
@@ -279,7 +279,7 @@
                             </li>
                           </ul>
                         </div>
- 
+
                         <div class="dropdown" style="display: inline-block">
                           <button class="btn btn-default dropdown-toggle" data-toggle="dropdown" data-title="Trigger Settings" bs-tooltip="tooltip.title" data-container="body">
                             <i class="fa fa-cog"></i>
@@ -296,19 +296,20 @@
                   </tbody>
                 </table>
 
-                
+
                 <!-- Right controls -->
                 <div class="right-controls">
                   <span ng-show="!Features.GITHUB_BUILD" class="pull-left">No build trigger types enabled.</span>
                   <div class="dropdown">
-                    <button class="btn btn-primary dropdown-toggle" data-toggle="dropdown" ng-disabled="!Features.GITHUB_BUILD">                      
+                    <button class="btn btn-primary dropdown-toggle" data-toggle="dropdown" ng-disabled="!Features.GITHUB_BUILD">
                       New Trigger
                       <b class="caret"></b>
                     </button>
                     <ul class="dropdown-menu dropdown-menu-right pull-right">
                       <li>
                         <a href="{{ TriggerService.getRedirectUrl('github', repo.namespace, repo.name) }}">
-                          <i class="fa fa-github fa-lg"></i>GitHub - Repository Push
+                          <i class="fa fa-github fa-lg"></i>
+                          GitHub <span ng-if="KeyService.isEnterprise('github-trigger')">Enterprise</span> - Repository Push
                         </a>
                       </li>
                     </ul>
@@ -321,7 +322,7 @@
         </div>
 
         <!-- Public/private tab -->
-        <div id="publicprivate" class="tab-pane">         
+        <div id="publicprivate" class="tab-pane">
           <!-- Public/Private -->
           <div class="panel panel-default">
             <div class="panel-body">

From a8473db87f25dc27803fd7f0eac9d26f439302a8 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Wed, 26 Nov 2014 17:02:49 -0500
Subject: [PATCH 24/26] Make sure the realm is connected before heartbeat
 checks start.

---
 buildman/component/buildcomponent.py | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/buildman/component/buildcomponent.py b/buildman/component/buildcomponent.py
index 12e6b280b..bea163aa8 100644
--- a/buildman/component/buildcomponent.py
+++ b/buildman/component/buildcomponent.py
@@ -18,6 +18,7 @@ from data.database import BUILD_PHASE
 
 HEARTBEAT_DELTA = datetime.timedelta(seconds=30)
 HEARTBEAT_TIMEOUT = 10
+INITIAL_TIMEOUT = 25
 
 LOGGER = logging.getLogger(__name__)
 
@@ -304,17 +305,14 @@ class BuildComponent(BaseComponent):
     """ Updates the last known heartbeat. """
     self._last_heartbeat = datetime.datetime.now()
 
-  def _start_heartbeat(self, loop):
-    """ Begins an async loop to keep a heartbeat going with a client. """
-    trollius.set_event_loop(loop)
-    loop.run_until_complete(self._heartbeat())
-
   @trollius.coroutine
   def _heartbeat(self):
     """ Coroutine that runs every HEARTBEAT_TIMEOUT seconds, both checking the worker's heartbeat
         and updating the heartbeat in the build status dictionary (if applicable). This allows
         the build system to catch crashes from either end.
     """
+    yield From(trollius.sleep(INITIAL_TIMEOUT))
+
     while True:
       # If the component is no longer running or actively building, nothing more to do.
       if (self._component_status != ComponentStatus.RUNNING and
@@ -335,11 +333,7 @@ class BuildComponent(BaseComponent):
 
       # Check the heartbeat from the worker.
       LOGGER.debug('Checking heartbeat on realm %s', self.builder_realm)
-      if not self._last_heartbeat:
-        self._timeout()
-        return
-
-      if self._last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
+      if self._last_heartbeat and self._last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
         self._timeout()
         return
 
@@ -347,7 +341,7 @@ class BuildComponent(BaseComponent):
 
   def _timeout(self):
     self._set_status(ComponentStatus.TIMED_OUT)
-    LOGGER.warning('Build component (token "%s") timed out', self.expected_token)
+    LOGGER.warning('Build component with realm %s has timed out', self.builder_realm)
     self._dispose(timed_out=True)
 
   def _dispose(self, timed_out=False):

From 09cc4ba4c1dfded3efd1caed65536b4a0dd76ad8 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Sun, 30 Nov 2014 17:48:02 -0500
Subject: [PATCH 25/26] LOGGER -> logger.

While logger may be a global variable, it is not constant. Let the
linters complain!
---
 buildman/builder.py                  | 10 +++++-----
 buildman/component/buildcomponent.py | 26 +++++++++++++-------------
 buildman/manager/enterprise.py       |  6 +++---
 buildman/server.py                   | 28 ++++++++++++++--------------
 4 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/buildman/builder.py b/buildman/builder.py
index 86097c8a4..d6ba15ea6 100644
--- a/buildman/builder.py
+++ b/buildman/builder.py
@@ -9,7 +9,7 @@ from buildman.server import BuilderServer
 
 from trollius import SSLContext
 
-LOGGER = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
 
 BUILD_MANAGERS = {
   'enterprise': EnterpriseManager
@@ -17,22 +17,22 @@ BUILD_MANAGERS = {
 
 def run_build_manager():
   if not features.BUILD_SUPPORT:
-    LOGGER.debug('Building is disabled. Please enable the feature flag')
+    logger.debug('Building is disabled. Please enable the feature flag')
     return
 
   build_manager_config = app.config.get('BUILD_MANAGER')
   if build_manager_config is None:
     return
 
-  LOGGER.debug('Asking to start build manager with lifecycle "%s"', build_manager_config[0])
+  logger.debug('Asking to start build manager with lifecycle "%s"', build_manager_config[0])
   manager_klass = BUILD_MANAGERS.get(build_manager_config[0])
   if manager_klass is None:
     return
 
-  LOGGER.debug('Starting build manager with lifecycle "%s"', build_manager_config[0])
+  logger.debug('Starting build manager with lifecycle "%s"', build_manager_config[0])
   ssl_context = None
   if os.environ.get('SSL_CONFIG'):
-    LOGGER.debug('Loading SSL cert and key')
+    logger.debug('Loading SSL cert and key')
     ssl_context = SSLContext()
     ssl_context.load_cert_chain(os.environ.get('SSL_CONFIG') + '/ssl.cert',
                                 os.environ.get('SSL_CONFIG') + '/ssl.key')
diff --git a/buildman/component/buildcomponent.py b/buildman/component/buildcomponent.py
index bea163aa8..a7bf21c54 100644
--- a/buildman/component/buildcomponent.py
+++ b/buildman/component/buildcomponent.py
@@ -20,7 +20,7 @@ HEARTBEAT_DELTA = datetime.timedelta(seconds=30)
 HEARTBEAT_TIMEOUT = 10
 INITIAL_TIMEOUT = 25
 
-LOGGER = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
 
 class ComponentStatus(object):
   """ ComponentStatus represents the possible states of a component. """
@@ -51,7 +51,7 @@ class BuildComponent(BaseComponent):
     self.join(self.builder_realm)
 
   def onJoin(self, details):
-    LOGGER.debug('Registering methods and listeners for component %s', self.builder_realm)
+    logger.debug('Registering methods and listeners for component %s', self.builder_realm)
     yield From(self.register(self._on_ready, u'io.quay.buildworker.ready'))
     yield From(self.register(self._ping, u'io.quay.buildworker.ping'))
     yield From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat'))
@@ -75,7 +75,7 @@ class BuildComponent(BaseComponent):
     buildpack_url = self.user_files.get_file_url(build_job.repo_build().resource_key,
                                                  requires_cors=False)
 
-    LOGGER.debug('Retreiving build package: %s', buildpack_url)
+    logger.debug('Retreiving build package: %s', buildpack_url)
     buildpack = None
     try:
       buildpack = BuildPackage.from_url(buildpack_url)
@@ -85,7 +85,7 @@ class BuildComponent(BaseComponent):
 
     # Extract the base image information from the Dockerfile.
     parsed_dockerfile = None
-    LOGGER.debug('Parsing dockerfile')
+    logger.debug('Parsing dockerfile')
 
     build_config = build_job.build_config()
     try:
@@ -143,8 +143,8 @@ class BuildComponent(BaseComponent):
     }
 
     # Invoke the build.
-    LOGGER.debug('Invoking build: %s', self.builder_realm)
-    LOGGER.debug('With Arguments: %s', build_arguments)
+    logger.debug('Invoking build: %s', self.builder_realm)
+    logger.debug('With Arguments: %s', build_arguments)
 
     return (self
             .call("io.quay.builder.build", **build_arguments)
@@ -217,7 +217,7 @@ class BuildComponent(BaseComponent):
     # the pull/push progress, as well as the current step index.
     with self._build_status as status_dict:
       if self._build_status.set_phase(phase):
-        LOGGER.debug('Build %s has entered a new phase: %s', self.builder_realm, phase)
+        logger.debug('Build %s has entered a new phase: %s', self.builder_realm, phase)
 
       BuildComponent._process_pushpull_status(status_dict, phase, docker_data, self._image_info)
 
@@ -243,7 +243,7 @@ class BuildComponent(BaseComponent):
     })
 
     build_id = self._current_job.repo_build().uuid
-    LOGGER.warning('Build %s failed with message: %s', build_id, error_message)
+    logger.warning('Build %s failed with message: %s', build_id, error_message)
 
     # Mark that the build has finished (in an error state)
     self._build_finished(BuildJobResult.ERROR)
@@ -283,11 +283,11 @@ class BuildComponent(BaseComponent):
 
   def _on_ready(self, token):
     if self._component_status != 'waiting':
-      LOGGER.warning('Build component (token "%s") is already connected', self.expected_token)
+      logger.warning('Build component (token "%s") is already connected', self.expected_token)
       return
 
     if token != self.expected_token:
-      LOGGER.warning('Builder token mismatch. Expected: "%s". Found: "%s"', self.expected_token, token)
+      logger.warning('Builder token mismatch. Expected: "%s". Found: "%s"', self.expected_token, token)
       return
 
     self._set_status(ComponentStatus.RUNNING)
@@ -295,7 +295,7 @@ class BuildComponent(BaseComponent):
     # Start the heartbeat check and updating loop.
     loop = trollius.get_event_loop()
     loop.create_task(self._heartbeat())
-    LOGGER.debug('Build worker %s is connected and ready', self.builder_realm)
+    logger.debug('Build worker %s is connected and ready', self.builder_realm)
     return True
 
   def _set_status(self, phase):
@@ -332,7 +332,7 @@ class BuildComponent(BaseComponent):
         self.parent_manager.job_heartbeat(current_job)
 
       # Check the heartbeat from the worker.
-      LOGGER.debug('Checking heartbeat on realm %s', self.builder_realm)
+      logger.debug('Checking heartbeat on realm %s', self.builder_realm)
       if self._last_heartbeat and self._last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
         self._timeout()
         return
@@ -341,7 +341,7 @@ class BuildComponent(BaseComponent):
 
   def _timeout(self):
     self._set_status(ComponentStatus.TIMED_OUT)
-    LOGGER.warning('Build component with realm %s has timed out', self.builder_realm)
+    logger.warning('Build component with realm %s has timed out', self.builder_realm)
     self._dispose(timed_out=True)
 
   def _dispose(self, timed_out=False):
diff --git a/buildman/manager/enterprise.py b/buildman/manager/enterprise.py
index 741781078..824e02d53 100644
--- a/buildman/manager/enterprise.py
+++ b/buildman/manager/enterprise.py
@@ -8,7 +8,7 @@ from buildman.manager.basemanager import BaseManager
 from trollius.coroutines import From
 
 REGISTRATION_REALM = 'registration'
-LOGGER = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
 
 class DynamicRegistrationComponent(BaseComponent):
   """ Component session that handles dynamic registration of the builder components. """
@@ -17,12 +17,12 @@ class DynamicRegistrationComponent(BaseComponent):
     self.join(REGISTRATION_REALM)
 
   def onJoin(self, details):
-    LOGGER.debug('Registering registration method')
+    logger.debug('Registering registration method')
     yield From(self.register(self._worker_register, u'io.quay.buildworker.register'))
 
   def _worker_register(self):
     realm = self.parent_manager.add_build_component()
-    LOGGER.debug('Registering new build component+worker with realm %s', realm)
+    logger.debug('Registering new build component+worker with realm %s', realm)
     return realm
 
 
diff --git a/buildman/server.py b/buildman/server.py
index fccaeacfb..3863406f2 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -14,7 +14,7 @@ from datetime import datetime, timedelta
 from buildman.jobutil.buildjob import BuildJob, BuildJobLoadException
 from data.queue import WorkQueue
 
-LOGGER = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
 
 WORK_CHECK_TIMEOUT = 10
 TIMEOUT_PERIOD_MINUTES = 20
@@ -68,14 +68,14 @@ class BuilderServer(object):
     self._controller_app = controller_app
 
   def run(self, host, ssl=None):
-    LOGGER.debug('Initializing the lifecycle manager')
+    logger.debug('Initializing the lifecycle manager')
     self._lifecycle_manager.initialize()
 
-    LOGGER.debug('Initializing all members of the event loop')
+    logger.debug('Initializing all members of the event loop')
     loop = trollius.get_event_loop()
     trollius.Task(self._initialize(loop, host, ssl))
 
-    LOGGER.debug('Starting server on port %s, with controller on port %s', WEBSOCKET_PORT,
+    logger.debug('Starting server on port %s, with controller on port %s', WEBSOCKET_PORT,
                  CONTROLLER_PORT)
     try:
       loop.run_forever()
@@ -85,17 +85,17 @@ class BuilderServer(object):
       loop.close()
 
   def close(self):
-    LOGGER.debug('Requested server shutdown')
+    logger.debug('Requested server shutdown')
     self._current_status = 'shutting_down'
     self._lifecycle_manager.shutdown()
     self._shutdown_event.wait()
-    LOGGER.debug('Shutting down server')
+    logger.debug('Shutting down server')
 
   def _register_component(self, realm, component_klass, **kwargs):
     """ Registers a component with the server. The component_klass must derive from
         BaseComponent.
     """
-    LOGGER.debug('Registering component with realm %s', realm)
+    logger.debug('Registering component with realm %s', realm)
 
     component = component_klass(types.ComponentConfig(realm=realm), realm=realm, **kwargs)
     component.server = self
@@ -109,7 +109,7 @@ class BuilderServer(object):
     return component
 
   def _unregister_component(self, component):
-    LOGGER.debug('Unregistering component with realm %s and token %s',
+    logger.debug('Unregistering component with realm %s and token %s',
                   component.builder_realm, component.expected_token)
 
     self._current_components.remove(component)
@@ -137,25 +137,25 @@ class BuilderServer(object):
   @trollius.coroutine
   def _work_checker(self):
     while self._current_status == 'running':
-      LOGGER.debug('Checking for more work')
+      logger.debug('Checking for more work')
       job_item = self._queue.get(processing_time=self._lifecycle_manager.setup_time())
       if job_item is None:
-        LOGGER.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT)
+        logger.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT)
         yield From(trollius.sleep(WORK_CHECK_TIMEOUT))
         continue
 
       try:
         build_job = BuildJob(job_item)
       except BuildJobLoadException as irbe:
-        LOGGER.exception(irbe)
+        logger.exception(irbe)
         self._queue.incomplete(job_item, restore_retry=False)
 
-      LOGGER.debug('Build job found. Checking for an avaliable worker.')
+      logger.debug('Build job found. Checking for an avaliable worker.')
       if self._lifecycle_manager.schedule(build_job, self._loop):
         self._job_count = self._job_count + 1
-        LOGGER.debug('Build job scheduled. Running: %s', self._job_count)
+        logger.debug('Build job scheduled. Running: %s', self._job_count)
       else:
-        LOGGER.debug('All workers are busy. Requeuing.')
+        logger.debug('All workers are busy. Requeuing.')
         self._queue.incomplete(job_item, restore_retry=True, retry_after=0)
 
       yield From(trollius.sleep(WORK_CHECK_TIMEOUT))

From 4f5bf8185a185908550cad20ce829ac926f34cd1 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <joseph.schorr@coreos.com>
Date: Mon, 1 Dec 2014 12:11:23 -0500
Subject: [PATCH 26/26] Add version checking to the python side

---
 buildman/component/buildcomponent.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/buildman/component/buildcomponent.py b/buildman/component/buildcomponent.py
index a7bf21c54..d518d3453 100644
--- a/buildman/component/buildcomponent.py
+++ b/buildman/component/buildcomponent.py
@@ -20,6 +20,8 @@ HEARTBEAT_DELTA = datetime.timedelta(seconds=30)
 HEARTBEAT_TIMEOUT = 10
 INITIAL_TIMEOUT = 25
 
+SUPPORTED_WORKER_VERSIONS = ['0.1-beta']
+
 logger = logging.getLogger(__name__)
 
 class ComponentStatus(object):
@@ -281,14 +283,18 @@ class BuildComponent(BaseComponent):
     """ Ping pong. """
     return 'pong'
 
-  def _on_ready(self, token):
+  def _on_ready(self, token, version):
+    if not version in SUPPORTED_WORKER_VERSIONS:
+      logger.warning('Build component (token "%s") is running an out-of-date version: %s', version)
+      return False
+
     if self._component_status != 'waiting':
       logger.warning('Build component (token "%s") is already connected', self.expected_token)
-      return
+      return False
 
     if token != self.expected_token:
       logger.warning('Builder token mismatch. Expected: "%s". Found: "%s"', self.expected_token, token)
-      return
+      return False
 
     self._set_status(ComponentStatus.RUNNING)