Merge branch 'master' into tutorial

Conflicts:
	config.py
	static/js/app.js
	test/data/test.db

application.py

@@ -1,5 +1,4 @@
 import logging
-import os
 
 from app import app as application
 from data.model import db as model_db
@@ -21,10 +20,6 @@ from endpoints.realtime import realtime
 
 logger = logging.getLogger(__name__)
 
-if application.config.get('INCLUDE_TEST_ENDPOINTS', False):
-  logger.debug('Loading test endpoints.')
-  import endpoints.test
-
 application.register_blueprint(web)
 application.register_blueprint(index, url_prefix='/v1')
 application.register_blueprint(tags, url_prefix='/v1')

config.py

@@ -1,5 +1,4 @@
 import logging
-import os
 import logstash_formatter
 
 from peewee import MySQLDatabase, SqliteDatabase
@@ -12,6 +11,7 @@ from util import analytics
 
 from test.teststorage import FakeStorage, FakeUserfiles
 from test import analytics as fake_analytics
+from test.testlogs import TestBuildLogs
 
 
 class FlaskConfig(object):
@@ -96,6 +96,11 @@ class UserEventConfig(object):
   USER_EVENTS = UserEventBuilder('logs.quay.io')
 
 
+class TestBuildLogs(object):
+  BUILDLOGS = TestBuildLogs('logs.quay.io', 'devtable', 'building',
+                            'deadbeef-dead-beef-dead-beefdeadbeef')
+
+
 class StripeTestConfig(object):
   STRIPE_SECRET_KEY = 'sk_test_PEbmJCYrLXPW0VRLSnWUiZ7Y'
   STRIPE_PUBLISHABLE_KEY = 'pk_test_uEDHANKm9CHCvVa2DLcipGRh'
@@ -145,13 +150,13 @@ class BuildNodeConfig(object):
   BUILD_NODE_PULL_TOKEN = 'F02O2E86CQLKZUQ0O81J8XDHQ6F0N1V36L9JTOEEK6GKKMT1GI8PTJQT4OU88Y6G'
 
 
-def logs_init_builder(level=logging.DEBUG):
+def logs_init_builder(level=logging.DEBUG,
+                      formatter=logstash_formatter.LogstashFormatter()):
   @staticmethod
   def init_logs():
     handler = logging.StreamHandler()
     root_logger = logging.getLogger('')
     root_logger.setLevel(level)
-    formatter = logstash_formatter.LogstashFormatter()
     handler.setFormatter(formatter)
     root_logger.addHandler(handler)
 
@@ -164,17 +169,15 @@ class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles,
   LOGGING_CONFIG = logs_init_builder(logging.WARN)
   POPULATE_DB_TEST_DATA = True
   TESTING = True
-  INCLUDE_TEST_ENDPOINTS = True
 
 
 class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB,
                   StripeTestConfig, MixpanelTestConfig, GitHubTestConfig,
                   DigitalOceanConfig, BuildNodeConfig, S3Userfiles,
-                  RedisBuildLogs, UserEventConfig):
+                  RedisBuildLogs, UserEventConfig, TestBuildLogs):
-  LOGGING_CONFIG = logs_init_builder()
+  LOGGING_CONFIG = logs_init_builder(formatter=logging.Formatter())
   SEND_FILE_MAX_AGE_DEFAULT = 0
   POPULATE_DB_TEST_DATA = True
-  INCLUDE_TEST_ENDPOINTS = True
 
 
 class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL,

data/buildlogs.py

@@ -3,6 +3,10 @@ import json
 
 
 class BuildLogs(object):
+  ERROR = 'error'
+  COMMAND = 'command'
+  PHASE = 'phase'
+
   def __init__(self, redis_host):
     self._redis = redis.StrictRedis(host=redis_host)
 
@@ -17,24 +21,27 @@ class BuildLogs(object):
     """
     return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj))
 
-  def append_log_message(self, build_id, log_message):
+  def append_log_message(self, build_id, log_message, log_type=None):
     """
     Wraps the message in an envelope and push it to the end of the log entry
-    list and returns the new length of the list.
+    list and returns the index at which it was inserted.
     """
     log_obj = {
       'message': log_message
     }
-    return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj))
 
-  def get_log_entries(self, build_id, start_index, end_index):
+    if log_type:
+      log_obj['type'] = log_type
+
+    return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj)) - 1
+
+  def get_log_entries(self, build_id, start_index):
     """ 
     Returns a tuple of the current length of the list and an iterable of the
-    requested log entries. End index is inclusive.
+    requested log entries.
     """
     llen = self._redis.llen(self._logs_key(build_id))
-    log_entries = self._redis.lrange(self._logs_key(build_id), start_index,
+    log_entries = self._redis.lrange(self._logs_key(build_id), start_index, -1)
-                                     end_index)
     return (llen, (json.loads(entry) for entry in log_entries))
 
   @staticmethod

data/database.py

@@ -215,6 +215,8 @@ class RepositoryBuild(BaseModel):
   resource_key = CharField()
   tag = CharField()
   phase = CharField(default='waiting')
+  started = DateTimeField(default=datetime.now)
+  display_name = CharField()
 
 
 class QueueItem(BaseModel):

data/model.py

@@ -1309,9 +1309,11 @@ def list_repository_builds(namespace_name, repository_name,
   return fetched
 
 
-def create_repository_build(repo, access_token, resource_key, tag):
+def create_repository_build(repo, access_token, resource_key, tag,
+                            display_name):
   return RepositoryBuild.create(repository=repo, access_token=access_token,
-                                resource_key=resource_key, tag=tag)
+                                resource_key=resource_key, tag=tag,
+                                display_name=display_name)
 
 
 def create_webhook(repo, params_obj):

data/queue.py

@@ -64,5 +64,5 @@ class WorkQueue(object):
 
 
 image_diff_queue = WorkQueue('imagediff')
-dockerfile_build_queue = WorkQueue('dockerfilebuild')
+dockerfile_build_queue = WorkQueue('dockerfilebuild2')
 webhook_queue = WorkQueue('webhook')

data/userfiles.py

@@ -59,3 +59,9 @@ class UserRequestFiles(object):
     full_key = os.path.join(self._prefix, file_id)
     k = Key(self._bucket, full_key)
     return k.generate_url(expires_in)
+
+  def get_file_checksum(self, file_id):
+    self._initialize_s3()
+    full_key = os.path.join(self._prefix, file_id)
+    k = self._bucket.lookup(full_key)
+    return k.etag[1:-1][:7]

endpoints/api.py

@@ -70,7 +70,7 @@ def get_route_data():
   routes = []
   for rule in app.url_map.iter_rules():
     if rule.endpoint.startswith('api.'):
-      endpoint_method = globals()[rule.endpoint[4:]] # Remove api.
+      endpoint_method = app.view_functions[rule.endpoint]
       is_internal = '__internal_call' in dir(endpoint_method)
       is_org_api = '__user_call' in dir(endpoint_method)
       methods = list(rule.methods.difference(['HEAD', 'OPTIONS']))
@@ -1154,6 +1154,8 @@ def build_status_view(build_obj):
   return {
     'id': build_obj.uuid,
     'phase': build_obj.phase,
+    'started': build_obj.started,
+    'display_name': build_obj.display_name,
     'status': status,
   }
 
@@ -1191,25 +1193,22 @@ def get_repo_build_status(namespace, repository, build_uuid):
 def get_repo_build_logs(namespace, repository, build_uuid):
   permission = ModifyRepositoryPermission(namespace, repository)
   if permission.can():
+    response_obj = {}
+
     build = model.get_repository_build(namespace, repository, build_uuid)
 
-    start = int(request.args.get('start', -1000))
+    start = int(request.args.get('start', 0))
-    end = int(request.args.get('end', -1))
-    count, logs = build_logs.get_log_entries(build.uuid, start, end)
 
-    if start < 0:
+    count, logs = build_logs.get_log_entries(build.uuid, start)
-      start = max(0, count + start)
 
-    if end < 0:
+    response_obj.update({
-      end = count + end
-
-    return jsonify({
       'start': start,
-      'end': end,
       'total': count,
       'logs': [log for log in logs],
     })
 
+    return jsonify(response_obj)
+
   abort(403)  # Permission denied
 
 
@@ -1224,11 +1223,13 @@ def request_repo_build(namespace, repository):
 
     repo = model.get_repository(namespace, repository)
     token = model.create_access_token(repo, 'write')
+    display_name = user_files.get_file_checksum(dockerfile_id)
+    logger.debug('**********Md5: %s' % display_name)
 
     host = urlparse.urlparse(request.url).netloc
     tag = '%s/%s/%s' % (host, repo.namespace, repo.name)
     build_request = model.create_repository_build(repo, token, dockerfile_id,
-                                                  tag)
+                                                  tag, display_name)
     dockerfile_build_queue.put(json.dumps({
       'build_uuid': build_request.uuid,
       'namespace': namespace,

endpoints/test.py

@@ -1,61 +0,0 @@
-import math
-
-from random import SystemRandom
-from flask import jsonify
-from app import app
-
-
-def generate_image_completion(rand_func):
-  images = {}
-  for image_id in range(rand_func.randint(1, 11)):
-    total = int(math.pow(abs(rand_func.gauss(0, 1000)), 2))
-    current = rand_func.randint(0, total)
-    image_id = 'image_id_%s' % image_id
-    images[image_id] = {
-      'total': total,
-      'current': current,
-    }
-  return images
-
-
-@app.route('/test/build/status', methods=['GET'])
-def generate_random_build_status():
-  response = {
-    'id': 1,
-    'total_commands': None,
-    'current_command': None,
-    'push_completion': 0.0,
-    'status': None,
-    'message': None,
-    'image_completion': {},
-  }
-
-  random = SystemRandom()
-  phases = {
-    'waiting': {},
-    'starting': {
-      'total_commands': 7,
-      'current_command': 0,
-    },
-    'initializing': {},
-    'error': {
-      'message': 'Oops!'
-    },
-    'complete': {},
-    'building': {
-      'total_commands': 7,
-      'current_command': random.randint(1, 7),    
-    },
-    'pushing': {
-      'total_commands': 7,
-      'current_command': 7,
-      'push_completion': random.random(),
-      'image_completion': generate_image_completion(random),
-    },
-  }
-
-  phase = random.choice(phases.keys())
-  response['status'] = phase
-  response.update(phases[phase])
-
-  return jsonify(response)

initdb.py

@@ -275,6 +275,13 @@ def populate_database():
                                    'Empty repository which is building.',
                                    False, [], (0, [], None))
 
+  token = model.create_access_token(building, 'write')
+  tag = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name)
+  build = model.create_repository_build(building, token, '123-45-6789', tag,
+                                        'build-name')
+  build.uuid = 'deadbeef-dead-beef-dead-beefdeadbeef'
+  build.save()
+
   org = model.create_organization('buynlarge', 'quay@devtable.com',
                                   new_user_1)
   org.stripe_id = TEST_STRIPE_ID
@@ -298,19 +305,11 @@ def populate_database():
   model.add_user_to_team(new_user_2, reader_team)
   model.add_user_to_team(reader, reader_team)
 
-  token = model.create_access_token(building, 'write')
-  tag = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name)
-  build = model.create_repository_build(building, token, '123-45-6789', tag)
-
-  build.build_node_id = 1
-  build.phase = 'building'
-  build.status_url = 'http://localhost:5000/test/build/status'
-  build.save()
-
   __generate_repository(new_user_1, 'superwide', None, False, [],
                         [(10, [], 'latest2'),
                               (2, [], 'latest3'),
-                              (2, [(1, [], 'latest11'), (2, [], 'latest12')], 'latest4'),
+                              (2, [(1, [], 'latest11'), (2, [], 'latest12')],
+                               'latest4'),
                               (2, [], 'latest5'),
                               (2, [], 'latest6'),
                               (2, [], 'latest7'),

requirements-nover.txt

@@ -22,3 +22,4 @@ logstash_formatter
 redis
 hiredis
 git+https://github.com/dotcloud/docker-py.git
+loremipsum

requirements.txt

@@ -23,6 +23,7 @@ html5lib==1.0b3
 itsdangerous==0.23
 lockfile==0.9.1
 logstash-formatter==0.5.8
+loremipsum==1.0.2
 marisa-trie==0.5.1
 mixpanel-py==3.1.1
 mock==1.0.1

static/css/quay.css

@@ -507,35 +507,103 @@ i.toggle-icon:hover {
   color: #428bca;
 }
 
-.status-boxes .popover {
+.status-box a {
-  margin-right: 20px;
+  padding: 6px;
+  color: black;
 }
 
-.status-boxes .popover-content {
+.status-box a b {
-  width: 260px;
+  margin-right: 10px;
 }
 
-.build-statuses {
+.build-info {
+  margin: 4px;
+  padding: 4px;
+  margin-left: 6px;
+  margin-right: 6px;
+  border-bottom: 1px solid #eee;
+}
+
+.build-info.clickable:hover {
+  background: rgba(66, 139, 202, 0.2);
+  cursor: pointer;
+  border-radius: 4px;
+}
+
+
+.build-info:last-child {
+  border-bottom: 0px;
+}
+
+.phase-icon {
+  border-radius: 50%;
+  display: inline-block;
+  width: 12px;
+  height: 12px;
+  margin-right: 6px;
+}
+
+.active .build-tab-link .phase-icon {
+  box-shadow: 0px 0px 10px #FFFFFF, 0px 0px 10px #FFFFFF;
+}
+
+.build-status .phase-icon {
+  margin-top: 4px;
+  float: left;
+}
+
+.phase-icon.error {
+  background-color: red;
+}
+
+.phase-icon.waiting, .phase-icon.starting, .phase-icon.initializing {
+  background-color: #ddd;
+}
+
+.phase-icon.building {
+  background-color: #f0ad4e;
+}
+
+.phase-icon.pushing {
+  background-color: #5cb85c;
+}
+
+.phase-icon.complete {
+  background-color: #428bca;
+}
+
+.build-status {
+  display: inline-block;
 }
 
 .build-status-container {
   padding: 4px;
   margin-bottom: 10px;
   border-bottom: 1px solid #eee;
-  width: 230px;
+  width: 350px;
 }
 
 .build-status-container .build-message {
   display: block;
   white-space: nowrap;
-  font-size: 12px;
+  font-size: 14px;
+  margin-bottom: 10px;
+  padding-bottom: 10px;
+  border-bottom: 1px solid #eee;
+  margin-left: 20px;
 }
 
 .build-status-container .progress {
-  height: 12px;
+  height: 10px;
   margin: 0px;
   margin-top: 10px;
-  width: 230px;
+  margin-left: 20px;
+  width: 310px;
+}
+
+.build-status-container .timing {
+  margin-left: 20px;
+  margin-top: 6px;
 }
 
 .build-status-container:last-child {
@@ -1633,6 +1701,185 @@ p.editable:hover i {
   padding-left: 44px;
 }
 
+
+.repo-build .build-id:before {
+  content: "Build ID: "
+}
+
+.repo-build .build-id {
+  float: right;
+  font-size: 12px;
+  color: #aaa;
+  padding: 10px;
+}
+
+.repo-build .build-pane .timing {
+  float: right;
+}
+
+.repo-build .build-tab-link {
+  white-space: nowrap;
+}
+
+.repo-build .build-pane .build-header {
+  padding-top: 10px;
+  border-bottom: 1px solid #eee;
+  padding-bottom: 10px;
+}
+
+.repo-build .build-pane .build-progress {
+  margin-top: 16px;
+  margin-bottom: 10px;
+}
+
+.repo-build .build-pane .build-progress .progress {
+  height: 14px;
+  margin-bottom: 0px;
+}
+
+.repo-build .build-pane .quay-spinner {
+  margin-top: 4px;
+  display: inline-block;
+}
+
+.repo-build .build-pane .build-logs {
+  background: #222;
+  color: white;
+  padding: 10px;
+  overflow: auto;
+}
+
+.repo-build .build-pane .build-logs .container-header {
+  padding: 2px;
+}
+
+.repo-build .build-pane .build-logs .container-logs {
+  margin: 4px;
+  padding-bottom: 4px;
+}
+
+.repo-build .build-pane .build-logs .command-title,
+.repo-build .build-pane .build-logs .log-entry .message {
+  font-family: Consolas, "Lucida Console", Monaco, monospace;
+  font-size: 13px;
+}
+
+.repo-build .build-pane .build-logs .container-header {
+  cursor: pointer;
+  position: relative;
+}
+
+.repo-build .build-pane .build-logs .container-header i.fa.chevron {
+  color: #666;
+  margin-right: 4px;
+  width: 14px;
+  text-align: center;
+
+  position: absolute;
+  top: 6px;
+  left: 0px;
+}
+
+.repo-build .build-pane .build-logs .log-container.command {
+  margin-left: 42px;
+}
+
+.repo-build .build-pane .build-logs .container-header.building {
+  margin-bottom: 10px;
+}
+
+.repo-build .build-pane .build-logs .container-header.pushing {
+  margin-top: 10px;
+}
+
+.repo-build .build-log-error-element {
+  position: relative;
+  display: inline-block;
+  margin: 10px;
+  padding: 10px;
+  background: rgba(255, 0, 0, 0.17);
+  border-radius: 10px;
+  margin-left: 22px;
+}
+
+.repo-build .build-log-error-element i.fa {
+  color: red;
+  position: absolute;
+  top: 13px;
+  left: 11px;
+}
+
+.repo-build .build-log-error-element .error-message {
+  display: inline-block;
+  margin-left: 25px;
+}
+
+.repo-build .build-pane .build-logs .container-header .label {
+  padding-top: 4px;
+  text-align: right;
+  margin-right: 4px;
+  width: 86px;
+  display: inline-block;
+
+  border-right: 4px solid #aaa;
+  background-color: #444;
+
+  position: absolute;
+  top: 4px;
+  left: 24px;
+}
+
+.repo-build .build-pane .build-logs .container-header .container-content {
+  display: block;
+  padding-left: 20px;
+}
+
+.repo-build .build-pane .build-logs .container-header .container-content.build-log-command {
+  padding-left: 120px;
+}
+
+.label.FROM {
+  border-color: #5bc0de !important;
+}
+
+.label.CMD, .label.EXPOSE, .label.ENTRYPOINT {
+  border-color: #428bca !important;
+}
+
+.label.RUN, .label.ADD {
+  border-color: #5cb85c !important;
+}
+
+.label.ENV, .label.VOLUME, .label.USER, .label.WORKDIR {
+  border-color: #f0ad4e !important;
+}
+
+.label.MAINTAINER {
+  border-color: #aaa !important;
+}
+
+.repo-build .build-pane .build-logs .log-entry {
+  position: relative;
+}
+
+.repo-build .build-pane .build-logs .log-entry .message {
+  display: inline-block;
+  margin-left: 46px;
+}
+
+.repo-build .build-pane .build-logs .log-entry .id {
+  color: #aaa;
+  padding-right: 6px;
+  margin-right: 6px;
+  text-align: right;
+  font-size: 12px;
+  width: 40px;
+
+  position: absolute;
+  top: 4px;
+  left: 4px;
+}
+
 .repo-admin .right-info {
   font-size: 11px;
   margin-top: 10px;
@@ -1676,16 +1923,6 @@ p.editable:hover i {
   cursor: pointer;
 }
 
-.repo .build-info {
-  padding: 10px;
-  margin: 0px;
-}
-
-.repo .build-info .progress {
-  margin: 0px;
-  margin-top: 10px;
-}
-
 .repo .section {
   display: block;
   margin-bottom: 20px;

static/directives/build-log-command.html

@@ -0,0 +1,6 @@
+<span class="command" bindonce>
+  <span class="label" bo-class="getCommandKind(command.message)" bo-show="getCommandKind(command.message)"
+        bo-text="getCommandKind(command.message)">
+  </span>
+  <span class="command-title" bo-html="getCommandTitleHtml(command.message)"></span>
+</span>

static/directives/build-log-error.html

@@ -0,0 +1,4 @@
+<span bindonce class="build-log-error-element">
+  <i class="fa fa-exclamation-triangle"></i>
+  <span class="error-message" bo-text="error.message"></span>
+</span>

static/directives/build-log-phase.html

@@ -0,0 +1,4 @@
+<span bindonce class="build-log-phase-element">
+  <span class="phase-icon" ng-class="phase.message"></span>
+  <span class="build-message" phase="phase.message"></span>
+</span>

static/directives/build-message.html

@@ -0,0 +1 @@
+<span class="build-message-element">{{ getBuildMessage(phase) }}</span>

static/directives/build-progress.html

@@ -0,0 +1,6 @@
+<div class="build-progress-element">
+  <div class="progress" ng-class="getPercentage(build) < 100 ? 'active progress-striped' : ''">
+    <div class="progress-bar"  role="progressbar" aria-valuenow="{{ getPercentage(build) }}" aria-valuemin="0" aria-valuemax="100" style="{{ 'width: ' + getPercentage(build) + '%' }}">
+    </div>
+  </div>
+</div>

static/directives/build-status.html

@@ -1,8 +1,11 @@
 <div id="build-status-container" class="build-status-container">
-  <span class="build-message">{{ getBuildMessage(build) }}</span>
+  <div>
-  <div class="progress" ng-class="getBuildProgress(build) < 100 ? 'active progress-striped' : ''" ng-show="getBuildProgress(build) >= 0">
+    <span class="phase-icon" ng-class="build.phase"></span>
-    <div class="progress-bar"  role="progressbar" aria-valuenow="{{ getBuildProgress(build) }}" aria-valuemin="0" aria-valuemax="100" style="{{ 'width: ' + getBuildProgress(build) + '%' }}">
+    <span class="build-message" phase="build.phase"></span>
   </div>
+  <div class="timing">
+    <i class="fa fa-clock-o"></i>
+    Started: <span am-time-ago="build.started || 0"></span>
   </div>
-
+  <div class="build-progress" build="build"></div>
 </div>

static/js/app.js

@@ -102,8 +102,7 @@ function getMarkedDown(string) {
   return Markdown.getSanitizingConverter().makeHtml(string || '');
 }
 
-// Start the application code itself.
+quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'angular-tour', 'restangular', 'angularMoment', 'angulartics', /*'angulartics.google.analytics',*/ 'angulartics.mixpanel', '$strap.directives', 'ngCookies', 'ngSanitize', 'angular-md5', 'pasvaz.bindonce'], function($provide, cfpLoadingBarProvider) {
-quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'angular-tour', 'restangular', 'angularMoment', 'angulartics', /*'angulartics.google.analytics',*/ 'angulartics.mixpanel', '$strap.directives', 'ngCookies', 'ngSanitize', 'angular-md5'], function($provide, cfpLoadingBarProvider) {
     cfpLoadingBarProvider.includeSpinner = false;
 
     $provide.factory('UtilService', ['$sanitize', function($sanitize) {
@@ -151,7 +150,7 @@ quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'angu
     $provide.factory('ApiService', ['Restangular', function(Restangular) {
       var apiService = {};
 
-      var getResource = function(path) {
+      var getResource = function(path, opt_background) {
         var resource = {};
         resource.url = path;
         resource.withOptions = function(options) {
@@ -169,6 +168,12 @@ quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'angu
             'hasError': false
           };
 
+          if (opt_background) {
+            performer.withHttpConfig({
+              'ignoreLoadingBar': true
+            });
+          }
+
           performer.get(options).then(function(resp) {
             result.value = processor(resp);
             result.loading = false;
@@ -240,27 +245,33 @@ quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'angu
       var buildMethodsForEndpoint = function(endpoint) {
         var method = endpoint.methods[0].toLowerCase();
         var methodName = formatMethodName(endpoint['name']);
-        apiService[methodName] = function(opt_options, opt_parameters) {
+        apiService[methodName] = function(opt_options, opt_parameters, opt_background) {
-          return Restangular.one(buildUrl(endpoint['path'], opt_parameters))['custom' + method.toUpperCase()](opt_options);
+          var one = Restangular.one(buildUrl(endpoint['path'], opt_parameters));
+          if (opt_background) {
+            one.withHttpConfig({
+              'ignoreLoadingBar': true
+            });
+          }
+          return one['custom' + method.toUpperCase()](opt_options);
         };
 
         if (method == 'get') {
-          apiService[methodName + 'AsResource'] = function(opt_parameters) {
+          apiService[methodName + 'AsResource'] = function(opt_parameters, opt_background) {
-            return getResource(buildUrl(endpoint['path'], opt_parameters));
+            return getResource(buildUrl(endpoint['path'], opt_parameters), opt_background);
           };
         }
 
         if (endpoint['user_method']) {
-          apiService[getGenericMethodName(endpoint['user_method'])] = function(orgname, opt_options, opt_parameters) {
+          apiService[getGenericMethodName(endpoint['user_method'])] = function(orgname, opt_options, opt_parameters, opt_background) {
             if (orgname) {
               if (orgname.name) {
                 orgname = orgname.name;
               }
 
-              var params = jQuery.extend({'orgname' : orgname}, opt_parameters || {});
+              var params = jQuery.extend({'orgname' : orgname}, opt_parameters || {}, opt_background);
               return apiService[methodName](opt_options, params);
             } else {
-              return apiService[formatMethodName(endpoint['user_method'])](opt_options, opt_parameters);
+              return apiService[formatMethodName(endpoint['user_method'])](opt_options, opt_parameters, opt_background);
             }
           };
         }
@@ -779,6 +790,7 @@ quayApp = angular.module('quay', ['ngRoute', 'chieffancypants.loadingBar', 'angu
                                                      fixFooter: false}).
       when('/repository/:namespace/:name/image/:image', {templateUrl: '/static/partials/image-view.html', controller: ImageViewCtrl, reloadOnSearch: false}).
       when('/repository/:namespace/:name/admin', {templateUrl: '/static/partials/repo-admin.html', controller:RepoAdminCtrl, reloadOnSearch: false}).
+      when('/repository/:namespace/:name/build', {templateUrl: '/static/partials/repo-build.html', controller:RepoBuildCtrl, reloadOnSearch: false}).
       when('/repository/', {title: 'Repositories', description: 'Public and private docker repositories list',
                             templateUrl: '/static/partials/repo-list.html', controller: RepoListCtrl}).
       when('/user/', {title: 'Account Settings', description:'Account settings for Quay.io', templateUrl: '/static/partials/user-admin.html',
@@ -2471,6 +2483,119 @@ quayApp.directive('namespaceSelector', function () {
 });
 
 
+quayApp.directive('buildLogPhase', function () {
+  var directiveDefinitionObject = {
+    priority: 0,
+    templateUrl: '/static/directives/build-log-phase.html',
+    replace: false,
+    transclude: false,
+    restrict: 'C',
+    scope: {
+      'phase': '=phase'
+    },
+    controller: function($scope, $element) {
+    }
+  };
+  return directiveDefinitionObject;
+});
+
+
+quayApp.directive('buildLogError', function () {
+  var directiveDefinitionObject = {
+    priority: 0,
+    templateUrl: '/static/directives/build-log-error.html',
+    replace: false,
+    transclude: false,
+    restrict: 'C',
+    scope: {
+      'error': '=error'
+    },
+    controller: function($scope, $element) {
+    }
+  };
+  return directiveDefinitionObject;
+});
+
+
+quayApp.directive('buildLogCommand', function () {
+  var directiveDefinitionObject = {
+    priority: 0,
+    templateUrl: '/static/directives/build-log-command.html',
+    replace: false,
+    transclude: false,
+    restrict: 'C',
+    scope: {
+      'command': '=command'
+    },
+    controller: function($scope, $element, $sanitize) {
+      var registryHandlers = {
+        'quay.io': function(pieces) {
+          var rnamespace =  pieces[pieces.length - 2];
+          var rname = pieces[pieces.length - 1];
+          return '/repository/' + rnamespace + '/' + rname + '/'; 
+        },
+
+        '': function(pieces) {
+          var rnamespace = pieces.length == 1 ? '_' : pieces[0];
+          var rname = pieces[pieces.length - 1];
+          return 'https://index.docker.io/u/' + rnamespace + '/' + rname + '/';
+        }
+      };
+
+      var kindHandlers = {
+        'FROM': function(title) {
+          var pieces = title.split('/');
+          var registry = pieces.length < 3 ? '' : pieces[0];
+          if (!registryHandlers[registry]) {
+            return title;
+          }
+          
+          return '<i class="fa fa-hdd-o"></i> <a href="' + registryHandlers[registry](pieces) + '">' + title + '</a>';
+        }
+      };
+     
+      $scope.getCommandKind = function(fullTitle) {
+        var colon = fullTitle.indexOf(':');
+        var title = getTitleWithoutStep(fullTitle);
+        if (!title) {
+          return null;
+        }
+
+        var space = title.indexOf(' ');
+        return title.substring(0, space);
+      };
+
+      $scope.getCommandTitleHtml = function(fullTitle) {
+        var title = getTitleWithoutStep(fullTitle) || fullTitle;
+        var space = title.indexOf(' ');
+        if (space <= 0) {
+          return $sanitize(title);
+        }
+        
+        var kind = $scope.getCommandKind(fullTitle);
+        var sanitized = $sanitize(title.substring(space + 1));
+        
+        var handler = kindHandlers[kind || ''];
+        if (handler) {
+          return handler(sanitized);
+        } else {
+          return sanitized;
+        }
+      };
+
+      var getTitleWithoutStep = function(fullTitle) {
+        var colon = fullTitle.indexOf(':');
+        if (colon <= 0) {
+          return null;
+        }
+        
+        return $.trim(fullTitle.substring(colon + 1));
+      };
+    }
+  };
+  return directiveDefinitionObject;
+});
+
 quayApp.directive('buildStatus', function () {
   var directiveDefinitionObject = {
     priority: 0,
@@ -2482,7 +2607,63 @@ quayApp.directive('buildStatus', function () {
       'build': '=build'
     },
     controller: function($scope, $element) {
-        $scope.getBuildProgress = function(buildInfo) {
+    }
+  };
+  return directiveDefinitionObject;
+});
+
+
+quayApp.directive('buildMessage', function () {
+  var directiveDefinitionObject = {
+    priority: 0,
+    templateUrl: '/static/directives/build-message.html',
+    replace: false,
+    transclude: false,
+    restrict: 'C',
+    scope: {
+      'phase': '=phase'
+    },
+    controller: function($scope, $element) {
+      $scope.getBuildMessage = function (phase) {
+        switch (phase) {
+          case 'starting':
+          case 'initializing':
+            return 'Starting Dockerfile build';
+          
+          case 'waiting':
+            return 'Waiting for available build worker';
+          
+          case 'building':
+            return 'Building image from Dockerfile';
+          
+          case 'pushing':
+            return 'Pushing image built from Dockerfile';
+            
+          case 'complete':
+            return 'Dockerfile build completed and pushed';
+          
+          case 'error':
+            return 'Dockerfile build failed';
+        }
+      };
+    }
+  };
+  return directiveDefinitionObject;
+});
+
+
+quayApp.directive('buildProgress', function () {
+  var directiveDefinitionObject = {
+    priority: 0,
+    templateUrl: '/static/directives/build-progress.html',
+    replace: false,
+    transclude: false,
+    restrict: 'C',
+    scope: {
+      'build': '=build'
+    },
+    controller: function($scope, $element) {
+      $scope.getPercentage = function(buildInfo) {
         switch (buildInfo.phase) {
           case 'building':
             return (buildInfo.status.current_command / buildInfo.status.total_commands) * 100;
@@ -2505,32 +2686,6 @@ quayApp.directive('buildStatus', function () {
         
         return -1;
       };
-
-        $scope.getBuildMessage = function(buildInfo) {
-            switch (buildInfo.phase) {
-            case 'initializing':
-                return 'Starting Dockerfile build';
-                break;
-
-            case 'starting':
-            case 'waiting':
-            case 'building':
-                return 'Building image from Dockerfile';
-                break;
-
-            case 'pushing':
-                return 'Pushing image built from Dockerfile';
-                break;
-
-            case 'complete':
-                return 'Dockerfile build completed and pushed';
-                break;
-                
-            case 'error':
-                return 'Dockerfile build failed.';
-                break;
-            }
-        };
     }
   };
   return directiveDefinitionObject;
@@ -2545,6 +2700,13 @@ quayApp.directive('ngBlur', function() {
   };
 });
 
+quayApp.directive('ngVisible', function () {
+  return function (scope, element, attr) {
+    scope.$watch(attr.ngVisible, function (visible) {
+      element.css('visibility', visible ? 'visible' : 'hidden');
+    });
+  };
+});
 
 quayApp.run(['$location', '$rootScope', 'Restangular', 'UserService', 'PlanService', '$http', '$timeout',
     function($location, $rootScope, Restangular, UserService, PlanService, $http, $timeout) {

static/js/controllers.js

@@ -326,6 +326,11 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
 
   $scope.getFormattedCommand = ImageMetadataService.getFormattedCommand;
 
+  $scope.showBuild = function(buildInfo) {
+    $location.path('/repository/' + namespace + '/' + name + '/build');
+    $location.search('current', buildInfo.id);
+  };
+
   $scope.getTooltipCommand = function(image) {
     var sanitized = ImageMetadataService.getEscapedFormattedCommand(image);
     return '<span class=\'codetooltip\'>' + sanitized + '</span>';
@@ -653,13 +658,11 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
   };
 
   var getBuildInfo = function(repo) {
-    // Note: We use restangular manually here because we need to turn off the loading bar.
+    var params = {
-    var buildInfo = Restangular.one('repository/' + repo.namespace + '/' + repo.name + '/build/');
+      'repository': repo.namespace + '/' + repo.name
-    buildInfo.withHttpConfig({
+    };
-      'ignoreLoadingBar': true
-    });
 
-    buildInfo.get().then(function(resp) {
+    ApiService.getRepoBuilds(null, params, true).then(function(resp) {
       var runningBuilds = [];
       for (var i = 0; i < resp.builds.length; ++i) {
         var build = resp.builds[i];
@@ -745,6 +748,197 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
   loadViewInfo();
 }
 
+function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope, $location, $interval, $sanitize) {
+  var namespace = $routeParams.namespace;
+  var name = $routeParams.name;
+  var pollTimerHandle = null;
+
+  $scope.$on('$destroy', function() {
+    stopPollTimer();
+  });
+
+  // Watch for changes to the current parameter.
+  $scope.$on('$routeUpdate', function(){
+    if ($location.search().current) {
+      $scope.setCurrentBuild($location.search().current, false);
+    }
+  });
+
+  $scope.builds = [];
+  $scope.polling = false;
+
+  $scope.adjustLogHeight = function() {
+    $('.build-logs').height($(window).height() - 365);
+  };
+
+  $scope.hasLogs = function(container) {
+    return ((container.logs && container.logs.length) || (container._logs && container._logs.length));
+  };
+
+  $scope.toggleLogs = function(container) {
+    if (container._logs) {
+      container.logs = container._logs;
+      container._logs = null;
+    } else {
+      container._logs = container.logs;
+      container.logs = null;
+    }
+  };  
+
+  $scope.setCurrentBuild = function(buildId, opt_updateURL) {
+    // Find the build.
+    for (var i = 0; i < $scope.builds.length; ++i) {
+      if ($scope.builds[i].id == buildId) {
+        $scope.setCurrentBuildInternal($scope.builds[i], opt_updateURL);
+        return;
+      }
+    }
+  };
+
+  $scope.setCurrentBuildInternal = function(build, opt_updateURL) {
+    if (build == $scope.currentBuild) { return; }
+
+    stopPollTimer();
+
+    $scope.logEntries = null;
+    $scope.logStartIndex = null;
+    $scope.currentParentEntry = null;
+
+    $scope.currentBuild = build;
+
+    if (opt_updateURL) {
+      if (build) {
+        $location.search('current', build.id);
+      } else {
+        $location.search('current', null);
+      }
+    }
+
+    // Timeout needed to ensure the log element has been created
+    // before its height is adjusted.
+    setTimeout(function() {
+      $scope.adjustLogHeight();
+    }, 1);
+
+    // Load the first set of logs.
+    getBuildStatusAndLogs();
+
+    // If the build is currently processing, start the build timer.
+    checkPollTimer();
+  };
+
+  var checkPollTimer = function() {
+    var build = $scope.currentBuild;
+    if (!build) {
+      stopPollTimer();
+      return;
+    }
+
+    if (build['phase'] != 'complete' && build['phase'] != 'error') {
+      startPollTimer();
+      return true;
+    } else {
+      stopPollTimer();
+      return false;
+    }
+  };
+
+  var stopPollTimer = function() {    
+    $interval.cancel(pollTimerHandle);
+  };
+
+  var startPollTimer = function() {
+    stopPollTimer();
+    pollTimerHandle = $interval(getBuildStatusAndLogs, 2000);
+  };
+
+  var processLogs = function(logs, startIndex) {
+    if (!$scope.logEntries) { $scope.logEntries = []; }
+
+    for (var i = 0; i < logs.length; ++i) {
+      var entry = logs[i];
+      var type = entry['type'] || 'entry';
+      if (type == 'command' || type == 'phase' || type == 'error') {
+        entry['_logs'] = [];
+        entry['index'] = startIndex + i;
+
+        $scope.logEntries.push(entry);
+        $scope.currentParentEntry = entry;      
+      } else if ($scope.currentParentEntry) {
+        if ($scope.currentParentEntry['logs']) {
+          $scope.currentParentEntry['logs'].push(entry);
+        } else {
+          $scope.currentParentEntry['_logs'].push(entry);
+        }
+      }
+    }
+  };
+
+  var getBuildStatusAndLogs = function() {
+    if (!$scope.currentBuild || $scope.polling) { return; }
+
+    $scope.polling = true;
+    
+    var params = {
+      'repository': namespace + '/' + name,
+      'build_uuid': $scope.currentBuild.id
+    };
+
+    ApiService.getRepoBuildStatus(null, params, true).then(function(resp) {
+      // Note: We use extend here rather than replacing as Angular is depending on the
+      // root build object to remain the same object.
+      $.extend(true, $scope.currentBuild, resp);
+      checkPollTimer();
+
+      // Load the updated logs for the build.
+      var options = {
+        'start': $scope.logStartIndex
+      };
+
+      ApiService.getRepoBuildLogsAsResource(params, true).withOptions(options).get(function(resp) {       
+        processLogs(resp['logs'], resp['start']);
+        $scope.logStartIndex = resp['total'];
+        $scope.polling = false;
+      });
+    });
+  };
+
+  var fetchRepository = function() {
+    var params = {'repository': namespace + '/' + name};
+    $rootScope.title = 'Loading Repository...';
+    $scope.repository = ApiService.getRepoAsResource(params).get(function(repo) {
+      if (!repo.can_write) {
+        $rootScope.title = 'Unknown builds';
+        $scope.accessDenied = true;
+        return;
+      }
+
+      $rootScope.title = 'Repository Builds';
+      $scope.repo = repo;
+
+      getBuildInfo();
+    });
+  };
+
+  var getBuildInfo = function(repo) {
+    var params = {
+      'repository': namespace + '/' + name
+    };
+
+    ApiService.getRepoBuilds(null, params).then(function(resp) {
+      $scope.builds = resp.builds;
+
+      if ($location.search().current) {
+        $scope.setCurrentBuild($location.search().current, false);
+      } else if ($scope.builds.length > 0) {
+        $scope.setCurrentBuild($scope.builds[0].id, true);
+      }
+    });
+  };
+
+  fetchRepository();
+}
+
 function RepoAdminCtrl($scope, Restangular, ApiService, $routeParams, $rootScope) {
   var namespace = $routeParams.namespace;
   var name = $routeParams.name;
@@ -1002,8 +1196,13 @@ function RepoAdminCtrl($scope, Restangular, ApiService, $routeParams, $rootScope
     };
 
     $scope.repository = ApiService.getRepoAsResource(params).get(function(repo) {
-      $scope.repo = repo;
+      if (!repo.can_admin) {
+        $rootScope.title = 'Forbidden';
+        $scope.accessDenied = true;
+        return;
+      }
 
+      $scope.repo = repo;
       $rootScope.title = 'Settings - ' + namespace + '/' + name;
       $rootScope.description = 'Administrator settings for ' + namespace + '/' + name +
           ': Permissions, webhooks and other settings';

static/partials/repo-admin.html

@@ -1,5 +1,8 @@
 <div class="container" ng-show="deleting"><div class="quay-spinner"></div></div>
 <div class="resource-view" resource="repository" error-message="'No repository found'"></div>
+<div class="container repo repo-admin" ng-show="accessDenied">
+  You do not have permission to view this page
+</div>
 <div class="container repo repo-admin" ng-show="repo && !deleting">
   <div class="header row">
     <a href="{{ '/repository/' + repo.namespace + '/' + repo.name }}" class="back"><i class="fa fa-chevron-left"></i></a>

static/partials/repo-build.html

@@ -0,0 +1,83 @@
+<div class="resource-view" resource="repository" error-message="'No repository found'"></div>
+<div class="container repo repo-build" ng-show="accessDenied">
+  You do not have permission to view this page
+</div>
+<div class="container repo repo-build" ng-show="repo">
+  <div class="header row">
+    <a href="{{ '/repository/' + repo.namespace + '/' + repo.name }}" class="back"><i class="fa fa-chevron-left"></i></a>
+    <h3>
+      <span class="repo-circle no-background" repo="repo"></span>
+      <span class="repo-breadcrumb" repo="repo"></span>
+    </h3>
+  </div>
+
+  <div class="row" ng-show="!builds.length">
+    There are no builds for this repository
+  </div>
+  
+  <div class="row" ng-show="builds.length">
+    <!-- Side tabs -->
+    <div class="col-sm-2">
+      <ul class="nav nav-pills nav-stacked">
+        <li ng-class="currentBuild == build ? 'active' : ''" ng-repeat="build in builds">
+          <a class="build-tab-link" href="javascript:void(0)" ng-click="setCurrentBuild(build.id, true)">
+            <span class="phase-icon" ng-class="build.phase"></span>
+            <span>{{ build.display_name }}</span>
+          </a>
+        </li>
+      </ul>
+    </div>
+
+    <!-- Content -->
+    <div class="col-sm-10">      
+      <div class="tab-content" onresize="adjustLogHeight()">
+        <div ng-repeat="build in builds" class="tab-pane build-pane" ng-class="currentBuild == build ? 'active' : ''">
+          <div class="build-header">
+            <div class="timing">
+              <i class="fa fa-clock-o"></i>
+              Started: <span am-time-ago="build.started || 0"></span>
+            </div>
+            <span class="phase-icon" ng-class="build.phase"></span>
+            <span class="build-message" phase="build.phase"></span>
+            <div class="build-progress" build="build"></div>
+          </div>
+
+          <div class="build-logs">
+            <div ng-show="!logEntries">
+              <span class="quay-spinner"></span>
+            </div>
+
+            <div class="log-container" ng-class="container.type" ng-repeat="container in logEntries">
+              <div class="container-header" ng-class="container.type == 'phase' ? container.message : ''"
+                   ng-switch on="container.type" ng-click="toggleLogs(container)">
+                <i class="fa chevron"
+                   ng-class="container.logs ? 'fa-chevron-down' : 'fa-chevron-right'" ng-show="hasLogs(container)"></i>
+                <div ng-switch-when="phase">
+                  <span class="container-content build-log-phase" phase="container"></span>
+                </div>
+                <div ng-switch-when="error">
+                  <span class="container-content build-log-error" error="container"></span>
+                </div>
+                <div ng-switch-when="command">
+                  <span class="container-content build-log-command" command="container"></span>
+                </div>
+              </div>
+              
+              <!-- Display the entries for the container -->
+              <div class="container-logs" ng-show="container.logs">
+                <div class="log-entry" bindonce ng-repeat="entry in container.logs">
+                  <span class="id" bo-text="$index + container.index + 1"></span>
+                  <span class="message" bo-text="entry.message"></span>
+                </div>
+              </div>
+            </div>
+          </div>
+          <div>
+            <span class="quay-spinner" ng-show="polling"></span>
+            <span class="build-id">{{ build.id }}</span>
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>

static/partials/view-repo.html

@@ -38,13 +38,18 @@
 
     <!-- Status boxes -->
     <div class="status-boxes">
-      <div id="buildInfoBox" class="status-box" ng-show="repo.is_building"
+      <div id="buildInfoBox" class="status-box" ng-show="repo.is_building">
-           bs-popover="'static/partials/build-status-item.html'" data-placement="bottom">
+        <div class="dropdown" data-placement="top">
-        <span class="title">
-          <span class="quay-spinner"></span>
-          <b>Building Images</b>
-        </span>
           <span class="count" ng-class="buildsInfo ? 'visible' : ''"><span>{{ buildsInfo ? buildsInfo.length : '-' }}</span></span>
+          <a href="javascript:void(0)" class="dropdown-toggle" data-toggle="dropdown">Building Dockerfile<span ng-show="buildsInfo.length > 1">s</span> <b class="caret"></b></a>
+          <ul class="dropdown-menu pull-right">     
+            <li ng-repeat="buildInfo in buildsInfo">
+              <div class="build-info" ng-class="repo.can_write ? 'clickable' : ''" ng-click="showBuild(buildInfo)">
+                <span class="build-status" build="buildInfo"></span>
+              </div>
+            </li>
+          </ul>
+        </div>
       </div>
     </div>
 

templates/base.html

@@ -38,14 +38,12 @@
 
     <script src="//code.jquery.com/jquery.js"></script>
     <script src="//netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
-    <script src="//cdnjs.cloudflare.com/ajax/libs/bootbox.js/4.0.0/bootbox.min.js"></script>
 
     <script src="//ajax.googleapis.com/ajax/libs/angularjs/1.2.1/angular.min.js"></script>
     <script src="//ajax.googleapis.com/ajax/libs/angularjs/1.2.1/angular-route.min.js"></script>
     <script src="//ajax.googleapis.com/ajax/libs/angularjs/1.2.1/angular-sanitize.min.js"></script>
 
-    <script src="//cdn.jsdelivr.net/underscorejs/1.5.2/underscore-min.js"></script>
+    <script src="//cdn.jsdelivr.net/g/bootbox@4.1.0,underscorejs@1.5.2,restangular@1.2.0"></script>
-    <script src="//cdn.jsdelivr.net/restangular/1.2.0/restangular.min.js"></script>
 
     <script src="static/lib/loading-bar.js"></script>
     <script src="static/lib/angular-strap.min.js"></script>
@@ -53,6 +51,7 @@
     <script src="static/lib/angulartics-mixpanel.js"></script>
     <script src="static/lib/angulartics-google-analytics.js"></script>
     <script src="static/lib/angular-md5.js"></script>
+    <script src="static/lib/bindonce.min.js"></script>
 
     <script src="static/lib/angular-moment.min.js"></script>
     <script src="static/lib/angular-cookies.min.js"></script>

test/test_api_usage.py

@@ -856,7 +856,6 @@ class TestGetRepoBuilds(ApiTestCase):
 
     assert 'id' in build
     assert 'status' in build
-    assert 'message' in build
 
 
 class TestRequearRepoBuild(ApiTestCase):

test/testlogs.py

@@ -0,0 +1,189 @@
+import logging
+
+from random import SystemRandom
+from loremipsum import get_sentence
+from functools import wraps
+from copy import deepcopy
+
+from data.buildlogs import BuildLogs
+
+
+logger = logging.getLogger(__name__)
+random = SystemRandom()
+
+
+def maybe_advance_script(is_get_status=False):
+  def inner_advance(func):
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+      advance_units = random.randint(1, 500)
+      logger.debug('Advancing script %s units', advance_units)
+      while advance_units > 0 and self.remaining_script:
+        units = self.remaining_script[0][0]
+
+        if advance_units > units:
+          advance_units -= units
+          self.advance_script(is_get_status)
+        else:
+          break
+
+      return func(self, *args, **kwargs)
+    return wrapper
+  return inner_advance
+
+
+class TestBuildLogs(BuildLogs):
+  COMMAND_TYPES = ['FROM', 'MAINTAINER', 'RUN', 'CMD', 'EXPOSE', 'ENV', 'ADD', 
+                   'ENTRYPOINT', 'VOLUME', 'USER', 'WORKDIR']
+  STATUS_TEMPLATE = {
+    'total_commands': None,
+    'current_command': None,
+    'push_completion': 0.0,
+    'image_completion': {},
+  }
+
+  def __init__(self, redis_host, namespace, repository, test_build_id):
+    super(TestBuildLogs, self).__init__(redis_host)
+    self.namespace = namespace
+    self.repository = repository
+    self.test_build_id = test_build_id
+    self.remaining_script = self._generate_script()
+    logger.debug('Total script size: %s', len(self.remaining_script))
+    self._logs = []
+
+    self._status = {}
+    self._last_status = {}
+
+  def advance_script(self, is_get_status):
+    (_, log, status_wrapper) = self.remaining_script.pop(0)
+    if log is not None:
+      self._logs.append(log)
+
+    if status_wrapper is not None:
+      (phase, status) = status_wrapper
+
+      from data import model
+      build_obj = model.get_repository_build(self.namespace, self.repository,
+                                             self.test_build_id)
+      build_obj.phase = phase
+      build_obj.save()
+
+      self._status = status
+      if not is_get_status:
+        self._last_status = status
+
+  def _generate_script(self):
+    script = []
+
+    # generate the init phase
+    script.append(self._generate_phase(400, 'initializing'))
+    script.extend(self._generate_logs(random.randint(1, 3)))
+
+    # move to the building phase
+    script.append(self._generate_phase(400, 'building'))
+    total_commands = random.randint(5, 20)
+    for command_num in range(1, total_commands + 1):
+      command_weight = random.randint(50, 100)
+      script.append(self._generate_command(command_num, total_commands,
+                                           command_weight))
+
+      # we want 0 logs some percent of the time
+      num_logs = max(0, random.randint(-50, 400))
+      script.extend(self._generate_logs(num_logs))
+
+    # move to the pushing phase
+    script.append(self._generate_phase(400, 'pushing'))
+    script.extend(self._generate_push_statuses(total_commands))
+
+    # move to the error or complete phase
+    if random.randint(0, 1) == 0:
+      script.append(self._generate_phase(400, 'complete'))
+    else:
+      script.append(self._generate_phase(400, 'error'))
+      script.append((1, {'message': 'Something bad happened! Oh noes!',
+                         'type': self.ERROR}, None))
+
+    return script
+
+  def _generate_phase(self, start_weight, phase_name):
+    return (start_weight, {'message': phase_name, 'type': self.PHASE},
+            (phase_name, deepcopy(self.STATUS_TEMPLATE)))
+
+  def _generate_command(self, command_num, total_commands, command_weight):
+    sentence = get_sentence()
+    command = random.choice(self.COMMAND_TYPES)
+    if command == 'FROM':
+      sentence = random.choice(['ubuntu', 'lopter/raring-base',
+                                'quay.io/devtable/simple',
+                                'quay.io/buynlarge/orgrepo',
+                                'stackbrew/ubuntu:precise'])
+
+    msg = {
+      'message': 'Step %s: %s %s' % (command_num, command, sentence),
+      'type': self.COMMAND,
+    }
+    status = deepcopy(self.STATUS_TEMPLATE)
+    status['total_commands'] = total_commands
+    status['current_command'] = command_num
+    return (command_weight, msg, ('building', status))
+
+  @staticmethod
+  def _generate_logs(count):
+    return [(1, {'message': get_sentence()}, None) for _ in range(count)]
+
+  @staticmethod
+  def _compute_total_completion(statuses, total_images):
+    percentage_with_sizes = float(len(statuses.values()))/total_images
+    sent_bytes = sum([status[u'current'] for status in statuses.values()])
+    total_bytes = sum([status[u'total'] for status in statuses.values()])
+    return float(sent_bytes)/total_bytes*percentage_with_sizes
+
+  @staticmethod
+  def _generate_push_statuses(total_commands):
+    push_status_template = deepcopy(TestBuildLogs.STATUS_TEMPLATE)
+    push_status_template['current_command'] = total_commands
+    push_status_template['total_commands'] = total_commands
+
+    push_statuses = []
+
+    one_mb = 1 * 1024 * 1024
+
+    num_images = random.randint(2, 7)
+    sizes = [random.randint(one_mb, one_mb * 5) for _ in range(num_images)]
+
+    image_completion = {}
+    for image_num, image_size in enumerate(sizes):
+      image_id = 'image_id_%s' % image_num
+
+      image_completion[image_id] = {
+        'current': 0,
+        'total': image_size,
+      }
+
+      for i in range(one_mb, image_size, one_mb):
+        image_completion[image_id]['current'] = i
+        new_status = deepcopy(push_status_template)
+        new_status['image_completion'] = deepcopy(image_completion)
+
+        completion = TestBuildLogs._compute_total_completion(image_completion,
+                                                             num_images)
+        new_status['push_completion'] = completion
+        push_statuses.append((250, None, ('pushing', new_status)))
+
+    return push_statuses
+
+  @maybe_advance_script()
+  def get_log_entries(self, build_id, start_index):
+    if build_id == self.test_build_id:
+      return (len(self._logs), self._logs[start_index:])
+    else:
+      return super(TestBuildLogs, self).get_log_entries(build_id, start_index)
+
+  @maybe_advance_script(True)
+  def get_status(self, build_id):
+    if build_id == self.test_build_id:
+      returnable_status = self._last_status
+      self._last_status = self._status
+      return returnable_status
+    else:
+      return super(TestBuildLogs, self).get_status(build_id)

test/teststorage.py

@@ -35,3 +35,6 @@ class FakeUserfiles(object):
 
   def get_file_url(self, file_id, expires_in=300):
     return ('http://fake/url')
+
+  def get_file_checksum(self, file_id):
+    return 'abcdefg'

workers/README.md

@@ -0,0 +1,39 @@
+to prepare a new build node host:
+
+```
+sudo apt-get update
+sudo apt-get install -y git python-virtualenv python-dev phantomjs libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev libevent-dev gdebi-core
+```
+
+check out the code, install the kernel, custom docker, nsexec, and reboot:
+
+```
+git clone https://bitbucket.org/yackob03/quay.git
+cd quay
+sudo gdebi --n binary_dependencies/builder/linux-headers-3.11.0-17_3.11.0-17.28_all.deb            
+sudo gdebi --n binary_dependencies/builder/linux-headers-3.11.0-17-generic_3.11.0-17.28_amd64.deb  
+sudo gdebi --n binary_dependencies/builder/linux-image-3.11.0-17-generic_3.11.0-17.28_amd64.deb    
+sudo gdebi --n binary_dependencies/builder/linux-image-extra-3.11.0-17-generic_3.11.0-17.28_amd64.deb
+sudo gdebi --n binary_dependencies/builder/nsexec_1.22ubuntu1trusty1_amd64.deb
+sudo gdebi --n binary_dependencies/builder/lxc-docker-0.8.0-tutum_0.8.0-tutum-20140212002736-afad5c0-dirty_amd64.deb
+sudo chown -R 100000:100000 /var/lib/docker
+sudo shutdown -r now
+```
+
+pull some base images if you want (optional)
+```
+sudo docker pull ubuntu
+sudo docker pull stackbrew/ubuntu
+sudo docker pull busybox
+sudo docker pull lopter/raring-base
+```
+
+start the worker
+
+```
+cd quay
+virtualenv --distribute venv
+source venv/bin/activate
+pip install -r requirements.txt
+sudo STACK=prod venv/bin/python -m workers.dockerfilebuild -D
+```

workers/dockerfilebuild.py

@@ -10,6 +10,7 @@ import shutil
 from docker import Client, APIError
 from tempfile import TemporaryFile, mkdtemp
 from zipfile import ZipFile
+from functools import partial
 
 from data.queue import dockerfile_build_queue
 from data import model
@@ -53,9 +54,9 @@ class DockerfileBuildContext(object):
     self._build_dir = build_context_dir
     self._tag_name = tag_name
     self._push_token = push_token
-    self._build_uuid = build_uuid
+    self._cl = Client(timeout=1200, version='1.7')
-    self._cl = Client(timeout=1200)
+    self._status = StatusWrapper(build_uuid)
-    self._status = StatusWrapper(self._build_uuid)
+    self._build_logger = partial(build_logs.append_log_message, build_uuid)
 
     dockerfile_path = os.path.join(self._build_dir, "Dockerfile")
     self._num_steps = DockerfileBuildContext.__count_steps(dockerfile_path)
@@ -93,22 +94,25 @@ class DockerfileBuildContext(object):
     with self._status as status:
       status['total_commands'] = self._num_steps
 
-    logger.debug('Building to tag names: %s' % self._tag_name)
+    logger.debug('Building to tag named: %s' % self._tag_name)
     build_status = self._cl.build(path=self._build_dir, tag=self._tag_name,
                                   stream=True)
 
     current_step = 0
     built_image = None
     for status in build_status:
-      logger.debug('Status: %s', str(status))
+      status_str = str(status.encode('utf-8'))
-      build_logs.append_log_message(self._build_uuid, str(status))
+      logger.debug('Status: %s', status_str)
       step_increment = re.search(r'Step ([0-9]+) :', status)
       if step_increment:
+        self._build_logger(status_str, build_logs.COMMAND)
         current_step = int(step_increment.group(1))
         logger.debug('Step now: %s/%s' % (current_step, self._num_steps))
         with self._status as status:
           status['current_command'] = current_step
         continue
+      else:
+        self._build_logger(status_str)
 
       complete = re.match(r'Successfully built ([a-z0-9]+)$', status)
       if complete:
@@ -189,7 +193,11 @@ class DockerfileBuildContext(object):
     repos = set()
     for image in self._cl.images():
       images_to_remove.add(image['Id'])
-      repos.add(image['Repository'])
+
+      for tag in image['RepoTags']:
+        tag_repo = tag.split(':')[0]
+        if tag_repo != '<none>':
+          repos.add(tag_repo)
 
     for repo in repos:
       repo_url = 'https://index.docker.io/v1/repositories/%s/images' % repo
@@ -254,10 +262,15 @@ class DockerfileBuildWorker(Worker):
     tag_name = repository_build.tag
     access_token = repository_build.access_token.code
 
-    start_msg = ('Starting job with resource url: %s tag: %s and token: %s' %
+    log_appender = partial(build_logs.append_log_message,
-                 (resource_url, tag_name, access_token))
+                           repository_build.uuid)
+
+    log_appender('initializing', build_logs.PHASE)
+
+    start_msg = ('Starting job with resource url: %s tag: %s' % (resource_url,
+                                                                 tag_name))
     logger.debug(start_msg)
-    build_logs.append_log_message(repository_build.uuid, start_msg)
+    log_appender(start_msg)
 
     docker_resource = requests.get(resource_url)
     c_type = docker_resource.headers['content-type']
@@ -265,40 +278,44 @@ class DockerfileBuildWorker(Worker):
     filetype_msg = ('Request to build file of type: %s with tag: %s' %
                     (c_type, tag_name))
     logger.info(filetype_msg)
-    build_logs.append_log_message(repository_build.uuid, filetype_msg)
+    log_appender(filetype_msg)
 
     if c_type not in self._mime_processors:
       raise RuntimeError('Invalid dockerfile content type: %s' % c_type)
 
     build_dir = self._mime_processors[c_type](docker_resource)
-    uuid = repository_build.uuid
+    log_appender('building', build_logs.PHASE)
     repository_build.phase = 'building'
     repository_build.save()
 
-    try:
     with DockerfileBuildContext(build_dir, tag_name, access_token,
                                 repository_build.uuid) as build_ctxt:
+      try:
         built_image = build_ctxt.build()
 
         if not built_image:
+          log_appender('error', build_logs.PHASE)
           repository_build.phase = 'error'
           repository_build.save()
-          build_logs.append_log_message(uuid, 'Unable to build dockerfile.')
+          log_appender('Unable to build dockerfile.', build_logs.ERROR)
           return False
 
+        log_appender('pushing', build_logs.PHASE)
         repository_build.phase = 'pushing'
         repository_build.save()
 
         build_ctxt.push(built_image)
 
+        log_appender('complete', build_logs.PHASE)
         repository_build.phase = 'complete'
         repository_build.save()
 
       except Exception as exc:
+        log_appender('error', build_logs.PHASE)
         logger.exception('Exception when processing request.')
         repository_build.phase = 'error'
         repository_build.save()
-      build_logs.append_log_message(uuid, exc.message)
+        log_appender(str(exc), build_logs.ERROR)
         return False
 
     return True