Merge branch 'master' into redalert

.dockerignore

@@ -0,0 +1,11 @@
+conf/stack
+screenshots
+test/data/registry
+venv
+.git
+.gitignore
+Bobfile
+README.md
+license.py
+requirements-nover.txt
+run-local.sh

Dockerfile.buildworker

@@ -1,45 +1,30 @@
-FROM phusion/baseimage:0.9.10
+FROM phusion/baseimage:0.9.11
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV HOME /root
+WORKDIR /
 
-RUN apt-get update  # 20JUN2014
+# Install the dependencies.
-RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62-dev libevent-dev gdebi-core g++ libmagic1
+RUN apt-get update  # 15JUL2014
+
+# New ubuntu packages should be added as their own apt-get install lines below the existing install commands
+RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62-dev libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap2-dev libsasl2-dev
+
+# Build the python dependencies
+ADD requirements.txt requirements.txt
+RUN virtualenv --distribute venv
+RUN venv/bin/pip install -r requirements.txt
 
 ### End common section ###
 
-RUN apt-get install -y libldap2-dev libsasl2-dev
-
 RUN apt-get install -y lxc aufs-tools
 
 RUN usermod -v 100000-200000 -w 100000-200000 root
 
 ADD binary_dependencies/builder binary_dependencies/builder
-
 RUN gdebi --n binary_dependencies/builder/*.deb
 
-RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+ADD . .
-
-ADD requirements.txt requirements.txt
-RUN virtualenv --distribute venv
-RUN venv/bin/pip install -r requirements.txt
-
-ADD buildstatus buildstatus
-ADD data data
-ADD features features
-ADD storage storage
-ADD util util
-ADD workers workers
-
-ADD app.py app.py
-ADD config.py config.py
-ADD license.pyc license.pyc
-
-# Remove this if we ever stop depending on test data for the default config
-ADD test test
-
-ADD conf conf
-RUN rm -rf /conf/stack
 
 ADD conf/init/svlogd_config /svlogd_config
 ADD conf/init/preplogsdir.sh /etc/my_init.d/

Dockerfile.web

@@ -1,67 +1,35 @@
-FROM phusion/baseimage:0.9.10
+FROM phusion/baseimage:0.9.11
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV HOME /root
+WORKDIR /
 
 # Install the dependencies.
-RUN apt-get update  # 20JUN2014
+RUN apt-get update  # 15JUL2014
 
 # New ubuntu packages should be added as their own apt-get install lines below the existing install commands
-RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62-dev libevent-dev gdebi-core g++ libmagic1
+RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62-dev libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap2-dev libsasl2-dev
-
-# PhantomJS
-RUN apt-get install -y phantomjs
-
-# Grunt
-RUN apt-get install -y nodejs npm
-RUN ln -s /usr/bin/nodejs /usr/bin/node
-RUN npm install -g grunt-cli
-
-# LDAP
-RUN apt-get install -y libldap2-dev libsasl2-dev
-
-ADD binary_dependencies binary_dependencies
-RUN gdebi --n binary_dependencies/*.deb
-
-RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
+# Build the python dependencies
 ADD requirements.txt requirements.txt
 RUN virtualenv --distribute venv
 RUN venv/bin/pip install -r requirements.txt
 
-# Add the static assets and run grunt
+# Install the binary dependencies
-ADD grunt grunt
+ADD binary_dependencies binary_dependencies
-ADD static static
+RUN gdebi --n binary_dependencies/*.deb
+
+# Grunt
+RUN ln -s /usr/bin/nodejs /usr/bin/node
+RUN npm install -g grunt-cli
+
+# Add all of the files!
+ADD . .
+
+# Run grunt
 RUN cd grunt && npm install
 RUN cd grunt && grunt
 
-# Add the backend assets
-ADD auth auth
-ADD buildstatus buildstatus
-ADD data data
-ADD endpoints endpoints
-ADD features features
-ADD screenshots screenshots
-ADD storage storage
-ADD templates templates
-ADD util util
-ADD workers workers
-
-ADD license.pyc license.pyc
-ADD app.py app.py
-ADD application.py application.py
-ADD config.py config.py
-ADD initdb.py initdb.py
-ADD external_libraries.py external_libraries.py
-ADD alembic.ini alembic.ini
-
-# Add the config
-ADD conf conf
-
-# This command must be rm -f (not -rf) to fail in case stack is ever a dir,
-# which may contain secrets
-RUN rm -f /conf/stack
-
 ADD conf/init/svlogd_config /svlogd_config
 ADD conf/init/preplogsdir.sh /etc/my_init.d/
 ADD conf/init/runmigration.sh /etc/my_init.d/
@@ -72,16 +40,12 @@ ADD conf/init/diffsworker /etc/service/diffsworker
 ADD conf/init/webhookworker /etc/service/webhookworker
 
 # Download any external libs.
-RUN mkdir static/fonts
+RUN mkdir static/fonts static/ldn
-RUN mkdir static/ldn
-
 RUN venv/bin/python -m external_libraries
 
-# Add the tests last because they're prone to accidental changes, then run them
+# Run the tests
-ADD test test
 RUN TEST=true venv/bin/python -m unittest discover
 
-RUN rm -rf /conf/stack
 VOLUME ["/conf/stack", "/var/log", "/datastorage"]
 
 EXPOSE 443 80

data/model/legacy.py

@@ -1413,12 +1413,14 @@ def create_delegate_token(namespace_name, repository_name, friendly_name,
 
 
 def get_repository_delegate_tokens(namespace_name, repository_name):
-  selected = AccessToken.select(AccessToken, Role)
+  return (AccessToken.select(AccessToken, Role)
-  with_repo = selected.join(Repository)
+    .join(Repository)
-  with_role = with_repo.switch(AccessToken).join(Role)
+    .switch(AccessToken)
-  return with_role.where(Repository.name == repository_name,
+    .join(Role)
-                         Repository.namespace == namespace_name,
+    .switch(AccessToken)
-                         AccessToken.temporary == False)
+    .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER)
+    .where(Repository.name == repository_name, Repository.namespace == namespace_name,
+           AccessToken.temporary == False, RepositoryBuildTrigger.uuid >> None))
 
 
 def get_repo_delegate_token(namespace_name, repository_name, code):
@@ -1446,7 +1448,7 @@ def set_repo_delegate_token_role(namespace_name, repository_name, code, role):
 
 def delete_delegate_token(namespace_name, repository_name, code):
   token = get_repo_delegate_token(namespace_name, repository_name, code)
-  token.delete_instance()
+  token.delete_instance(recursive=True)
   return token
 
 

endpoints/api/repository.py

@@ -138,21 +138,6 @@ class RepositoryList(ApiResource):
 
     return response
 
-def image_view(image):
-  extended_props = image
-  if image.storage and image.storage.id:
-    extended_props = image.storage
-
-  command = extended_props.command
-  return {
-    'id': image.docker_image_id,
-    'created': format_date(extended_props.created),
-    'comment': extended_props.comment,
-    'command': json.loads(command) if command else None,
-    'ancestors': image.ancestors,
-    'dbid': image.id,
-    'size': extended_props.image_size
-  }
 
 @resource('/v1/repository/<repopath:repository>')
 class Repository(RepositoryParamResource):
@@ -181,13 +166,10 @@ class Repository(RepositoryParamResource):
     logger.debug('Get repo: %s/%s' % (namespace, repository))
 
     def tag_view(tag):
-      image = model.get_tag_image(namespace, repository, tag.name)
-      if not image:
-        return {}
-
       return {
         'name': tag.name,
-        'image': image_view(image),
+        'image_id': tag.image.docker_image_id,
+        'dbid': tag.image.id
       }
 
     organization = None

endpoints/api/repotoken.py

@@ -60,7 +60,7 @@ class RepositoryTokenList(RepositoryParamResource):
 
     log_action('add_repo_accesstoken', namespace,
                {'repo': repository, 'token': token_params['friendlyName']},
-               repo = model.get_repository(namespace, repository))
+               repo=model.get_repository(namespace, repository))
 
     return token_view(token), 201
 
@@ -116,7 +116,7 @@ class RepositoryToken(RepositoryParamResource):
     log_action('change_repo_permission', namespace,
                {'repo': repository, 'token': token.friendly_name, 'code': code,
                 'role': new_permission['role']},
-               repo = model.get_repository(namespace, repository))
+               repo=model.get_repository(namespace, repository))
 
     return token_view(token)
 
@@ -129,6 +129,6 @@ class RepositoryToken(RepositoryParamResource):
     log_action('delete_repo_accesstoken', namespace,
                {'repo': repository, 'token': token.friendly_name,
                 'code': code},
-               repo = model.get_repository(namespace, repository))
+               repo=model.get_repository(namespace, repository))
 
     return 'Deleted', 204

endpoints/api/trigger.py

@@ -81,6 +81,9 @@ class BuildTrigger(RepositoryParamResource):
                   'service': trigger.service.name, 'config': config_dict},
                  repo=model.get_repository(namespace, repository))
 
+    if trigger.write_token is not None:
+      trigger.write_token.delete_instance()
+
     trigger.delete_instance(recursive=True)
     return 'No Content', 204
 

requirements-nover.txt

@@ -8,19 +8,17 @@ python-dateutil
 boto
 pymysql
 stripe
-gunicorn
+gunicorn<19.0
 gevent
 mixpanel-py
 beautifulsoup4
 marisa-trie
 apscheduler
-python-daemon
 paramiko
-python-digitalocean
 xhtml2pdf
 redis
 hiredis
-git+https://github.com/DevTable/docker-py.git
+docker-py
 loremipsum
 pygithub
 flask-restful

requirements.txt

@@ -1,56 +1,54 @@
 APScheduler==2.1.2
 Flask==0.10.1
-Flask-Login==0.2.10
+Flask-Login==0.2.11
 Flask-Mail==0.9.0
 Flask-Principal==0.4.0
 Flask-RESTful==0.2.12
-Jinja2==2.7.2
+Jinja2==2.7.3
 LogentriesLogger==0.2.1
-Mako==0.9.1
+Mako==1.0.0
-MarkupSafe==0.21
+MarkupSafe==0.23
-Pillow==2.4.0
+Pillow==2.5.1
-PyGithub==1.24.1
+PyGithub==1.25.0
 PyMySQL==0.6.2
-PyPDF2==1.21
+PyPDF2==1.22
-SQLAlchemy==0.9.4
+SQLAlchemy==0.9.6
 Unidecode==0.04.16
-Werkzeug==0.9.4
+Werkzeug==0.9.6
-alembic==0.6.4
+alembic==0.6.5
 aniso8601==0.82
 argparse==1.2.1
 beautifulsoup4==4.3.2
 blinker==1.3
-boto==2.27.0
+boto==2.31.1
-git+https://github.com/DevTable/docker-py.git
+docker-py==0.3.2
 ecdsa==0.11
 gevent==1.0.1
 greenlet==0.4.2
 gunicorn==18.0
-hiredis==0.1.3
+hiredis==0.1.4
 html5lib==0.999
 itsdangerous==0.24
 jsonschema==2.3.0
-lockfile==0.9.1
 loremipsum==1.0.2
 marisa-trie==0.6
-mixpanel-py==3.1.2
+mixpanel-py==3.1.3
+mock==1.0.1
 git+https://github.com/NateFerrero/oauth2lib.git
-paramiko==1.13.0
+paramiko==1.14.0
-peewee==2.2.3
+peewee==2.2.5
 py-bcrypt==0.4
 pycrypto==2.6.1
-python-daemon==1.6
 python-dateutil==2.2
-python-digitalocean==0.7
 python-ldap==2.4.15
 python-magic==0.4.6
-pytz==2014.2
+pytz==2014.4
-raven==4.2.1
+raven==5.0.0
-redis==2.9.1
+redis==2.10.1
 reportlab==2.7
-requests==2.2.1
+requests==2.3.0
-six==1.6.1
+six==1.7.3
-stripe==1.14.0
+stripe==1.18.0
 websocket-client==0.11.0
 wsgiref==0.1.2
 xhtml2pdf==0.0.6

static/directives/tag-specific-images-view.html

@@ -5,7 +5,7 @@
          ng-class="getImageListingClasses(image)">
       <span class="image-listing-circle"></span>
       <span class="image-listing-line"></span>
-      <span class="context-tooltip image-listing-id" bs-tooltip="" data-title="getFirstTextLine(image.comment)"
+      <span class="context-tooltip image-listing-id" bs-tooltip="" data-title="{{ getFirstTextLine(image.comment) }}"
             data-html="true">
         {{ image.id.substr(0, 12) }}
       </span>

static/js/app.js

@@ -4478,7 +4478,7 @@ quayApp.directive('dockerfileCommand', function () {
         },
 
         '': function(pieces) {
-          var rnamespace = pieces.length == 1 ? '_' : pieces[0];
+          var rnamespace = pieces.length == 1 ? '_' : 'u/' + pieces[0];
           var rname = pieces[pieces.length - 1].split(':')[0];
           return 'https://registry.hub.docker.com/' + rnamespace + '/' + rname + '/';
         }
@@ -4550,7 +4550,7 @@ quayApp.directive('dockerfileView', function () {
           }
 
           var lineInfo = {
-            'text': UtilService.textToSafeHtml(line),
+            'text': line,
             'kind': kind
           };
           $scope.lines.push(lineInfo);
@@ -5224,7 +5224,7 @@ quayApp.directive('tagSpecificImagesView', function () {
         }
  
         var currentTag = $scope.repository.tags[$scope.tag];
-        if (image.dbid == currentTag.image.dbid) {
+        if (image.dbid == currentTag.dbid) {
           classes += 'tag-image ';
         }
         
@@ -5234,8 +5234,6 @@ quayApp.directive('tagSpecificImagesView', function () {
       var forAllTagImages = function(tag, callback) {
         if (!tag) { return; }
 
-        callback(tag.image);
-
         if (!$scope.imageByDBID) {
           $scope.imageByDBID = [];
           for (var i = 0; i < $scope.images.length; ++i) {
@@ -5244,7 +5242,14 @@ quayApp.directive('tagSpecificImagesView', function () {
           }
         }
 
-        var ancestors = tag.image.ancestors.split('/');
+        var tag_image = $scope.imageByDBID[tag.dbid];
+        if (!tag_image) {
+          return;
+        }
+        
+        callback(tag_image);
+
+        var ancestors = tag_image.ancestors.split('/');
         for (var i = 0; i < ancestors.length; ++i) {
           var image = $scope.imageByDBID[ancestors[i]];
           if (image) {

static/js/controllers.js

@@ -450,6 +450,8 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
   };
 
   $scope.loadImageChanges = function(image) {
+    if (!image) { return; }
+
     var params = {'repository': namespace + '/' + name, 'image_id': image.id};
     $scope.currentImageChangeResource = ApiService.getImageChangesAsResource(params).get(function(ci) {
       $scope.currentImageChanges = ci;
@@ -466,31 +468,6 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
       addedDisplayed - removedDisplayed - changedDisplayed;
   };
 
-  $scope.setImage = function(imageId, opt_updateURL) {
-    var image = null;
-    for (var i = 0; i < $scope.images.length; ++i) {
-      var currentImage = $scope.images[i];
-      if (currentImage.id == imageId || currentImage.id.substr(0, 12) == imageId) {
-        image = currentImage;
-        break;
-      }
-    }
-
-    if (!image) { return; }
-
-    $scope.currentTag = null;
-    $scope.currentImage = image;
-    $scope.loadImageChanges(image);
-    if ($scope.tree) {
-      $scope.tree.setImage(image.id);
-    }
-
-    if (opt_updateURL) {
-      $location.search('tag', null);
-      $location.search('image', imageId.substr(0, 12));
-    }
-  };  
-
   $scope.showAddTag = function(image) {
     $scope.toTagImage = image;
     $('#addTagModal').modal('show');
@@ -513,6 +490,10 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
     $('#confirmdeleteTagModal').modal('show');
   };
 
+  $scope.findImageForTag = function(tag) {
+    return tag && $scope.imageByDBID && $scope.imageByDBID[tag.dbid];
+  };
+
   $scope.createOrMoveTag = function(image, tagName, opt_invalid) {
     if (opt_invalid) { return; }
 
@@ -592,13 +573,38 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
     return size;
   };
 
+  $scope.setImage = function(imageId, opt_updateURL) {
+    var image = null;
+    for (var i = 0; i < $scope.images.length; ++i) {
+      var currentImage = $scope.images[i];
+      if (currentImage.id == imageId || currentImage.id.substr(0, 12) == imageId) {
+        image = currentImage;
+        break;
+      }
+    }
+
+    if (!image) { return; }
+
+    $scope.currentTag = null;
+    $scope.currentImage = image;
+    $scope.loadImageChanges(image);
+    if ($scope.tree) {
+      $scope.tree.setImage(image.id);
+    }
+
+    if (opt_updateURL) {
+      $location.search('tag', null);
+      $location.search('image', imageId.substr(0, 12));
+    }
+  };
+
   $scope.setTag = function(tagName, opt_updateURL) {
     var repo = $scope.repo;
     if (!repo) { return; }
 
     var proposedTag = repo.tags[tagName];
     if (!proposedTag) {
-      // We must find a good default
+      // We must find a good default.
       for (tagName in repo.tags) {
         if (!proposedTag || tagName == 'latest') {
           proposedTag = repo.tags[tagName];          
@@ -608,8 +614,8 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
 
     if (proposedTag) {
       $scope.currentTag = proposedTag;
-      $scope.currentImage = proposedTag.image;
+      $scope.currentImage = null;
-      $scope.loadImageChanges($scope.currentImage);
+
       if ($scope.tree) {
         $scope.tree.setTag(proposedTag.name);
       } 
@@ -686,9 +692,15 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi
   var forAllTagImages = function(tag, callback) {
     if (!tag || !$scope.imageByDBID) { return; }
 
-    callback(tag.image);
+    var tag_image =  $scope.imageByDBID[tag.dbid];
+    if (!tag_image) { return; }
 
-    var ancestors = tag.image.ancestors.split('/');
+    // Callback the tag's image itself.
+    callback(tag_image);
+
+    // Callback any parent images.
+    if (!tag_image.ancestors) { return; }
+    var ancestors = tag_image.ancestors.split('/');
     for (var i = 0; i < ancestors.length; ++i) {
       var image = $scope.imageByDBID[ancestors[i]];
       if (image) {

static/partials/view-repo.html

@@ -77,7 +77,7 @@
          content-changed="updateForDescription" field-title="'repository description'"></div>
 
     <!-- Empty message -->
-    <div class="repo-content" ng-show="!currentTag.image && !currentImage && !repo.is_building">
+    <div class="repo-content" ng-show="!currentTag.image_id && !currentImage && !repo.is_building">
       <div class="empty-message">
         This repository is empty
       </div>
@@ -100,14 +100,14 @@
 
     </div>
 
-    <div class="repo-content" ng-show="!currentTag.image && repo.is_building">
+    <div class="repo-content" ng-show="!currentTag.image_id && repo.is_building">
       <div class="empty-message">
         A build is currently processing. If this takes longer than an hour, please <a href="/contact">contact us</a>
       </div>
     </div>
 
     <!-- Content view -->
-    <div class="repo-content" ng-show="currentTag.image || currentImage">
+    <div class="repo-content" ng-show="currentTag.image_id || currentImage">
       <!-- Image History -->
       <div id="image-history" style="max-height: 10px;">
         <div class="row">
@@ -163,7 +163,14 @@
                 <div id="current-tag" ng-show="currentTag">
                   <dl class="dl-normal">
                     <dt>Last Modified</dt>
-                    <dd am-time-ago="parseDate(currentTag.image.created)"></dd>
+                    <dd ng-if="!findImageForTag(currentTag, images)">
+                      <span class="quay-spinner"></span>
+                    </dd>
+
+                    <dd am-time-ago="parseDate(findImageForTag(currentTag, images).created)"
+                        ng-if="findImageForTag(currentTag, images)">
+                    </dd>
+
                     <dt>Total Compressed Size</dt>
                     <dd><span class="context-tooltip"
                               data-title="The amount of data sent between Docker and Quay.io when pushing/pulling"

workers/dockerfilebuild.py

@@ -3,7 +3,6 @@ import logging.config
 logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False)  
 
 import logging
-import daemon
 import argparse
 import os
 import requests

workers/worker.py

@@ -40,6 +40,9 @@ class WorkerStatusHandler(BaseHTTPRequestHandler):
       # Return the worker status
       code = 200 if self.server.worker.is_healthy() else 503
       self.send_response(code)
+      self.send_header('Content-Type', 'text/plain')
+      self.end_headers()
+      self.wfile.write('OK')
     elif self.path == '/terminate':
       # Return whether it is safe to terminate the worker process
       code = 200 if self.server.worker.is_terminated() else 503