Merge branch 'grunt-js-folder' of https://github.com/coreos-inc/quay into ackbar

app.py

@@ -4,30 +4,31 @@ import json
 
 from flask import Flask, Config, request, Request
 from flask.ext.principal import Principal
-from flask.ext.login import LoginManager
+from flask.ext.login import LoginManager, UserMixin
 from flask.ext.mail import Mail
 
 import features
 
 from storage import Storage
+
+from avatars.avatars import Avatar
+
 from data import model
 from data import database
 from data.userfiles import Userfiles
 from data.users import UserAuthentication
+from data.billing import Billing
 from data.buildlogs import BuildLogs
 from data.archivedlogs import LogArchive
-from data.queue import WorkQueue
 from data.userevent import UserEventsBuilderModule
-
-from avatars.avatars import Avatar
+from data.queue import WorkQueue
 
 from util.analytics import Analytics
-from data.billing import Billing
-from util.config.provider import FileConfigProvider, TestConfigProvider
 from util.exceptionlog import Sentry
 from util.names import urn_generator
 from util.oauth import GoogleOAuthConfig, GithubOAuthConfig
 from util.queuemetrics import QueueMetrics
+from util.config.provider import FileConfigProvider, TestConfigProvider
 from util.config.configutil import generate_secret_key
 from util.config.superusermanager import SuperUserManager
 
@@ -103,17 +104,18 @@ analytics = Analytics(app)
 billing = Billing(app)
 sentry = Sentry(app)
 build_logs = BuildLogs(app)
-queue_metrics = QueueMetrics(app)
 authentication = UserAuthentication(app)
 userevents = UserEventsBuilderModule(app)
 superusers = SuperUserManager(app)
+queue_metrics = QueueMetrics(app)
+
+tf = app.config['DB_TRANSACTION_FACTORY']
 
 github_login = GithubOAuthConfig(app.config, 'GITHUB_LOGIN_CONFIG')
 github_trigger = GithubOAuthConfig(app.config, 'GITHUB_TRIGGER_CONFIG')
 google_login = GoogleOAuthConfig(app.config, 'GOOGLE_LOGIN_CONFIG')
 oauth_apps = [github_login, github_trigger, google_login]
 
-tf = app.config['DB_TRANSACTION_FACTORY']
 image_diff_queue = WorkQueue(app.config['DIFFS_QUEUE_NAME'], tf)
 dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf,
                                    reporter=queue_metrics.report)
@@ -128,5 +130,29 @@ if app.config['SECRET_KEY'] is None:
   logger.debug('Generating in-memory secret key')
   app.config['SECRET_KEY'] = generate_secret_key()
 
+@login_manager.user_loader
+def load_user(user_uuid):
+  logger.debug('User loader loading deferred user with uuid: %s' % user_uuid)
+  return LoginWrappedDBUser(user_uuid)
+
+class LoginWrappedDBUser(UserMixin):
+  def __init__(self, user_uuid, db_user=None):
+    self._uuid = user_uuid
+    self._db_user = db_user
+
+  def db_user(self):
+    if not self._db_user:
+      self._db_user = model.get_user_by_uuid(self._uuid)
+    return self._db_user
+
+  def is_authenticated(self):
+    return self.db_user() is not None
+
+  def is_active(self):
+    return self.db_user().verified
+
+  def get_id(self):
+    return unicode(self._uuid)
+
 def get_app_url():
   return '%s://%s' % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'])

auth/permissions.py

@@ -89,6 +89,8 @@ class QuayDeferredPermissionUser(Identity):
     if not self._permissions_loaded:
       logger.debug('Loading user permissions after deferring.')
       user_object = model.get_user_by_uuid(self.id)
+      if user_object is None:
+        return super(QuayDeferredPermissionUser, self).can(permission)
 
       if user_object is None:
         return super(QuayDeferredPermissionUser, self).can(permission)

conf/hosted-http-base.conf

@@ -1,3 +1,5 @@
+# vim: ft=nginx
+
 server {
     listen 80 default_server;
     server_name _;

conf/http-base.conf

@@ -1,3 +1,5 @@
+# vim: ft=nginx
+
 types_hash_max_size 2048;
 include /usr/local/nginx/conf/mime.types.default;
 
@@ -30,4 +32,4 @@ upstream build_manager_controller_server {
 
 upstream build_manager_websocket_server {
     server localhost:8787;
-}
+}

conf/nginx-nossl.conf

@@ -1,8 +1,12 @@
+# vim: ft=nginx
+
 include root-base.conf;
 
 http {
     include http-base.conf;
 
+    include rate-limiting.conf;
+
     server {
         include server-base.conf;
 

conf/nginx.conf

@@ -1,3 +1,5 @@
+# vim: ft=nginx
+
 include root-base.conf;
 
 http {
@@ -5,6 +7,8 @@ http {
 
     include hosted-http-base.conf;
 
+    include rate-limiting.conf;
+
     server {
         include server-base.conf;
 

conf/rate-limiting.conf

@@ -0,0 +1,6 @@
+# vim: ft=nginx
+
+limit_req_zone $binary_remote_addr zone=webapp:10m rate=10r/s;
+limit_req_zone $binary_remote_addr zone=api:10m rate=1r/s;
+limit_req_status 429;
+limit_req_log_level warn;

conf/root-base.conf

@@ -1,3 +1,5 @@
+# vim: ft=nginx
+
 pid /tmp/nginx.pid;
 error_log /var/log/nginx/nginx.error.log;
 

conf/server-base.conf

@@ -1,3 +1,5 @@
+# vim: ft=nginx
+
 client_body_temp_path /var/log/nginx/client_body 1 2;
 server_name _;
 
@@ -19,6 +21,8 @@ proxy_set_header Transfer-Encoding $http_transfer_encoding;
 
 location / {
     proxy_pass   http://web_app_server;
+
+    #limit_req zone=webapp burst=10 nodelay;
 }
 
 location /realtime {
@@ -37,6 +41,8 @@ location /v1/ {
     proxy_temp_path /var/log/nginx/proxy_temp 1 2;
 
     client_max_body_size 20G;
+
+    #limit_req zone=api burst=5 nodelay;
 }
 
 location /c1/ {
@@ -47,6 +53,8 @@ location /c1/ {
     proxy_pass   http://verbs_app_server;
     proxy_read_timeout 2000;
     proxy_temp_path /var/log/nginx/proxy_temp 1 2;
+
+    #limit_req zone=api burst=5 nodelay;
 }
 
 location /static/ {

data/database.py

@@ -70,13 +70,14 @@ read_slave = Proxy()
 db_random_func = CallableProxy()
 
 
-def validate_database_url(url):
+def validate_database_url(url, connect_timeout=5):
   driver = _db_from_url(url, {
-    'connect_timeout': 5
+    'connect_timeout': connect_timeout
   })
   driver.connect()
   driver.close()
 
+
 def _db_from_url(url, db_kwargs):
   parsed_url = make_url(url)
 
@@ -89,6 +90,10 @@ def _db_from_url(url, db_kwargs):
   if parsed_url.password:
     db_kwargs['password'] = parsed_url.password
 
+  # Note: sqlite does not support connect_timeout.
+  if parsed_url.drivername == 'sqlite' and 'connect_timeout' in db_kwargs:
+    del db_kwargs['connect_timeout']
+
   return SCHEME_DRIVERS[parsed_url.drivername](parsed_url.database, **db_kwargs)
 
 
@@ -129,8 +134,9 @@ def close_db_filter(_):
 
 
 class QuayUserField(ForeignKeyField):
-  def __init__(self, allows_robots=False, *args, **kwargs):
+  def __init__(self, allows_robots=False, robot_null_delete=False, *args, **kwargs):
     self.allows_robots = allows_robots
+    self.robot_null_delete = robot_null_delete
     if not 'rel_model' in kwargs:
       kwargs['rel_model'] = User
 
@@ -164,7 +170,11 @@ class User(BaseModel):
       for query, fk in self.dependencies(search_nullable=True):
         if isinstance(fk, QuayUserField) and fk.allows_robots:
           model = fk.model_class
-          model.delete().where(query).execute()
+
+          if fk.robot_null_delete:
+            model.update(**{fk.name: None}).where(query).execute()
+          else:
+            model.delete().where(query).execute()
 
       # Delete the instance itself.
       super(User, self).delete_instance(recursive=False, delete_nullable=False)
@@ -466,7 +476,7 @@ class LogEntry(BaseModel):
   kind = ForeignKeyField(LogEntryKind, index=True)
   account = QuayUserField(index=True, related_name='account')
   performer = QuayUserField(allows_robots=True, index=True, null=True,
-                            related_name='performer')
+                            related_name='performer', robot_null_delete=True)
   repository = ForeignKeyField(Repository, index=True, null=True)
   datetime = DateTimeField(default=datetime.now, index=True)
   ip = CharField(null=True)

data/model/legacy.py

@@ -14,7 +14,7 @@ from data.database import (User, Repository, Image, AccessToken, Role, Repositor
                            ExternalNotificationEvent, ExternalNotificationMethod,
                            RepositoryNotification, RepositoryAuthorizedEmail, TeamMemberInvite,
                            DerivedImageStorage, ImageStorageTransformation, random_string_generator,
-                           db, BUILD_PHASE, QuayUserField)
+                           db, BUILD_PHASE, QuayUserField, validate_database_url)
 from peewee import JOIN_LEFT_OUTER, fn
 from util.validation import (validate_username, validate_email, validate_password,
                              INVALID_PASSWORD_MESSAGE)
@@ -2257,11 +2257,20 @@ def delete_user(user):
   # TODO: also delete any repository data associated
 
 
-def check_health():
+def check_health(app_config):
+  # Attempt to connect to the database first. If the DB is not responding,
+  # using the validate_database_url will timeout quickly, as opposed to
+  # making a normal connect which will just hang (thus breaking the health
+  # check).
+  try:
+    validate_database_url(app_config['DB_URI'], connect_timeout=3)
+  except Exception:
+    logger.exception('Could not connect to the database')
+    return False
+
   # We will connect to the db, check that it contains some log entry kinds
   try:
-    found_count = LogEntryKind.select().count()
-    return found_count > 0
+    return bool(list(LogEntryKind.select().limit(1)))
   except:
     return False
 

emails/base.html

@@ -4,6 +4,12 @@
     <meta name="viewport" content="width=device-width" />
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
     <title>{{ subject }}</title>
+
+    {% if action_metadata %}
+    <script type="application/ld+json">
+    {{ action_metadata }}
+    </script>
+    {% endif %}
   </head>
   <body bgcolor="#FFFFFF" style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; margin: 0; padding: 0;"><style type="text/css">
       @media only screen and (max-width: 600px) {

endpoints/common.py

@@ -4,14 +4,17 @@ import json
 import string
 import datetime
 
+# Register the various exceptions via decorators.
+import endpoints.decorated
+
 from flask import make_response, render_template, request, abort, session
-from flask.ext.login import login_user, UserMixin
+from flask.ext.login import login_user
 from flask.ext.principal import identity_changed
 from random import SystemRandom
 
 from data import model
 from data.database import db
-from app import app, login_manager, dockerfile_build_queue, notification_queue, oauth_apps
+from app import app, oauth_apps, dockerfile_build_queue, LoginWrappedDBUser
 
 from auth.permissions import QuayDeferredPermissionUser
 from auth import scopes
@@ -21,7 +24,6 @@ from functools import wraps
 from config import getFrontendVisibleConfig
 from external_libraries import get_external_javascript, get_external_css
 from endpoints.notificationhelper import spawn_notification
-from util.useremails import CannotSendEmailException
 
 import features
 
@@ -84,34 +86,8 @@ def param_required(param_name):
   return wrapper
 
 
-@login_manager.user_loader
-def load_user(user_uuid):
-  logger.debug('User loader loading deferred user with uuid: %s' % user_uuid)
-  return _LoginWrappedDBUser(user_uuid)
-
-
-class _LoginWrappedDBUser(UserMixin):
-  def __init__(self, user_uuid, db_user=None):
-    self._uuid = user_uuid
-    self._db_user = db_user
-
-  def db_user(self):
-    if not self._db_user:
-      self._db_user = model.get_user_by_uuid(self._uuid)
-    return self._db_user
-
-  def is_authenticated(self):
-    return self.db_user() is not None
-
-  def is_active(self):
-    return self.db_user().verified
-
-  def get_id(self):
-    return unicode(self._uuid)
-
-
 def common_login(db_user):
-  if login_user(_LoginWrappedDBUser(db_user.uuid, db_user)):
+  if login_user(LoginWrappedDBUser(db_user.uuid, db_user)):
     logger.debug('Successfully signed in as: %s (%s)' % (db_user.username, db_user.uuid))
     new_identity = QuayDeferredPermissionUser(db_user.uuid, 'user_uuid', {scopes.DIRECT_LOGIN})
     identity_changed.send(app, identity=new_identity)
@@ -121,17 +97,6 @@ def common_login(db_user):
     logger.debug('User could not be logged in, inactive?.')
     return False
 
-
-@app.errorhandler(model.DataModelException)
-def handle_dme(ex):
-  logger.exception(ex)
-  return make_response(json.dumps({'message': ex.message}), 400)
-
-@app.errorhandler(CannotSendEmailException)
-def handle_emailexception(ex):
-  message = 'Could not send email. Please contact an administrator and report this problem.'
-  return make_response(json.dumps({'message': message}), 400)
-
 def random_string():
   random = SystemRandom()
   return ''.join([random.choice(string.ascii_uppercase + string.digits) for _ in range(8)])

endpoints/decorated.py

@@ -0,0 +1,19 @@
+import logging
+import json
+
+from flask import make_response
+from app import app
+from util.useremails import CannotSendEmailException
+from data import model
+
+logger = logging.getLogger(__name__)
+
+@app.errorhandler(model.DataModelException)
+def handle_dme(ex):
+  logger.exception(ex)
+  return make_response(json.dumps({'message': ex.message}), 400)
+
+@app.errorhandler(CannotSendEmailException)
+def handle_emailexception(ex):
+  message = 'Could not send email. Please contact an administrator and report this problem.'
+  return make_response(json.dumps({'message': message}), 400)

endpoints/index.py

@@ -380,6 +380,11 @@ def get_search():
   resp.mimetype = 'application/json'
   return resp
 
+# Note: This is *not* part of the Docker index spec. This is here for our own health check,
+# since we have nginx handle the _ping below.
+@index.route('/_internal_ping')
+def internal_ping():
+  return make_response('true', 200)
 
 @index.route('/_ping')
 @index.route('/_ping')

endpoints/trigger.py

@@ -226,7 +226,7 @@ class GithubBuildTrigger(BuildTrigger):
         'personal': False,
         'repos': repo_list,
         'info': {
-          'name': org.name,
+          'name': org.name or org.login,
           'avatar_url': org.avatar_url
         }
       })

endpoints/web.py

@@ -6,7 +6,7 @@ from flask import (abort, redirect, request, url_for, make_response, Response,
 from avatar_generator import Avatar
 from flask.ext.login import current_user
 from urlparse import urlparse
-from health.healthcheck import HealthCheck
+from health.healthcheck import get_healthchecker
 
 from data import model
 from data.model.oauth import DatabaseAuthorizationProvider
@@ -30,6 +30,9 @@ import features
 
 logger = logging.getLogger(__name__)
 
+# Capture the unverified SSL errors.
+logging.captureWarnings(True)
+
 web = Blueprint('web', __name__)
 
 STATUS_TAGS = app.config['STATUS_TAGS']
@@ -164,33 +167,27 @@ def v1():
   return index('')
 
 
+# TODO(jschorr): Remove this mirrored endpoint once we migrate ELB.
 @web.route('/health', methods=['GET'])
+@web.route('/health/instance', methods=['GET'])
 @no_cache
-def health():
-  db_healthy = model.check_health()
-  buildlogs_healthy = build_logs.check_health()
-
-  check = HealthCheck.get_check(app.config['HEALTH_CHECKER'][0], app.config['HEALTH_CHECKER'][1])
-  (data, is_healthy) = check.conduct_healthcheck(db_healthy, buildlogs_healthy)
-
-  response = jsonify(dict(data=data, is_healthy=is_healthy))
-  response.status_code = 200 if is_healthy else 503
+def instance_health():
+  checker = get_healthchecker(app)
+  (data, status_code) = checker.check_instance()
+  response = jsonify(dict(data=data, status_code=status_code))
+  response.status_code = status_code
   return response
 
 
+# TODO(jschorr): Remove this mirrored endpoint once we migrate pingdom.
 @web.route('/status', methods=['GET'])
+@web.route('/health/endtoend', methods=['GET'])
 @no_cache
-def status():
-  db_healthy = model.check_health()
-  buildlogs_healthy = build_logs.check_health()
-
-  response = jsonify({
-    'db_healthy': db_healthy,
-    'buildlogs_healthy': buildlogs_healthy,
-    'is_testing': app.config['TESTING'],
-  })
-  response.status_code = 200 if db_healthy and buildlogs_healthy else 503
-
+def endtoend_health():
+  checker = get_healthchecker(app)
+  (data, status_code) = checker.check_endtoend()
+  response = jsonify(dict(data=data, status_code=status_code))
+  response.status_code = status_code
   return response
 
 

grunt/Gruntfile.js

@@ -25,7 +25,7 @@ module.exports = function(grunt) {
         },
       },
       build: {
-        src: ['../static/lib/**/*.js', '../static/js/*.js', '../static/dist/template-cache.js'],
+        src: ['../static/lib/**/*.js', '../static/js/**/*.js', '../static/dist/template-cache.js'],
         dest: '../static/dist/<%= pkg.name %>.js'
       }
     },

health/healthcheck.py

@@ -1,47 +1,84 @@
 import boto.rds2
 import logging
+from health.services import check_all_services
 
 logger = logging.getLogger(__name__)
 
-class HealthCheck(object):
-  def __init__(self):
-    pass
+def get_healthchecker(app):
+  """ Returns a HealthCheck instance for the given app. """
+  return HealthCheck.get_checker(app)
 
-  def conduct_healthcheck(self, db_healthy, buildlogs_healthy):
+
+class HealthCheck(object):
+  def __init__(self, app):
+    self.app = app
+
+  def check_instance(self):
     """
-    Conducts any custom healthcheck work, returning a dict representing the HealthCheck
-    output and a boolean indicating whether the instance is healthy.
+    Conducts a check on this specific instance, returning a dict representing the HealthCheck
+    output and a number indicating the health check response code.
     """
-    raise NotImplementedError
+    service_statuses = check_all_services(self.app)
+    return self.get_instance_health(service_statuses)
+
+  def check_endtoend(self):
+    """
+    Conducts a check on all services, returning a dict representing the HealthCheck
+    output and a number indicating the health check response code.
+    """
+    service_statuses = check_all_services(self.app)
+    return self.calculate_overall_health(service_statuses)
+
+  def get_instance_health(self, service_statuses):
+    """
+    For the given service statuses, returns a dict representing the HealthCheck
+    output and a number indicating the health check response code. By default,
+    this simply ensures that all services are reporting as healthy.
+    """
+    return self.calculate_overall_health(service_statuses)
+
+  def calculate_overall_health(self, service_statuses, skip=None, notes=None):
+    """ Returns true if and only if all the given service statuses report as healthy. """
+    is_healthy = True
+    notes = notes or []
+
+    for service_name in service_statuses:
+      if skip and service_name in skip:
+        notes.append('%s skipped in compute health' % service_name)
+        continue
+
+      is_healthy = is_healthy and service_statuses[service_name]
+
+    data = {
+      'services': service_statuses,
+      'notes': notes,
+      'is_testing': self.app.config['TESTING']
+    }
+
+    return (data, 200 if is_healthy else 503)
+
 
   @classmethod
-  def get_check(cls, name, parameters):
+  def get_checker(cls, app):
+    name = app.config['HEALTH_CHECKER'][0]
+    parameters = app.config['HEALTH_CHECKER'][1] or {}
+
     for subc in cls.__subclasses__():
       if subc.check_name() == name:
-        return subc(**parameters)
+        return subc(app, **parameters)
 
     raise Exception('Unknown health check with name %s' % name)
 
 
 class LocalHealthCheck(HealthCheck):
-  def __init__(self):
-    pass
-
   @classmethod
   def check_name(cls):
     return 'LocalHealthCheck'
 
-  def conduct_healthcheck(self, db_healthy, buildlogs_healthy):
-    data = {
-      'db_healthy': db_healthy,
-      'buildlogs_healthy': buildlogs_healthy
-    }
-
-    return (data, db_healthy and buildlogs_healthy)
-
 
 class ProductionHealthCheck(HealthCheck):
-  def __init__(self, access_key, secret_key):
+  def __init__(self, app, access_key, secret_key):
+    super(ProductionHealthCheck, self).__init__(app)
     self.access_key = access_key
     self.secret_key = secret_key
 
@@ -49,36 +86,38 @@ class ProductionHealthCheck(HealthCheck):
   def check_name(cls):
     return 'ProductionHealthCheck'
 
-  def conduct_healthcheck(self, db_healthy, buildlogs_healthy):
-    data = {
-      'db_healthy': db_healthy,
-      'buildlogs_healthy': buildlogs_healthy
-    }
+  def get_instance_health(self, service_statuses):
+    # Note: We skip the redis check because if redis is down, we don't want ELB taking the
+    # machines out of service. Redis is not considered a high avaliability-required service.
+    skip = ['redis']
+    notes = []
 
-    # Only report unhealthy if the machine cannot connect to the DB. Redis isn't required for
-    # mission critical/high avaliability operations.
+    # If the database is marked as unhealthy, check the status of RDS directly. If RDS is
+    # reporting as available, then the problem is with this instance. Otherwise, the problem is
+    # with RDS, and so we skip the DB status so we can keep this machine as 'healthy'.
+    db_healthy = service_statuses['database']
     if not db_healthy:
-      # If the database is marked as unhealthy, check the status of RDS directly. If RDS is
-      # reporting as available, then the problem is with this instance. Otherwise, the problem is
-      # with RDS, and we can keep this machine as 'healthy'.
-      is_rds_working = False
-      try:
-        region = boto.rds2.connect_to_region('us-east-1',
-          aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key)
-        response = region.describe_db_instances()['DescribeDBInstancesResponse']
-        result = response['DescribeDBInstancesResult']
-        instances = result['DBInstances']
-        status = instances[0]['DBInstanceStatus']
-        is_rds_working = status == 'available'
-      except:
-        logger.exception("Exception while checking RDS status")
-        pass
+      rds_status = self._get_rds_status()
+      notes.append('DB reports unhealthy; RDS status: %s' % rds_status)
 
-      data['db_available_checked'] = True
-      data['db_available_status'] = is_rds_working
+      # If the RDS is in any state but available, then we skip the DB check since it will
+      # fail and bring down the instance.
+      if rds_status != 'available':
+        skip.append('database')
 
-      # If RDS is down, then we still report the machine as healthy, so that it can handle
-      # requests once RDS comes back up.
-      return (data, not is_rds_working)
+    return self.calculate_overall_health(service_statuses, skip=skip, notes=notes)
 
-    return (data, db_healthy)
+
+  def _get_rds_status(self):
+    """ Returns the status of the RDS instance as reported by AWS. """
+    try:
+      region = boto.rds2.connect_to_region('us-east-1',
+        aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key)
+      response = region.describe_db_instances()['DescribeDBInstancesResponse']
+      result = response['DescribeDBInstancesResult']
+      instances = result['DBInstances']
+      status = instances[0]['DBInstanceStatus']
+      return status
+    except:
+      logger.exception("Exception while checking RDS status")
+      return 'error'

health/services.py

@@ -0,0 +1,46 @@
+import logging
+from data import model
+from app import build_logs
+
+logger = logging.getLogger(__name__)
+
+def _check_registry_gunicorn(app):
+  """ Returns the status of the registry gunicorn workers. """
+  # Compute the URL for checking the registry endpoint. We append a port if and only if the
+  # hostname contains one.
+  client = app.config['HTTPCLIENT']
+  hostname_parts = app.config['SERVER_HOSTNAME'].split(':')
+  port = ''
+  if len(hostname_parts) == 2:
+    port = ':' + hostname_parts[1]
+
+  registry_url = '%s://localhost%s/v1/_internal_ping' % (app.config['PREFERRED_URL_SCHEME'], port)
+  try:
+    return client.get(registry_url, verify=False, timeout=2).status_code == 200
+  except Exception:
+    logger.exception('Exception when checking registry health: %s', registry_url)
+    return False
+
+
+def _check_database(app):
+  """ Returns the status of the database, as accessed from this instance. """
+  return model.check_health(app.config)
+
+def _check_redis(app):
+  """ Returns the status of Redis, as accessed from this instance. """
+  return build_logs.check_health()
+
+
+_SERVICES = {
+  'registry_gunicorn': _check_registry_gunicorn,
+  'database': _check_database,
+  'redis': _check_redis
+}
+
+def check_all_services(app):
+  """ Returns a dictionary containing the status of all the services defined. """
+  status = {}
+  for name in _SERVICES:
+    status[name] = _SERVICES[name](app)
+
+  return status

registry.py

@@ -7,7 +7,6 @@ from endpoints.index import index
 from endpoints.tags import tags
 from endpoints.registry import registry
 
-
 application.register_blueprint(index, url_prefix='/v1')
 application.register_blueprint(tags, url_prefix='/v1')
 application.register_blueprint(registry, url_prefix='/v1')

requirements-nover.txt

@@ -1,4 +1,4 @@
-autobahn
+autobahn==0.9.3-3
 aiowsgi
 trollius
 peewee

requirements.txt

@@ -8,24 +8,21 @@ Jinja2==2.7.3
 LogentriesLogger==0.2.1
 Mako==1.0.0
 MarkupSafe==0.23
-Pillow==2.6.1
-PyMySQL==0.6.2
-PyPDF2==1.23
+Pillow==2.7.0
+PyMySQL==0.6.3
+PyPDF2==1.24
 PyYAML==3.11
 SQLAlchemy==0.9.8
+WebOb==1.4
 Werkzeug==0.9.6
-alembic==0.7.0
-git+https://github.com/DevTable/aniso8601-fake.git
-git+https://github.com/DevTable/anunidecode.git
-git+https://github.com/DevTable/avatar-generator.git
-git+https://github.com/DevTable/pygithub.git
 aiowsgi==0.3
+alembic==0.7.4
 autobahn==0.9.3-3
 backports.ssl-match-hostname==3.4.0.2
 beautifulsoup4==4.3.2
 blinker==1.3
-boto==2.34.0
-docker-py==0.6.0
+boto==2.35.1
+docker-py==0.7.1
 ecdsa==0.11
 futures==2.2.0
 gevent==1.0.1
@@ -36,26 +33,31 @@ hiredis==0.1.5
 html5lib==0.999
 itsdangerous==0.24
 jsonschema==2.4.0
-marisa-trie==0.6
-mixpanel-py==3.2.0
-git+https://github.com/NateFerrero/oauth2lib.git
-paramiko==1.15.1
-peewee==2.4.3
+marisa-trie==0.7
+mixpanel-py==3.2.1
+paramiko==1.15.2
+peewee==2.4.5
 psycopg2==2.5.4
 py-bcrypt==0.4
 pycrypto==2.6.1
-python-dateutil==2.2
-python-ldap==2.4.18
+python-dateutil==2.4.0
+python-ldap==2.4.19
 python-magic==0.4.6
-pytz==2014.9
+pytz==2014.10
 raven==5.1.1
 redis==2.10.3
 reportlab==2.7
-requests==2.4.3
-six==1.8.0
-stripe==1.19.1
-trollius==1.0.3
+requests==2.5.1
+six==1.9.0
+stripe==1.20.1
+trollius==1.0.4
 tzlocal==1.1.2
-websocket-client==0.21.0
+waitress==0.8.9
+websocket-client==0.23.0
 wsgiref==0.1.2
 xhtml2pdf==0.0.6
+git+https://github.com/DevTable/aniso8601-fake.git
+git+https://github.com/DevTable/anunidecode.git
+git+https://github.com/DevTable/avatar-generator.git
+git+https://github.com/DevTable/pygithub.git
+git+https://github.com/NateFerrero/oauth2lib.git

static/css/quay.css

@@ -1068,12 +1068,6 @@ i.toggle-icon:hover {
   border: 1px dashed #ccc;
 }
 
-.new-repo .initialize-repo .init-description {
-  color: #444;
-  font-size: 12px;
-  text-align: center;
-}
-
 .new-repo .initialize-repo .file-drop {
   margin: 10px;
 }
@@ -1319,13 +1313,16 @@ i.toggle-icon:hover {
   position: relative;
 }
 
-.plan-price:after {
-  content: "/ mo";
-  position: absolute;
-  bottom: 0px;
-  right: 20px;
-  font-size: 12px;
-  color: #aaa;
+
+@media (min-width: 768px) {
+  .plan-price:after {
+    content: "/ mo";
+    position: absolute;
+    bottom: 0px;
+    right: 20px;
+    font-size: 12px;
+    color: #aaa;
+  }
 }
 
 .plans-list .plan .count {
@@ -1488,9 +1485,6 @@ i.toggle-icon:hover {
   right: 0px;
 }
 
-.landing-filter.signedin {
-}
-
 .landing-content {
   z-index: 2;
 }
@@ -1500,7 +1494,6 @@ i.toggle-icon:hover {
 }
 
 .landing .call-to-action {
-  height: 40px;
   font-size: 18px;
   padding-left: 14px;
   padding-right: 14px;
@@ -1639,7 +1632,7 @@ i.toggle-icon:hover {
   padding-left: 70px;
 }
 
-.landing-page .twitter-tweet .avatar img {
+.landing-page .twitter-tweet .twitter-avatar img {
   border-radius: 4px;
   border: 2px solid rgb(70, 70, 70);
   width: 50px;
@@ -3519,6 +3512,22 @@ p.editable:hover i {
   font-size: 16px;
 }
 
+.plans-table ul {
+  margin-top: 10px;
+  padding: 0px;
+}
+
+
+.plans-table ul li {
+  padding: 4px;
+  margin: 0px;
+}
+
+.plans-table ul li .plan-info {
+  padding: 4px;
+}
+
+
 .repo-breadcrumb-element .crumb {
   cursor: pointer;
 }
@@ -4929,4 +4938,20 @@ i.slack-icon {
 .modal-footer.alert {
   text-align: left;
   margin-bottom: -16px;
-}
+}
+
+.dockerfile-build-form table td {
+  vertical-align: top;
+  white-space: nowrap;
+}
+
+.dockerfile-build-form input[type="file"] {
+  margin: 0px;
+}
+
+.dockerfile-build-form .help-text {
+  font-size: 13px;
+  color: #aaa;
+  margin-bottom: 20px;
+  padding-left: 22px;
+}

static/directives/dockerfile-build-dialog.html

@@ -15,7 +15,7 @@
           </div>
           <div class="dockerfile-build-form" repository="repository" upload-failed="handleBuildFailed(message)"
                build-started="handleBuildStarted(build)" build-failed="handleBuildFailed(message)" start-now="startCounter"
-               has-dockerfile="hasDockerfile" uploading="uploading" building="building"></div>
+               is-ready="hasDockerfile" uploading="uploading" building="building"></div>
         </div>
         <div class="modal-footer">
           <button type="button" class="btn btn-primary" ng-click="startBuild()" ng-disabled="building || uploading || !hasDockerfile">Start Build</button>

static/directives/dockerfile-build-form.html

@@ -11,9 +11,44 @@
   </div>
 
   <div class="container" ng-show="!uploading && !building">
-    <div class="init-description">
-      Upload a <b>Dockerfile</b> or an archive (<code>.zip</code> or <code>.tar.gz</code>) containing a Dockerfile <b>in the root directory</b>
-    </div>
-    <input id="file-drop" class="file-drop" type="file" file-present="internal.hasDockerfile">
+    <table>
+      <tr>
+        <td style="vertical-align: middle;">Dockerfile or <code>.tar.gz</code> or <code>.zip</code>:</td>
+        <td><input id="file-drop" class="file-drop" type="file" file-present="internal.hasDockerfile">
+      </tr>
+      <tr>
+        <td></td>
+        <td>
+          <div class="help-text">If an archive, the Dockerfile must be at the root</div>
+        </td>
+      </tr>
+      <tr>
+        <td>Base Image Pull Credentials:</td>
+        <td>
+          <!-- Select credentials -->
+          <div class="btn-group btn-group-sm">
+            <button type="button" class="btn btn-default"
+                    ng-class="is_public ? 'active btn-info' : ''"
+                    ng-click="is_public = true">
+              None
+            </button>
+            <button type="button" class="btn btn-default"
+                    ng-class="is_public ? '' : 'active btn-info'"
+                    ng-click="is_public = false">
+              <i class="fa fa-wrench"></i>
+              Robot account
+            </button>
+          </div>
+
+          <!-- Robot Select -->
+          <div ng-show="!is_public" style="margin-top: 10px">
+            <div class="entity-search" namespace="repository.namespace"
+                 placeholder="'Select robot account for pulling...'"
+                 current-entity="pull_entity"
+                 allowed-entities="['robot']"></div>
+          </div>
+        </td>
+      </tr>
+    </table>
   </div>
 </div>

static/directives/header-bar.html

@@ -19,8 +19,21 @@
     <li><a ng-href="{{ user.organizations.length ? '/organizations/' : '/tour/organizations/' }}" target="{{ appLinkTarget() }}" quay-section="organization">Organizations</a></li>
   </ul>
 
+  <!-- Phone -->
+  <ul class="nav navbar-nav navbar-right visible-xs" ng-switch on="user.anonymous">
+    <li ng-switch-when="false">
+      <a href="/user/" class="user-view" target="{{ appLinkTarget() }}">
+        <span class="avatar" size="32" hash="user.avatar"></span>
+        {{ user.username }}
+      </a>
+    </li>
+    <li ng-switch-default>
+      <a class="user-view" href="/signin/" target="{{ appLinkTarget() }}">Sign in</a>
+    </li>
+  </ul>
 
-  <ul class="nav navbar-nav navbar-right" ng-switch on="user.anonymous">
+  <!-- Normal -->
+  <ul class="nav navbar-nav navbar-right hidden-xs" ng-switch on="user.anonymous">
     <li>
       <form class="navbar-form navbar-left" role="search">
         <div class="form-group">

static/directives/plan-manager.html

@@ -31,11 +31,16 @@
        ng-show="!planLoading"></div>
 
   <!-- Plans Table -->
+  <div class="visible-xs" style="margin-top: 10px"></div>
+
   <table class="table table-hover plans-list-table" ng-show="!planLoading">
     <thead>
       <td>Plan</td>
-      <td>Private Repositories</td>
-      <td style="min-width: 64px">Price</td>
+      <td>
+        <span class="hidden-xs">Private Repositories</span>
+        <span class="visible-xs"><i class="fa fa-hdd-o"></i></span>
+      </td>
+      <td style="min-width: 64px"><span class="hidden-xs">Price</span><span class="visible-xs">$/mo</span></td>
       <td></td>
     </thead>
 

static/directives/plans-table.html

@@ -1,23 +1,41 @@
 <div class="plans-table-element">
-  <table class="table table-hover plans-table-table" ng-show="plans">
-    <thead>
-      <th>Plan</th>
-      <th>Private Repositories</th>
-      <th style="min-width: 85px">Price</th>
-      <th></th>
-    </thead>
+  <ul class="plans-table-list visible-xs">
+    <li ng-repeat="plan in plans" ng-class="currentPlan == plan ? 'active' : ''">
 
-    <tr ng-repeat="plan in plans" ng-class="currentPlan == plan ? 'active' : ''">
-      <td>{{ plan.title }}</td>
-      <td>{{ plan.privateRepos }}</td>
-      <td><div class="plan-price">${{ plan.price / 100 }}</div></td>
-      <td class="controls">
         <a class="btn" href="javascript:void(0)"
            ng-class="currentPlan == plan ? 'btn-primary' : 'btn-default'"
            ng-click="setPlan(plan)">
-          {{ currentPlan == plan ? 'Selected' : 'Choose' }}
+          {{ plan.title }}
         </a>
-      </td>
-    </tr>
-  </table>
+
+      <div class="plan-info">
+        ${{ plan.price / 100 }} / month -
+        {{ plan.privateRepos }} repositories
+      </div>
+    </li>
+  </ul>
+
+  <div class="hidden-xs">
+    <table class="table table-hover plans-table-table" ng-show="plans">
+      <thead>
+        <th>Plan</th>
+        <th>Private Repositories</th>
+        <th style="min-width: 85px">Price</th>
+        <th></th>
+      </thead>
+
+      <tr ng-repeat="plan in plans" ng-class="currentPlan == plan ? 'active' : ''">
+        <td>{{ plan.title }}</td>
+        <td>{{ plan.privateRepos }}</td>
+        <td><div class="plan-price">${{ plan.price / 100 }}</div></td>
+        <td class="controls">
+          <a class="btn" href="javascript:void(0)"
+             ng-class="currentPlan == plan ? 'btn-primary' : 'btn-default'"
+             ng-click="setPlan(plan)">
+            {{ currentPlan == plan ? 'Selected' : 'Choose' }}
+          </a>
+        </td>
+      </tr>
+    </table>
+  </div>
 </div>

static/directives/twitter-view.html

@@ -4,7 +4,7 @@
   </p>
   <div class="attribute">
     <span class="info-wrap">
-      <span class="avatar"><img ng-src="{{ avatarUrl }}" fallback-src="/static/img/default-twitter.png"></span>
+      <span class="twitter-avatar"><img ng-src="{{ avatarUrl }}" fallback-src="/static/img/default-twitter.png"></span>
       <span class="info">
         <span class="author">{{ authorName }} (@{{authorUser}})</span>
         <a class="reference" ng-href="{{ messageUrl }}">{{ messageDate }}</a>

static/js/app.js

@@ -2647,7 +2647,7 @@ quayApp.directive('focusablePopoverContent', ['$timeout', '$popover', function (
 
         if (!scope) { return; }
         scope.$apply(function() {
-          if (!scope) { return; }
+          if (!scope || !$scope.$hide) { return; }
           scope.$hide();
         });
       };
@@ -4358,6 +4358,8 @@ quayApp.directive('entitySearch', function () {
 
                 if (classes.length > 1) {
                   classes[classes.length - 1] = 'or ' +  classes[classes.length - 1];
+                } else if (classes.length == 0) {
+                return '<div class="tt-empty">No matching entities found</div>';
                 }
 
                 var class_string = '';
@@ -4443,7 +4445,6 @@ quayApp.directive('entitySearch', function () {
 
       $scope.$watch('namespace', function(namespace) {
         if (!namespace) { return; }
-
         $scope.isAdmin = UserService.isNamespaceAdmin(namespace);
         $scope.isOrganization = !!UserService.getOrganization(namespace);
       });
@@ -6233,7 +6234,7 @@ quayApp.directive('dockerfileBuildForm', function () {
     scope: {
       'repository': '=repository',
       'startNow': '=startNow',
-      'hasDockerfile': '=hasDockerfile',
+      'isReady': '=isReady',
       'uploadFailed': '&uploadFailed',
       'uploadStarted': '&uploadStarted',
       'buildStarted': '&buildStarted',
@@ -6244,6 +6245,8 @@ quayApp.directive('dockerfileBuildForm', function () {
     },
     controller: function($scope, $element, ApiService) {
       $scope.internal = {'hasDockerfile': false};
+      $scope.pull_entity = null;
+      $scope.is_public = true;
 
       var handleBuildFailed = function(message) {
         message = message || 'Dockerfile build failed to start';
@@ -6317,8 +6320,12 @@ quayApp.directive('dockerfileBuildForm', function () {
           'file_id': fileId
         };
 
+        if (!$scope.is_public && $scope.pull_entity) {
+          data['pull_robot'] = $scope.pull_entity['name'];
+        }
+
         var params = {
-          'repository': repo.namespace + '/' + repo.name
+          'repository': repo.namespace + '/' + repo.name,
         };
 
         ApiService.requestRepoBuild(data, params).then(function(resp) {
@@ -6396,9 +6403,13 @@ quayApp.directive('dockerfileBuildForm', function () {
         });
       };
 
-      $scope.$watch('internal.hasDockerfile', function(d) {
-        $scope.hasDockerfile = d;
-      });
+      var checkIsReady = function() {
+        $scope.isReady = $scope.internal.hasDockerfile && ($scope.is_public || $scope.pull_entity);
+      };
+
+      $scope.$watch('pull_entity', checkIsReady);
+      $scope.$watch('is_public', checkIsReady);
+      $scope.$watch('internal.hasDockerfile', checkIsReady);
 
       $scope.$watch('startNow', function() {
         if ($scope.startNow && $scope.repository && !$scope.uploading && !$scope.building) {

static/partials/landing-normal.html

@@ -207,7 +207,7 @@
               </li>
 
               <li>
-                <div class="twitter-view" avatar-url="https://pbs.twimg.com/profile_images/2578175278/ykn3l9ktfdy1hia5odij_bigger.jpeg"
+                <div class="twitter-view" avatar-url="https://pbs.twimg.com/profile_images/483391930147954688/pvJAHzy__bigger.jpeg"
                      author-name="Frank Macreery" author-user="fancyremarker" message-url="https://twitter.com/fancyremarker/statuses/448528623692025857"
                      message-date="March 25, 2014">
                   <a href="https://twitter.com/quayio">@quayio</a> releases Docker build flair! <a href="http://t.co/72ULgveLj4">pic.twitter.com/72ULgveLj4</a>

static/partials/new-organization.html

@@ -54,14 +54,14 @@
             <input id="orgName" name="orgName" type="text" class="form-control" placeholder="Organization Name"
                    ng-model="org.name" required autofocus data-trigger="manual" data-content="{{ createError }}"
                    data-placement="bottom" data-container="body" ng-pattern="/^[a-z0-9_]{4,30}$/">
-            <span class="description">This will also be the namespace for your repositories</span>
+            <span class="description">This will also be the namespace for your repositories. Must be alphanumeric and all lowercase.</span>
           </div>
 
           <div class="form-group nested">
             <label for="orgName">Organization Email</label>
             <input id="orgEmail" name="orgEmail" type="email" class="form-control" placeholder="Organization Email"
                    ng-model="org.email" required>
-            <span class="description">This address must be different from your account's email</span>
+            <span class="description">This address must be different from your account's email.</span>
           </div>
 
           <!-- Plans Table -->

static/partials/new-repo.html

@@ -143,9 +143,9 @@
         <div class="section-title">Upload <span ng-if="repo.initialize == 'dockerfile'">Dockerfile</span><span ng-if="repo.initialize == 'zipfile'">Archive</span></div>
         <div style="padding-top: 20px;">
           <div class="initialize-repo">
-            <div class="dockerfile-build-form" repository="createdForBuild" upload-failed="handleBuildFailed(message)"
+            <div class="dockerfile-build-form" repository="createdForBuild || repo" upload-failed="handleBuildFailed(message)"
                  build-started="handleBuildStarted()" build-failed="handleBuildFailed(message)" start-now="createdForBuild"
-                 has-dockerfile="hasDockerfile" uploading="uploading" building="building"></div>
+                 is-ready="hasDockerfile" uploading="uploading" building="building"></div>
           </div>
         </div>
       </div>

static/partials/team-view.html

@@ -2,7 +2,7 @@
   <div class="team-view container">
     <div class="organization-header" organization="organization" team-name="teamname">
       <div ng-show="canEditMembers" class="side-controls">
-        <div class="hidden-sm hidden-xs">
+        <div class="hidden-xs">
         <button class="btn btn-success"
                 id="showAddMember"
                 data-title="Add Team Member"
@@ -82,7 +82,7 @@
       </table>
 
       <div ng-show="canEditMembers">
-        <div ng-if-media="'(max-width: 560px)'">
+        <div ng-if-media="'(max-width: 767px)'">
           <div ng-include="'/static/directives/team-view-add.html'"></div>
         </div>
       </div>

test/test_api_usage.py

@@ -1965,6 +1965,9 @@ class TestOrgRobots(ApiTestCase):
     pull_robot = model.get_user(membername)
     model.create_build_trigger(repo, 'fakeservice', 'sometoken', user, pull_robot=pull_robot)
 
+    # Add some log entries for the robot.
+    model.log_action('pull_repo', ORGANIZATION, performer=pull_robot, repository=repo)
+
     # Delete the robot and verify it works.
     self.deleteResponse(OrgRobot,
                         params=dict(orgname=ORGANIZATION, robot_shortname='bender'))

util/analytics.py

@@ -1,8 +1,9 @@
 import json
 import logging
 
-from multiprocessing import Process, Queue
-from mixpanel import Consumer, Mixpanel
+from Queue import Queue
+from threading import Thread
+from mixpanel import BufferedConsumer, Mixpanel
 
 
 logger = logging.getLogger(__name__)
@@ -17,24 +18,23 @@ class MixpanelQueingConsumer(object):
     self._mp_queue.put(json.dumps([endpoint, json_message]))
 
 
-class SendToMixpanel(Process):
+class SendToMixpanel(Thread):
   def __init__(self, request_queue):
-    Process.__init__(self)
+    Thread.__init__(self)
+    self.daemon = True
 
     self._mp_queue = request_queue
-    self._consumer = Consumer()
-    self.daemon = True
+    self._consumer = BufferedConsumer()
 
   def run(self):
     logger.debug('Starting mixpanel sender process.')
     while True:
       mp_request = self._mp_queue.get()
-      logger.debug('Got queued mixpanel reqeust.')
+      logger.debug('Got queued mixpanel request.')
       try:
         self._consumer.send(*json.loads(mp_request))
       except:
-        # Make sure we don't crash if Mixpanel request fails.
-        pass
+        logger.exception('Failed to send Mixpanel request.')
 
 
 class FakeMixpanel(object):

util/queuemetrics.py

@@ -1,7 +1,9 @@
 import logging
 import boto
 
-from multiprocessing import Process, Queue
+from Queue import Queue
+from threading import Thread
+
 
 logger = logging.getLogger(__name__)
 
@@ -12,6 +14,7 @@ class NullReporter(object):
 
 
 class QueueingCloudWatchReporter(object):
+  """ QueueingCloudWatchReporter reports metrics to the "SendToCloudWatch" process """
   def __init__(self, request_queue, namespace, need_capacity_name, build_percent_name):
     self._namespace = namespace
     self._need_capacity_name = need_capacity_name
@@ -34,26 +37,37 @@ class QueueingCloudWatchReporter(object):
                         unit='Percent')
 
 
-class SendToCloudWatch(Process):
+class SendToCloudWatch(Thread):
+  """ SendToCloudWatch loops indefinitely and pulls metrics off of a queue then sends it to
+  CloudWatch. """
   def __init__(self, request_queue, aws_access_key, aws_secret_key):
-    Process.__init__(self)
+    Thread.__init__(self)
+    self.daemon = True
+
     self._aws_access_key = aws_access_key
     self._aws_secret_key = aws_secret_key
     self._put_metrics_queue = request_queue
-    self.daemon = True
 
   def run(self):
-    logger.debug('Starting cloudwatch sender process.')
-    connection = boto.connect_cloudwatch(self._aws_access_key, self._aws_secret_key)
+    try:
+      logger.debug('Starting CloudWatch sender process.')
+      connection = boto.connect_cloudwatch(self._aws_access_key, self._aws_secret_key)
+    except:
+      logger.exception('Failed to connect to CloudWatch.')
+
     while True:
       put_metric_args, kwargs = self._put_metrics_queue.get()
-      logger.debug('Got queued put metrics reqeust.')
-      connection.put_metric_data(*put_metric_args, **kwargs)
+      logger.debug('Got queued put metrics request.')
+      try:
+        connection.put_metric_data(*put_metric_args, **kwargs)
+      except:
+        logger.exception('Failed to write to CloudWatch')
 
 
 class QueueMetrics(object):
   def __init__(self, app=None):
     self.app = app
+    self.sender = None
     if app is not None:
       self.state = self.init_app(app)
     else:
@@ -72,8 +86,7 @@ class QueueMetrics(object):
       request_queue = Queue()
       reporter = QueueingCloudWatchReporter(request_queue, namespace, req_capacity_name,
                                             build_percent_name)
-      sender = SendToCloudWatch(request_queue, access_key, secret_key)
-      sender.start()
+      self.sender = SendToCloudWatch(request_queue, access_key, secret_key)
     else:
       reporter = NullReporter()
 
@@ -82,5 +95,11 @@ class QueueMetrics(object):
     app.extensions['queuemetrics'] = reporter
     return reporter
 
+  def run(self):
+    logger.debug('Asked to start CloudWatch reporter')
+    if self.sender is not None:
+      logger.debug('Starting CloudWatch reporter')
+      self.sender.start()
+
   def __getattr__(self, name):
     return getattr(self.state, name, None)

util/useremails.py

@@ -1,5 +1,6 @@
 import logging
 import traceback
+import json
 
 from flask.ext.mail import Message
 
@@ -13,7 +14,42 @@ template_env = get_template_env("emails")
 class CannotSendEmailException(Exception):
   pass
 
-def send_email(recipient, subject, template_file, parameters):
+class GmailAction(object):
+  """ Represents an action that can be taken in Gmail in response to the email. """
+  def __init__(self, metadata):
+    self.metadata = metadata
+
+  @staticmethod
+  def confirm(name, url, description):
+    return GmailAction({
+      "@context": "http://schema.org",
+      "@type": "EmailMessage",
+      "action": {
+        "@type": 'ConfirmAction',
+        "name": name,
+        "handler": {
+          "@type": "HttpActionHandler",
+          "url": get_app_url() + '/' + url
+        }
+      },
+      "description": description
+    })
+
+  @staticmethod
+  def view(name, url, description):
+    return GmailAction({
+      "@context": "http://schema.org",
+      "@type": "EmailMessage",
+      "action": {
+        "@type": 'ViewAction',
+        "name": name,
+        "url": get_app_url() + '/' + url
+      },
+      "description": description
+    })
+
+
+def send_email(recipient, subject, template_file, parameters, action=None):
   app_title = app.config['REGISTRY_TITLE_SHORT']
   app_url = get_app_url()
 
@@ -29,7 +65,8 @@ def send_email(recipient, subject, template_file, parameters):
     'app_logo': 'https://quay.io/static/img/quay-logo.png', # TODO: make this pull from config
     'app_url': app_url,
     'app_title': app_title,
-    'app_link': app_link_handler
+    'app_link': app_link_handler,
+    'action_metadata': json.dumps(action.metadata) if action else None
   })
 
   rendered_html = template_env.get_template(template_file + '.html').render(parameters)
@@ -61,25 +98,34 @@ def send_change_email(username, email, token):
   })
 
 def send_confirmation_email(username, email, token):
+  action = GmailAction.confirm('Confirm E-mail', 'confirm?code=' + token,
+                               'Verification of e-mail address')
+
   send_email(email, 'Please confirm your e-mail address', 'confirmemail', {
     'username': username,
     'token': token
-  })
+  }, action=action)
 
 def send_repo_authorization_email(namespace, repository, email, token):
+  action = GmailAction.confirm('Verify E-mail', 'authrepoemail?code=' + token,
+                               'Verification of e-mail address')
+
   subject = 'Please verify your e-mail address for repository %s/%s' % (namespace, repository)
   send_email(email, subject, 'repoauthorizeemail', {
     'namespace': namespace,
     'repository': repository,
     'token': token
-  })
+  }, action=action)
 
 def send_recovery_email(email, token):
+  action = GmailAction.view('Recover Account', 'recovery?code=' + token,
+                            'Recovery of an account')
+
   subject = 'Account recovery'
   send_email(email, subject, 'recovery', {
     'email': email,
     'token': token
-  })
+  }, action=action)
 
 def send_payment_failed(email, username):
   send_email(email, 'Subscription Payment Failure', 'paymentfailure', {
@@ -87,12 +133,15 @@ def send_payment_failed(email, username):
   })
 
 def send_org_invite_email(member_name, member_email, orgname, team, adder, code):
+  action = GmailAction.view('Join %s' % team, 'confirminvite?code=' + code,
+                            'Invitation to join a team')
+
   send_email(member_email, 'Invitation to join team', 'teaminvite', {
     'inviter': adder,
     'token': code,
     'organization': orgname,
     'teamname': team
-  })
+  }, action=action)
 
 
 def send_invoice_email(email, contents):

web.py

@@ -1,7 +1,7 @@
 import logging
 import logging.config
 
-from app import app as application
+from app import app as application, queue_metrics
 
 from endpoints.api import api_bp
 from endpoints.web import web
@@ -9,6 +9,8 @@ from endpoints.webhooks import webhooks
 from endpoints.realtime import realtime
 from endpoints.callbacks import callback
 
+# Start the cloudwatch reporting.
+queue_metrics.run()
 
 application.register_blueprint(web)
 application.register_blueprint(callback, url_prefix='/oauth2')