Merge remote-tracking branch 'upstream/master' into python-registry-v2

CHANGELOG.md

@@ -1,3 +1,22 @@
+### v1.11.2
+
+- Fixed security bug with LDAP login (#376)
+
+### 1.11.1
+
+- Loosened the check for mounted volumes bug (#353)
+- Strengthened HTTPS configuration (#329)
+- Disabled password change for non-DB auth (#347)
+- Added support for custom favicon (#343)
+- Fixed tarfile support for non-unicode pax fields (#328)
+- Fixed permissions on tag history API requiring READ instead of WRITE tokens (#316)
+- Added public access to time machine (#334)
+- Added missing JSON schema for 'refs' and 'branch_name' (#330)
+- Always create a new connection to Swift (#336)
+- Minor UI Fixes (#356, #341, #338, #337)
+- Minor trigger fixes (#357, #349)
+- Refactored and fixed internal code (#331)
+
 ### 1.11.0
 
 - Changed user pages to display public repositories (#321)

Dockerfile

@@ -1,6 +1,6 @@
 # vim:ft=dockerfile
 
-FROM phusion/baseimage:0.9.16
+FROM phusion/baseimage:0.9.17
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV HOME /root
@@ -42,6 +42,7 @@ ADD conf/init/copy_config_files.sh /etc/my_init.d/
 ADD conf/init/doupdatelimits.sh /etc/my_init.d/
 ADD conf/init/copy_syslog_config.sh /etc/my_init.d/
 ADD conf/init/runmigration.sh /etc/my_init.d/
+ADD conf/init/syslog-ng.conf /etc/syslog-ng/
 
 ADD conf/init/service/ /etc/service/
 

README.md

@@ -1,5 +1,7 @@
 # Quay.io - container image registry
 
+`master` branch build status: ![Docker Repository on Quay.io](https://quay.io/repository/quay/quay/status?token=7bffbc13-8bb0-4fb4-8a70-684a0cf485d3 "Docker Repository on Quay.io")
+
 Quay.io is a container image registry with managements APIs, a Docker registry API, a container build system.
 The application is implemented as a set of API endpoints written in python and an Angular.js frontend.
 
@@ -7,6 +9,42 @@ The application is implemented as a set of API endpoints written in python and a
 
 If you are doing local development on your workstation against the code base follow these instructions.
 
+### Docker
+
+Quay and its parts can run inside of docker containers.
+This method requires no installation of any python packages on your host machine.
+The `local-docker.sh` script is provided to prepare and run parts of quay.
+First, start redis:
+
+
+```
+docker run -d -p 6379:6379 redis
+```
+
+And clone the configuration repo:
+
+```
+git clone git@github.com:coreos-inc/quay-config.git ../quay-config
+ln -s ../../quay-config/local conf/stack
+```
+
+To build and run a docker container, pass one argument to local-docker.sh:
+
+- `dev`: run quay on port 5000
+- `buildman`: run the buildmanager
+- `notifications`: run the notification worker
+- `test`: run the unit tests
+
+For example:
+
+```
+./local-docker.sh dev
+````
+
+will start quay in a docker container.
+Now quay will be running on: http://127.0.0.1:5000
+The username is `devtable` and the password is `password`.
+
 ### OS X
 
 ```
@@ -15,8 +53,6 @@ cd quay
 ./contrib/osx/local-setup.sh
 ```
 
-## Running Development Environment
-
 Now run the server; it will use sqlite as the SQL server.
 
 ```

app.py

@@ -2,6 +2,7 @@ import logging
 import os
 import json
 
+from functools import partial
 from flask import Flask, request, Request, _request_ctx_stack
 from flask.ext.principal import Principal
 from flask.ext.login import LoginManager, UserMixin
@@ -20,17 +21,18 @@ from data.billing import Billing
 from data.buildlogs import BuildLogs
 from data.archivedlogs import LogArchive
 from data.userevent import UserEventsBuilderModule
-from data.queue import WorkQueue
+from data.queue import WorkQueue, MetricQueueReporter
+from util import get_app_url
 from util.saas.analytics import Analytics
 from util.saas.exceptionlog import Sentry
 from util.names import urn_generator
 from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig
 from util.security.signing import Signer
-from util.saas.queuemetrics import QueueMetrics
+from util.saas.cloudwatch import start_cloudwatch_sender
+from util.saas.metricqueue import MetricQueue
 from util.config.provider import FileConfigProvider, TestConfigProvider
 from util.config.configutil import generate_secret_key
 from util.config.superusermanager import SuperUserManager
-from buildman.jobutil.buildreporter import BuildMetrics
 
 OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/'
 OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml'
@@ -129,8 +131,8 @@ authentication = UserAuthentication(app, OVERRIDE_CONFIG_DIRECTORY)
 userevents = UserEventsBuilderModule(app)
 superusers = SuperUserManager(app)
 signer = Signer(app, OVERRIDE_CONFIG_DIRECTORY)
-queue_metrics = QueueMetrics(app)
-build_metrics = BuildMetrics(app)
+metric_queue = MetricQueue()
+start_cloudwatch_sender(metric_queue, app)
 
 tf = app.config['DB_TRANSACTION_FACTORY']
 
@@ -141,8 +143,9 @@ google_login = GoogleOAuthConfig(app.config, 'GOOGLE_LOGIN_CONFIG')
 oauth_apps = [github_login, github_trigger, gitlab_trigger, google_login]
 
 image_diff_queue = WorkQueue(app.config['DIFFS_QUEUE_NAME'], tf)
+image_replication_queue = WorkQueue(app.config['REPLICATION_QUEUE_NAME'], tf)
 dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf,
-                                   reporter=queue_metrics.report)
+                                   reporter=MetricQueueReporter(metric_queue))
 notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf)
 
 database.configure(app.config)
@@ -173,5 +176,4 @@ class LoginWrappedDBUser(UserMixin):
   def get_id(self):
     return unicode(self._uuid)
 
-def get_app_url():
-  return '%s://%s' % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'])
+get_app_url = partial(get_app_url, app.config)

auth/jwt_auth.py

@@ -1,5 +1,4 @@
 import logging
-import jwt
 import re
 
 from datetime import datetime, timedelta
@@ -11,10 +10,11 @@ from cryptography.hazmat.backends import default_backend
 from cachetools import lru_cache
 
 from app import app
-from auth_context import set_grant_user_context
-from permissions import repository_read_grant, repository_write_grant
+from .auth_context import set_grant_user_context
+from .permissions import repository_read_grant, repository_write_grant
 from util.names import parse_namespace_repository
 from util.http import abort
+from util.security import strictjwt
 
 
 logger = logging.getLogger(__name__)
@@ -44,17 +44,14 @@ def identity_from_bearer_token(bearer_token, max_signed_s, public_key):
 
   # Load the JWT returned.
   try:
-    payload = jwt.decode(encoded, public_key, algorithms=['RS256'], audience='quay',
-                         issuer='token-issuer')
-  except jwt.InvalidTokenError:
+    payload = strictjwt.decode(encoded, public_key, algorithms=['RS256'], audience='quay',
+                               issuer='token-issuer')
+  except strictjwt.InvalidTokenError:
     raise InvalidJWTException('Invalid token')
 
   if not 'sub' in payload:
     raise InvalidJWTException('Missing sub field in JWT')
 
-  if not 'exp' in payload:
-    raise InvalidJWTException('Missing exp field in JWT')
-
   # Verify that the expiration is no more than 300 seconds in the future.
   if datetime.fromtimestamp(payload['exp']) > datetime.utcnow() + timedelta(seconds=max_signed_s):
     raise InvalidJWTException('Token was signed for more than %s seconds' % max_signed_s)

buildman/component/buildcomponent.py

@@ -116,14 +116,10 @@ class BuildComponent(BaseComponent):
     #  push_token: The token to use to push the built image.
     #  tag_names: The name(s) of the tag(s) for the newly built image.
     #  base_image: The image name and credentials to use to conduct the base image pull.
-    #   repository: The repository to pull (DEPRECATED 0.2)
-    #   tag: The tag to pull (DEPRECATED in 0.2)
     #   username: The username for pulling the base image (if any).
     #   password: The password for pulling the base image (if any).
     build_arguments = {
-      'build_package': self.user_files.get_file_url(build_job.repo_build.resource_key,
-                                                    requires_cors=False)
-                       if build_job.repo_build.resource_key is not None else "",
+      'build_package': build_job.get_build_package_url(self.user_files),
       'sub_directory': build_config.get('build_subdir', ''),
       'repository': repository_name,
       'registry': self.registry_hostname,

buildman/jobutil/buildjob.py

@@ -62,6 +62,17 @@ class BuildJob(object):
   def repo_build(self):
     return self._load_repo_build()
 
+  def get_build_package_url(self, user_files):
+    """ Returns the URL of the build package for this build, if any or empty string if none. """
+    archive_url = self.build_config.get('archive_url', None)
+    if archive_url:
+      return archive_url
+
+    if not self.repo_build.resource_key:
+      return ''
+
+    return user_files.get_file_url(self.repo_build.resource_key, requires_cors=False)
+
   @property
   def pull_credentials(self):
     """ Returns the pull credentials for this job, or None if none. """

buildman/jobutil/buildreporter.py

@@ -1,70 +0,0 @@
-from buildman.enums import BuildJobResult
-from util.saas.cloudwatch import get_queue
-
-
-class BuildReporter(object):
-  """
-  Base class for reporting build statuses to a metrics service.
-  """
-  def report_completion_status(self, status):
-    """
-    Method to invoke the recording of build's completion status to a metric service.
-    """
-    raise NotImplementedError
-
-
-class NullReporter(BuildReporter):
-  """
-  The /dev/null of BuildReporters.
-  """
-  def report_completion_status(self, *args):
-    pass
-
-
-class CloudWatchBuildReporter(BuildReporter):
-  """
-  Implements a BuildReporter for Amazon's CloudWatch.
-  """
-  def __init__(self, queue, namespace_name, completed_name, failed_name, incompleted_name):
-    self._queue = queue
-    self._namespace_name = namespace_name
-    self._completed_name = completed_name
-    self._failed_name = failed_name
-    self._incompleted_name = incompleted_name
-
-  def _send_to_queue(self, *args, **kwargs):
-    self._queue.put((args, kwargs))
-
-  def report_completion_status(self, status):
-    if status == BuildJobResult.COMPLETE:
-      status_name = self._completed_name
-    elif status == BuildJobResult.ERROR:
-      status_name = self._failed_name
-    elif status == BuildJobResult.INCOMPLETE:
-      status_name = self._incompleted_name
-    else:
-      return
-
-    self._send_to_queue(self._namespace_name, status_name, 1, unit='Count')
-
-
-class BuildMetrics(object):
-  """
-  BuildMetrics initializes a reporter for recording the status of build completions.
-  """
-  def __init__(self, app=None):
-    self._app = app
-    self._reporter = NullReporter()
-    if app is not None:
-      reporter_type = app.config.get('BUILD_METRICS_TYPE', 'Null')
-      if reporter_type == 'CloudWatch':
-        namespace = app.config['BUILD_METRICS_NAMESPACE']
-        completed_name = app.config['BUILD_METRICS_COMPLETED_NAME']
-        failed_name = app.config['BUILD_METRICS_FAILED_NAME']
-        incompleted_name = app.config['BUILD_METRICS_INCOMPLETED_NAME']
-        request_queue = get_queue(app)
-        self._reporter = CloudWatchBuildReporter(request_queue, namespace, completed_name,
-                                                 failed_name, incompleted_name)
-
-  def __getattr__(self, name):
-    return getattr(self._reporter, name, None)

buildman/server.py

@@ -16,7 +16,7 @@ from buildman.enums import BuildJobResult, BuildServerStatus
 from buildman.jobutil.buildstatus import StatusHandler
 from buildman.jobutil.buildjob import BuildJob, BuildJobLoadException
 from data import database
-from app import app, build_metrics
+from app import app, metric_queue
 
 logger = logging.getLogger(__name__)
 
@@ -151,7 +151,7 @@ class BuilderServer(object):
     if self._current_status == BuildServerStatus.SHUTDOWN and not self._job_count:
       self._shutdown_event.set()
 
-    build_metrics.report_completion_status(job_status)
+    report_completion_status(job_status)
 
   @trollius.coroutine
   def _work_checker(self):
@@ -225,3 +225,15 @@ class BuilderServer(object):
 
     # Initialize the work queue checker.
     yield From(self._work_checker())
+
+def report_completion_status(status):
+  if status == BuildJobResult.COMPLETE:
+    status_name = 'CompleteBuilds'
+  elif status == BuildJobResult.ERROR:
+    status_name = 'FailedBuilds'
+  elif status == BuildJobResult.INCOMPLETE:
+    status_name = 'IncompletedBuilds'
+  else:
+    return
+
+  metric_queue.put(status_name, 1, unit='Count')

buildman/templates/cloudconfig.yaml

@@ -4,6 +4,9 @@ ssh_authorized_keys:
 - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCC0m+hVmyR3vn/xoxJe9+atRWBxSK+YXgyufNVDMcb7H00Jfnc341QH3kDVYZamUbhVh/nyc2RP7YbnZR5zORFtgOaNSdkMYrPozzBvxjnvSUokkCCWbLqXDHvIKiR12r+UTSijPJE/Yk702Mb2ejAFuae1C3Ec+qKAoOCagDjpQ3THyb5oaKE7VPHdwCWjWIQLRhC+plu77ObhoXIFJLD13gCi01L/rp4mYVCxIc2lX5A8rkK+bZHnIZwWUQ4t8SIjWxIaUo0FE7oZ83nKuNkYj5ngmLHQLY23Nx2WhE9H6NBthUpik9SmqQPtVYbhIG+bISPoH9Xs8CLrFb0VRjz Joey's Mac
 - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCo6FhAP7mFFOAzM91gtaKW7saahtaN4lur42FMMztz6aqUycIltCmvxo+3FmrXgCG30maMNU36Vm1+9QRtVQEd+eRuoIWP28t+8MT01Fh4zPuE2Wca3pOHSNo3X81FfWJLzmwEHiQKs9HPQqUhezR9PcVWVkbMyAzw85c0UycGmHGFNb0UiRd9HFY6XbgbxhZv/mvKLZ99xE3xkOzS1PNsdSNvjUKwZR7pSUPqNS5S/1NXyR4GhFTU24VPH/bTATOv2ATH+PSzsZ7Qyz9UHj38tKC+ALJHEDJ4HXGzobyOUP78cHGZOfCB5FYubq0zmOudAjKIAhwI8XTFvJ2DX1P3 jimmyzelinskie
 - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNvw8qo9m8np7yQ/Smv/oklM8bo8VyNRZriGYBDuolWDL/mZpYCQnZJXphQo7RFdNABYistikjJlBuuwUohLf2uSq0iKoFa2TgwI43wViWzvuzU4nA02/ITD5BZdmWAFNyIoqeB50Ol4qUgDwLAZ+7Kv7uCi6chcgr9gTi99jY3GHyZjrMiXMHGVGi+FExFuzhVC2drKjbz5q6oRfQeLtNfG4psl5GU3MQU6FkX4fgoCx0r9R48/b7l4+TT7pWblJQiRfeldixu6308vyoTUEHasdkU3/X0OTaGz/h5XqTKnGQc6stvvoED3w+L3QFp0H5Z8sZ9stSsitmCBrmbcKZ jakemoshenko
+- ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAgEAo/JkbGO6R7g1ZxARi0xWVM7FOfN02snRAcIO6vT9M7xMUkWVLgD+hM/o91lk+UFiYdql0CATobpFWncRL36KaUqsbw9/1BlI40wg296XHXSSnxhxZ4L7ytf6G1tyN319HXlI2kh9vAf/fy++yDvkH8dI3k1oLoW+mZPET6Pff04/6AXXrRlS5mhmGv9irGwiDHtVKpj6lU8DN/UtOrv1tiQ0pgwEJq05fLGoQfgPNaBCnW2z4Ubpn2gyMcMBMpSwo4hCqJePd349e4bLmFcT+gXYg7Mnup1DoTDlowFFN56wpxQbdp96IxWzU+jYPaIAuRo+BJzCyOS8qBv0Z4RZrgop0qp2JYiVwmViO6TZhIDz6loQJXUOIleQmNgTbiZx8Bwv5GY2jMYoVwlBp7yy5bRjxfbFsJ0vU7TVzNAG7oEJy/74HmHmWzRQlSlQjesr8gRbm9zgR8wqc/L107UOWFg7Cgh8ZNjKuADbXqYuda1Y9m2upcfS26UPz5l5PW5uFRMHZSi8pb1XV6/0Z8H8vwsh37Ur6aLi/5jruRmKhdlsNrB1IiDicBsPW3yg7HHSIdPU4oBNPC77yDCT3l4CKr4el81RrZt7FbJPfY+Ig9Q5O+05f6I8+ZOlJGyZ/Qfyl2aVm1HnlJKuBqPxeic8tMng/9B5N7uZL6Y3k5jFU8c= quentin
+- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDI7LtxLItapmUbt3Gs+4Oxa1i22fkx1+aJDkAjiRWPSX3+cxOzuPfHX9uFzr+qj5hy4J7ErrPp8q9alu+il9lE26GQuUxOZiaUrXu4dRCXXdCqTHARWBxGUXjkxdMp2HIzFpBxmVqcRubrgM36LBzKapdDOqQdz7XnNm5Jmf0tH/N0+TgV60P0WVY1CxmTya+JHNFVgazhd+oIGEhTyW/eszMGcFUgZet7DQFytYIQXYSwwGpGdJ+0InKAJ2SzCt/yuUlSrhrVM8vSGeami1XYmgQiyth1zjteMd8uTrc9NREH7bZTNcMFBqVYE3BYQWGRrv8pMMgP9gxgLbxtVsUl barakmich-titania
+- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDUWB4aSjSRHCz5/6H9/EJhJVvRmPThvEzyHinaWPsuM9prBSLci9NF9WneVl30nczkvllA+w34kycdrS3fKpjTbODaEOLHBobWl3bccY0I6kr86q5z67NZffjCm/P/RL+dBaOiBWS8PV8oiDF1P6YdMo8Jk46n9fozmLCXHUuCw5BJ8PGjQqbsEzA3qFMeKZYdJHOizOfeIfKfCWYrrumVRY9v6SAUDoFOl4PZEM7QdGp9EoRYb9MNLgKLnZ4RjbcLoFwiqxY4KEM4zfjZPNOECiLCuJqvHM2QawwuO1klJ16HpJk+FzOTWQoZtT47LoE/XNSOcNtAOiD+OQ449ia1EArhm7+1DnLXvHXKIl1JtuqJz+wFCsbNSdB7P562OHAGRIxYK3DfE+0CZH1BeHYl7xiRBeCtZ+OZMIocqeJtq8taIS7Un5wnGcQWxFtQnr/f65EgbIi7G2dxPcjhr6K+GWYezsiReVVKnIClq2MHhABG9QOncKDIa47L3nyx3pm4ZfMbC2jmnK2pFgGGSfYDy4487JnAUOG1mzZ9vm4gDhatT+vZFSBOwv1e4CErBh/wYXooF5I0nGmE6y6zkKFqP+ZolJ6iXmXQ7Ea2oaGeyaprweBjkhHgghi4KbwKbClope4Zo9X9JJYBLQSW33sEEuy8MlSBpdZAbz9t/FvJaw== mjibson
 
 write_files:
 - path: /root/overrides.list

conf/init/service/storagereplication/log/run

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec logger -i -t storagereplication

conf/init/service/storagereplication/run

@@ -0,0 +1,8 @@
+#! /bin/bash
+
+echo 'Starting storage replication worker'
+
+cd /
+venv/bin/python -m workers.storagereplication 2>&1
+
+echo 'Repository storage replication exited'

conf/init/syslog-ng.conf

@@ -0,0 +1,143 @@
+@version: 3.5
+@include "scl.conf"
+@include "`scl-root`/system/tty10.conf"
+
+# Syslog-ng configuration file, compatible with default Debian syslogd
+# installation.
+
+# First, set some global options.
+options { chain_hostnames(off); flush_lines(0); use_dns(no); use_fqdn(no);
+	  owner("root"); group("adm"); perm(0640); stats_freq(0);
+	  bad_hostname("^gconfd$");
+};
+
+########################
+# Sources
+########################
+# This is the default behavior of sysklogd package
+# Logs may come from unix stream, but not from another machine.
+#
+source s_src {
+       unix-stream("/dev/log");
+       internal();
+};
+
+# If you wish to get logs from remote machine you should uncomment
+# this and comment the above source line.
+#
+#source s_net { tcp(ip(127.0.0.1) port(1000)); };
+
+########################
+# Destinations
+########################
+# First some standard logfile
+#
+destination d_auth { file("/var/log/auth.log"); };
+destination d_cron { file("/var/log/cron.log"); };
+destination d_daemon { file("/var/log/daemon.log"); };
+destination d_kern { file("/var/log/kern.log"); };
+destination d_lpr { file("/var/log/lpr.log"); };
+destination d_mail { file("/var/log/mail.log"); };
+destination d_syslog { file("/var/log/syslog"); };
+destination d_user { file("/var/log/user.log"); };
+destination d_uucp { file("/var/log/uucp.log"); };
+
+# This files are the log come from the mail subsystem.
+#
+destination d_mailinfo { file("/var/log/mail.info"); };
+destination d_mailwarn { file("/var/log/mail.warn"); };
+destination d_mailerr { file("/var/log/mail.err"); };
+
+# Logging for INN news system
+#
+destination d_newscrit { file("/var/log/news/news.crit"); };
+destination d_newserr { file("/var/log/news/news.err"); };
+destination d_newsnotice { file("/var/log/news/news.notice"); };
+
+# Some `catch-all' logfiles.
+#
+destination d_debug { file("/var/log/debug"); };
+destination d_error { file("/var/log/error"); };
+destination d_messages { file("/var/log/messages"); };
+
+# The named pipe /dev/xconsole is for the nsole' utility.  To use it,
+# you must invoke nsole' with the -file' option:
+#
+#    $ xconsole -file /dev/xconsole [...]
+#
+destination d_xconsole { pipe("/dev/xconsole"); };
+
+# Send the messages to an other host
+#
+#destination d_net { tcp("127.0.0.1" port(1000) log_fifo_size(1000)); };
+
+# Debian only
+destination d_ppp { file("/var/log/ppp.log"); };
+
+########################
+# Filters
+########################
+# Here's come the filter options. With this rules, we can set which
+# message go where.
+
+filter f_dbg { level(debug); };
+filter f_info { level(info); };
+filter f_notice { level(notice); };
+filter f_warn { level(warn); };
+filter f_err { level(err); };
+filter f_crit { level(crit .. emerg); };
+
+filter f_debug { level(debug) and not facility(auth, authpriv, news, mail); };
+filter f_error { level(err .. emerg) ; };
+
+filter f_auth { facility(auth, authpriv) and not filter(f_debug); };
+filter f_cron { facility(cron) and not filter(f_debug); };
+filter f_daemon { facility(daemon) and not filter(f_debug); };
+filter f_kern { facility(kern) and not filter(f_debug); };
+filter f_lpr { facility(lpr) and not filter(f_debug); };
+filter f_local { facility(local0, local1, local3, local4, local5,
+                        local6, local7) and not filter(f_debug); };
+filter f_mail { facility(mail) and not filter(f_debug); };
+filter f_news { facility(news) and not filter(f_debug); };
+filter f_syslog3 { not facility(auth, authpriv, mail) and not filter(f_debug); };
+filter f_uucp { facility(uucp) and not filter(f_debug); };
+
+filter f_cnews { level(notice, err, crit) and facility(news); };
+filter f_cother { level(debug, info, notice, warn) or facility(daemon, mail); };
+
+filter f_ppp { facility(local2) and not filter(f_debug); };
+filter f_console { level(warn .. emerg); };
+
+########################
+# Log paths
+########################
+log { source(s_src); filter(f_auth); destination(d_auth); };
+log { source(s_src); filter(f_cron); destination(d_cron); };
+log { source(s_src); filter(f_daemon); destination(d_daemon); };
+log { source(s_src); filter(f_kern); destination(d_kern); };
+log { source(s_src); filter(f_lpr); destination(d_lpr); };
+log { source(s_src); filter(f_syslog3); destination(d_syslog); };
+log { source(s_src); filter(f_uucp); destination(d_uucp); };
+
+log { source(s_src); filter(f_mail); destination(d_mail); };
+#log { source(s_src); filter(f_mail); filter(f_info); destination(d_mailinfo); };
+#log { source(s_src); filter(f_mail); filter(f_warn); destination(d_mailwarn); };
+#log { source(s_src); filter(f_mail); filter(f_err); destination(d_mailerr); };
+
+log { source(s_src); filter(f_news); filter(f_crit); destination(d_newscrit); };
+log { source(s_src); filter(f_news); filter(f_err); destination(d_newserr); };
+log { source(s_src); filter(f_news); filter(f_notice); destination(d_newsnotice); };
+
+#log { source(s_src); filter(f_ppp); destination(d_ppp); };
+
+log { source(s_src); filter(f_debug); destination(d_debug); };
+log { source(s_src); filter(f_error); destination(d_error); };
+
+# All messages send to a remote site
+#
+#log { source(s_src); destination(d_net); };
+
+###
+# Include all config files in /etc/syslog-ng/conf.d/
+###
+@include "/etc/syslog-ng/conf.d/*.conf"

config.py

@@ -20,7 +20,7 @@ CLIENT_WHITELIST = ['SERVER_HOSTNAME', 'PREFERRED_URL_SCHEME', 'MIXPANEL_KEY',
                     'STRIPE_PUBLISHABLE_KEY', 'ENTERPRISE_LOGO_URL', 'SENTRY_PUBLIC_DSN',
                     'AUTHENTICATION_TYPE', 'REGISTRY_TITLE', 'REGISTRY_TITLE_SHORT',
                     'CONTACT_INFO', 'AVATAR_KIND', 'LOCAL_OAUTH_HANDLER', 'DOCUMENTATION_LOCATION',
-                    'DOCUMENTATION_METADATA']
+                    'DOCUMENTATION_METADATA', 'SETUP_COMPLETE']
 
 
 def frontend_visible_config(config_dict):
@@ -129,6 +129,7 @@ class DefaultConfig(object):
   NOTIFICATION_QUEUE_NAME = 'notification'
   DIFFS_QUEUE_NAME = 'imagediff'
   DOCKERFILE_BUILD_QUEUE_NAME = 'dockerfilebuild'
+  REPLICATION_QUEUE_NAME = 'imagestoragereplication'
 
   # Super user config. Note: This MUST BE an empty list for the default config.
   SUPER_USERS = []
@@ -179,6 +180,9 @@ class DefaultConfig(object):
   # basic auth.
   FEATURE_REQUIRE_ENCRYPTED_BASIC_AUTH = False
 
+  # Feature Flag: Whether to automatically replicate between storage engines.
+  FEATURE_STORAGE_REPLICATION = False
+
   BUILD_MANAGER = ('enterprise', {})
 
   DISTRIBUTED_STORAGE_CONFIG = {
@@ -187,6 +191,7 @@ class DefaultConfig(object):
   }
 
   DISTRIBUTED_STORAGE_PREFERENCE = ['local_us']
+  DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS = ['local_us']
 
   # Health checker.
   HEALTH_CHECKER = ('LocalHealthCheck', {})

data/archivedlogs.py

@@ -1,8 +1,7 @@
 import logging
 
-from gzip import GzipFile
+from util.registry.gzipinputstream import GzipInputStream
 from flask import send_file, abort
-from cStringIO import StringIO
 
 from data.userfiles import DelegateUserfiles, UserfilesHandlers
 
@@ -17,10 +16,8 @@ class LogArchiveHandlers(UserfilesHandlers):
   def get(self, file_id):
     path = self._files.get_file_id_path(file_id)
     try:
-      with self._storage.stream_read_file(self._locations, path) as gzip_stream:
-        with GzipFile(fileobj=gzip_stream) as unzipped:
-          unzipped_buffer = StringIO(unzipped.read())
-          return send_file(unzipped_buffer, mimetype=JSON_MIMETYPE)
+      data_stream = self._storage.stream_read_file(self._locations, path)
+      return send_file(GzipInputStream(data_stream), mimetype=JSON_MIMETYPE)
     except IOError:
       abort(404)
 

data/billing.py

@@ -17,6 +17,7 @@ PLANS = [
     'deprecated': True,
     'free_trial_days': 14,
     'superseded_by': None,
+    'plans_page_hidden': False,
   },
   {
     'title': 'Basic',
@@ -28,6 +29,7 @@ PLANS = [
     'deprecated': True,
     'free_trial_days': 14,
     'superseded_by': None,
+    'plans_page_hidden': False,
   },
   {
     'title': 'Yacht',
@@ -39,6 +41,7 @@ PLANS = [
     'deprecated': True,
     'free_trial_days': 180,
     'superseded_by': 'bus-small-30',
+    'plans_page_hidden': False,
   },
   {
     'title': 'Personal',
@@ -50,6 +53,7 @@ PLANS = [
     'deprecated': True,
     'free_trial_days': 14,
     'superseded_by': 'personal-30',
+    'plans_page_hidden': False,
   },
   {
     'title': 'Skiff',
@@ -61,6 +65,7 @@ PLANS = [
     'deprecated': True,
     'free_trial_days': 14,
     'superseded_by': 'bus-micro-30',
+    'plans_page_hidden': False,
   },
   {
     'title': 'Yacht',
@@ -72,6 +77,7 @@ PLANS = [
     'deprecated': True,
     'free_trial_days': 14,
     'superseded_by': 'bus-small-30',
+    'plans_page_hidden': False,
   },
   {
     'title': 'Freighter',
@@ -83,6 +89,7 @@ PLANS = [
     'deprecated': True,
     'free_trial_days': 14,
     'superseded_by': 'bus-medium-30',
+    'plans_page_hidden': False,
   },
   {
     'title': 'Tanker',
@@ -94,6 +101,7 @@ PLANS = [
     'deprecated': True,
     'free_trial_days': 14,
     'superseded_by': 'bus-large-30',
+    'plans_page_hidden': False,
   },
 
   # Active plans
@@ -107,6 +115,7 @@ PLANS = [
     'deprecated': False,
     'free_trial_days': 30,
     'superseded_by': None,
+    'plans_page_hidden': False,
   },
   {
     'title': 'Personal',
@@ -118,6 +127,7 @@ PLANS = [
     'deprecated': False,
     'free_trial_days': 30,
     'superseded_by': None,
+    'plans_page_hidden': False,
   },
   {
     'title': 'Skiff',
@@ -129,6 +139,7 @@ PLANS = [
     'deprecated': False,
     'free_trial_days': 30,
     'superseded_by': None,
+    'plans_page_hidden': False,
   },
   {
     'title': 'Yacht',
@@ -140,6 +151,7 @@ PLANS = [
     'deprecated': False,
     'free_trial_days': 30,
     'superseded_by': None,
+    'plans_page_hidden': False,
   },
   {
     'title': 'Freighter',
@@ -151,6 +163,7 @@ PLANS = [
     'deprecated': False,
     'free_trial_days': 30,
     'superseded_by': None,
+    'plans_page_hidden': False,
   },
   {
     'title': 'Tanker',
@@ -162,6 +175,19 @@ PLANS = [
     'deprecated': False,
     'free_trial_days': 30,
     'superseded_by': None,
+    'plans_page_hidden': False,
+  },
+  {
+    'title': 'Carrier',
+    'price': 35000,
+    'privateRepos': 250,
+    'stripeId': 'bus-xlarge-30',
+    'audience': 'For extra large businesses',
+    'bus_features': True,
+    'deprecated': False,
+    'free_trial_days': 30,
+    'superseded_by': None,
+    'plans_page_hidden': True,
   },
 ]
 

data/database.py

@@ -544,6 +544,15 @@ class ImageStoragePlacement(BaseModel):
     )
 
 
+class UserRegion(BaseModel):
+  user = QuayUserField(index=True, allows_robots=False)
+  location = ForeignKeyField(ImageStorageLocation)
+
+  indexes = (
+    (('user', 'location'), True),
+  )
+
+
 class Image(BaseModel):
   # This class is intentionally denormalized. Even though images are supposed
   # to be globally unique we can't treat them as such for permissions and
@@ -733,6 +742,7 @@ class RepositoryNotification(BaseModel):
   repository = ForeignKeyField(Repository, index=True)
   event = ForeignKeyField(ExternalNotificationEvent)
   method = ForeignKeyField(ExternalNotificationMethod)
+  title = CharField(null=True)
   config_json = TextField()
 
 
@@ -777,4 +787,4 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission,
               ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification,
               RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage,
               TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind,
-              AccessTokenKind, Star, RepositoryActionCount, TagManifest, BlobUpload]
+              AccessTokenKind, Star, RepositoryActionCount, TagManifest, BlobUpload, UserRegion]

data/migrations/versions/499f6f08de3_add_title_field_to_notification.py

@@ -0,0 +1,26 @@
+"""Add title field to notification
+
+Revision ID: 499f6f08de3
+Revises: 246df01a6d51
+Create Date: 2015-08-21 14:18:07.287743
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '499f6f08de3'
+down_revision = '246df01a6d51'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('repositorynotification', sa.Column('title', sa.String(length=255), nullable=True))
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('repositorynotification', 'title')
+    ### end Alembic commands ###

data/migrations/versions/9512773a4a2_add_userregion_table.py

@@ -0,0 +1,35 @@
+"""Add UserRegion table
+
+Revision ID: 9512773a4a2
+Revises: 499f6f08de3
+Create Date: 2015-09-01 14:17:08.628052
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '9512773a4a2'
+down_revision = '499f6f08de3'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('userregion',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('user_id', sa.Integer(), nullable=False),
+    sa.Column('location_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['location_id'], ['imagestoragelocation.id'], name=op.f('fk_userregion_location_id_imagestoragelocation')),
+    sa.ForeignKeyConstraint(['user_id'], ['user.id'], name=op.f('fk_userregion_user_id_user')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_userregion'))
+    )
+    op.create_index('userregion_location_id', 'userregion', ['location_id'], unique=False)
+    op.create_index('userregion_user_id', 'userregion', ['user_id'], unique=False)
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('userregion')
+    ### end Alembic commands ###

data/model/notification.py

@@ -113,12 +113,12 @@ def delete_matching_notifications(target, kind_name, **kwargs):
     notification.delete_instance()
 
 
-def create_repo_notification(repo, event_name, method_name, config):
+def create_repo_notification(repo, event_name, method_name, config, title=None):
   event = ExternalNotificationEvent.get(ExternalNotificationEvent.name == event_name)
   method = ExternalNotificationMethod.get(ExternalNotificationMethod.name == method_name)
 
   return RepositoryNotification.create(repository=repo, event=event, method=method,
-                                       config_json=json.dumps(config))
+                                       config_json=json.dumps(config), title=title)
 
 
 def get_repo_notification(uuid):

data/model/oauth.py

@@ -8,8 +8,9 @@ from oauth2lib import utils
 
 from data.database import (OAuthApplication, OAuthAuthorizationCode, OAuthAccessToken, User,
                            AccessToken, random_string_generator)
-from data.model import user
+from data.model import user, config
 from auth import scopes
+from util import get_app_url
 
 
 logger = logging.getLogger(__name__)
@@ -45,7 +46,10 @@ class DatabaseAuthorizationProvider(AuthorizationProvider):
       return False
 
   def validate_redirect_uri(self, client_id, redirect_uri):
-    if redirect_uri == url_for('web.oauth_local_handler', _external=True):
+    internal_redirect_url = '%s%s' % (get_app_url(config.app_config),
+                                      url_for('web.oauth_local_handler'))
+
+    if redirect_uri == internal_redirect_url:
       return True
 
     try:

data/model/permission.py

@@ -17,14 +17,19 @@ def list_robot_permissions(robot_name):
           .where(User.username == robot_name, User.robot == True))
 
 
-def list_organization_member_permissions(organization):
+def list_organization_member_permissions(organization, limit_to_user=None):
   query = (RepositoryPermission
            .select(RepositoryPermission, Repository, User)
            .join(Repository)
            .switch(RepositoryPermission)
            .join(User)
-           .where(Repository.namespace_user == organization)
-           .where(User.robot == False))
+           .where(Repository.namespace_user == organization))
+
+  if limit_to_user is not None:
+    query = query.where(RepositoryPermission.user == limit_to_user)
+  else:
+    query = query.where(User.robot == False)
+
   return query
 
 

data/model/storage.py

@@ -11,6 +11,12 @@ from data.database import (ImageStorage, Image, DerivedImageStorage, ImageStorag
 logger = logging.getLogger(__name__)
 
 
+def add_storage_placement(storage, location_name):
+  """ Adds a storage placement for the given storage at the given location. """
+  location = ImageStorageLocation.get(name=location_name)
+  ImageStoragePlacement.create(location=location, storage=storage)
+
+
 def find_or_create_derived_storage(source, transformation_name, preferred_location):
   existing = find_derived_storage(source, transformation_name)
   if existing is not None:

data/model/user.py

@@ -8,7 +8,8 @@ from datetime import datetime, timedelta
 
 from data.database import (User, LoginService, FederatedLogin, RepositoryPermission, TeamMember,
                            Team, Repository, TupleSelector, TeamRole, Namespace, Visibility,
-                           EmailConfirmation, Role, db_for_update, random_string_generator)
+                           EmailConfirmation, Role, db_for_update, random_string_generator,
+                           UserRegion, ImageStorageLocation)
 from data.model import (DataModelException, InvalidPasswordException, InvalidRobotException,
                         InvalidUsernameException, InvalidEmailAddressException,
                         TooManyUsersException, TooManyLoginAttemptsException, db_transaction,
@@ -463,6 +464,13 @@ def get_user_by_id(user_db_id):
     return None
 
 
+def get_namespace_user_by_user_id(namespace_user_db_id):
+  try:
+    return User.get(User.id == namespace_user_db_id, User.robot == False)
+  except User.DoesNotExist:
+    raise InvalidUsernameException('User with id does not exist: %s' % namespace_user_db_id)
+
+
 def get_namespace_by_user_id(namespace_user_db_id):
   try:
     return User.get(User.id == namespace_user_db_id, User.robot == False).username
@@ -664,3 +672,8 @@ def get_pull_credentials(robotname):
       'registry': '%s://%s/v1/' % (config.app_config['PREFERRED_URL_SCHEME'],
                                    config.app_config['SERVER_HOSTNAME']),
   }
+
+def get_region_locations(user):
+  """ Returns the locations defined as preferred storage for the given user. """
+  query = UserRegion.select().join(ImageStorageLocation).where(UserRegion.user == user)
+  return set([region.location.name for region in query])

data/queue.py

@@ -13,6 +13,17 @@ class NoopWith:
   def __exit__(self, type, value, traceback):
     pass
 
+class MetricQueueReporter(object):
+  def __init__(self, metric_queue):
+    self._metric_queue = metric_queue
+
+  def __call__(self, currently_processing, running_count, total_count):
+    need_capacity_count = total_count - running_count
+    self._metric_queue.put('BuildCapacityShortage', need_capacity_count, unit='Count')
+
+    building_percent = 100 if currently_processing else 0
+    self._metric_queue.put('PercentBuilding', building_percent, unit='Percent')
+
 class WorkQueue(object):
   def __init__(self, queue_name, transaction_factory,
                canonical_name_match_list=None, reporter=None):

data/users/externaljwt.py

@@ -1,13 +1,15 @@
 import logging
 import json
 import os
-import jwt
 
 from datetime import datetime, timedelta
 from data.users.federated import FederatedUsers, VerifiedCredentials
+from util.security import strictjwt
+
 
 logger = logging.getLogger(__name__)
 
+
 class ExternalJWTAuthN(FederatedUsers):
   """ Delegates authentication to a REST endpoint that returns JWTs. """
   PUBLIC_KEY_FILENAME = 'jwt-authn.cert'
@@ -45,9 +47,9 @@ class ExternalJWTAuthN(FederatedUsers):
     # Load the JWT returned.
     encoded = result_data.get('token', '')
     try:
-      payload = jwt.decode(encoded, self.public_key, algorithms=['RS256'],
-                           audience='quay.io/jwtauthn', issuer=self.issuer)
-    except jwt.InvalidTokenError:
+      payload = strictjwt.decode(encoded, self.public_key, algorithms=['RS256'],
+                                 audience='quay.io/jwtauthn', issuer=self.issuer)
+    except strictjwt.InvalidTokenError:
       logger.exception('Exception when decoding returned JWT')
       return (None, 'Invalid username or password')
 

data/users/externalldap.py

@@ -9,6 +9,16 @@ from data.users.federated import FederatedUsers, VerifiedCredentials
 logger = logging.getLogger(__name__)
 
 
+class LDAPConnectionBuilder(object):
+  def __init__(self, ldap_uri, user_dn, user_pw):
+    self._ldap_uri = ldap_uri
+    self._user_dn = user_dn
+    self._user_pw = user_pw
+
+  def get_connection(self):
+    return LDAPConnection(self._ldap_uri, self._user_dn, self._user_pw)
+
+
 class LDAPConnection(object):
   def __init__(self, ldap_uri, user_dn, user_pw):
     self._ldap_uri = ldap_uri
@@ -20,13 +30,7 @@ class LDAPConnection(object):
     trace_level = 2 if os.environ.get('USERS_DEBUG') == '1' else 0
     self._conn = ldap.initialize(self._ldap_uri, trace_level=trace_level)
     self._conn.set_option(ldap.OPT_REFERRALS, 1)
-
-    try:
-      self._conn.simple_bind_s(self._user_dn, self._user_pw)
-    except ldap.INVALID_CREDENTIALS:
-      logger.exception('LDAP admin dn or password are invalid')
-      return None
-
+    self._conn.simple_bind_s(self._user_dn, self._user_pw)
     return self._conn
 
   def __exit__(self, exc_type, value, tb):
@@ -38,7 +42,7 @@ class LDAPUsers(FederatedUsers):
 
   def __init__(self, ldap_uri, base_dn, admin_dn, admin_passwd, user_rdn, uid_attr, email_attr):
     super(LDAPUsers, self).__init__('ldap')
-    self._ldap_conn = LDAPConnection(ldap_uri, admin_dn, admin_passwd)
+    self._ldap = LDAPConnectionBuilder(ldap_uri, admin_dn, admin_passwd)
     self._ldap_uri = ldap_uri
     self._base_dn = base_dn
     self._user_rdn = user_rdn
@@ -65,10 +69,15 @@ class LDAPUsers(FederatedUsers):
     return referral_dn
 
   def _ldap_user_search(self, username_or_email):
-    with self._ldap_conn as conn:
-      if conn is None:
-        return (None, 'LDAP Admin dn or password is invalid')
+    # Verify the admin connection works first. We do this here to avoid wrapping
+    # the entire block in the INVALID CREDENTIALS check.
+    try:
+      with self._ldap.get_connection():
+        pass
+    except ldap.INVALID_CREDENTIALS:
+      return (None, 'LDAP Admin dn or password is invalid')
 
+    with self._ldap.get_connection() as conn:
       logger.debug('Incoming username or email param: %s', username_or_email.__repr__())
       user_search_dn = ','.join(self._user_rdn + self._base_dn)
       query = u'(|({0}={2})({1}={2}))'.format(self._uid_attr, self._email_attr,

dev.df

@@ -18,4 +18,4 @@ RUN venv/bin/pip install -r requirements.txt
 
 WORKDIR /src/quay
 ENV PYTHONPATH=/
-ENV PATH=$PATH:/venv/bin
+ENV PATH=/venv/bin:$PATH

endpoints/api/__init__.py

@@ -1,7 +1,7 @@
 import logging
 import datetime
 
-from app import app
+from app import app, metric_queue
 from flask import Blueprint, request, make_response, jsonify, session
 from flask.ext.restful import Resource, abort, Api, reqparse
 from flask.ext.restful.utils.cors import crossdomain
@@ -20,6 +20,7 @@ from auth.auth_context import get_authenticated_user, get_validated_oauth_token
 from auth.auth import process_oauth
 from endpoints.csrf import csrf_protect
 from endpoints.decorators import check_anon_protection
+from util.saas.metricqueue import time_decorator
 
 
 logger = logging.getLogger(__name__)
@@ -28,7 +29,7 @@ api = Api()
 api.init_app(api_bp)
 api.decorators = [csrf_protect,
                   crossdomain(origin='*', headers=['Authorization', 'Content-Type']),
-                  process_oauth]
+                  process_oauth, time_decorator(api_bp.name, metric_queue)]
 
 
 class ApiException(Exception):

endpoints/api/build.py

@@ -3,8 +3,10 @@
 import logging
 import json
 import datetime
+import hashlib
 
 from flask import request
+from rfc3987 import parse as uri_parse
 
 from app import app, userfiles as user_files, build_logs, log_archive, dockerfile_build_queue
 from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource,
@@ -134,8 +136,11 @@ def build_status_view(build_obj):
     }
   }
 
-  if can_write and build_obj.resource_key is not None:
-    resp['archive_url'] = user_files.get_file_url(build_obj.resource_key, requires_cors=True)
+  if can_write:
+    if build_obj.resource_key is not None:
+      resp['archive_url'] = user_files.get_file_url(build_obj.resource_key, requires_cors=True)
+    elif job_config.get('archive_url', None):
+      resp['archive_url'] = job_config['archive_url']
 
   return resp
 
@@ -148,14 +153,15 @@ class RepositoryBuildList(RepositoryParamResource):
     'RepositoryBuildRequest': {
       'type': 'object',
       'description': 'Description of a new repository build.',
-      'required': [
-        'file_id',
-      ],
       'properties': {
         'file_id': {
           'type': 'string',
           'description': 'The file id that was generated when the build spec was uploaded',
         },
+        'archive_url': {
+          'type': 'string',
+          'description': 'The URL of the .tar.gz to build. Must start with "http" or "https".',
+        },
         'subdirectory': {
           'type': 'string',
           'description': 'Subdirectory in which the Dockerfile can be found',
@@ -204,7 +210,26 @@ class RepositoryBuildList(RepositoryParamResource):
     logger.debug('User requested repository initialization.')
     request_json = request.get_json()
 
-    dockerfile_id = request_json['file_id']
+    dockerfile_id = request_json.get('file_id', None)
+    archive_url = request_json.get('archive_url', None)
+
+    if not dockerfile_id and not archive_url:
+      raise InvalidRequest('file_id or archive_url required')
+
+    if archive_url:
+      archive_match = None
+      try:
+        archive_match = uri_parse(archive_url, 'URI')
+      except ValueError:
+        pass
+
+      if not archive_match:
+        raise InvalidRequest('Invalid Archive URL: Must be a valid URI')
+
+      scheme = archive_match.get('scheme', None)
+      if scheme != 'http' and scheme != 'https':
+        raise InvalidRequest('Invalid Archive URL: Must be http or https')
+
     subdir = request_json['subdirectory'] if 'subdirectory' in request_json else ''
     tags = request_json.get('docker_tags', ['latest'])
     pull_robot_name = request_json.get('pull_robot', None)
@@ -228,18 +253,24 @@ class RepositoryBuildList(RepositoryParamResource):
     # Check if the dockerfile resource has already been used. If so, then it
     # can only be reused if the user has access to the repository in which the
     # dockerfile was previously built.
-    associated_repository = model.build.get_repository_for_resource(dockerfile_id)
-    if associated_repository:
-      if not ModifyRepositoryPermission(associated_repository.namespace_user.username,
-                                        associated_repository.name):
-        raise Unauthorized()
+    if dockerfile_id:
+      associated_repository = model.build.get_repository_for_resource(dockerfile_id)
+      if associated_repository:
+        if not ModifyRepositoryPermission(associated_repository.namespace_user.username,
+                                          associated_repository.name):
+          raise Unauthorized()
 
     # Start the build.
     repo = model.repository.get_repository(namespace, repository)
 
+    build_name =  (user_files.get_file_checksum(dockerfile_id)
+                    if dockerfile_id
+                    else hashlib.sha224(archive_url).hexdigest()[0:7])
+
     prepared = PreparedBuild()
-    prepared.build_name = user_files.get_file_checksum(dockerfile_id)
+    prepared.build_name = build_name
     prepared.dockerfile_id = dockerfile_id
+    prepared.archive_url = archive_url
     prepared.tags = tags
     prepared.subdirectory = subdir
     prepared.is_manual = True

endpoints/api/organization.py

@@ -278,6 +278,46 @@ class OrganizationMemberList(ApiResource):
 class OrganizationMember(ApiResource):
   """ Resource for managing individual organization members. """
 
+  @require_scope(scopes.ORG_ADMIN)
+  @nickname('getOrganizationMember')
+  def get(self, orgname, membername):
+    """ Retrieves the details of a member of the organization.
+    """
+    permission = AdministerOrganizationPermission(orgname)
+    if permission.can():
+      # Lookup the user.
+      member = model.user.get_user(membername)
+      if not member:
+        raise NotFound()
+
+      organization = model.user.get_user_or_org(orgname)
+      if not organization:
+        raise NotFound()
+
+      # Lookup the user's information in the organization.
+      teams = list(model.team.get_user_teams_within_org(membername, organization))
+      if not teams:
+        raise NotFound()
+
+      repo_permissions = model.permission.list_organization_member_permissions(organization, member)
+
+      def local_team_view(team):
+        return {
+          'name': team.name,
+          'avatar': avatar.get_data_for_team(team),
+        }
+
+      return {
+        'name': member.username,
+        'kind': 'robot' if member.robot else 'user',
+        'avatar': avatar.get_data_for_user(member),
+        'teams': [local_team_view(team) for team in teams],
+        'repositories': [permission.repository.name for permission in repo_permissions]
+      }
+
+    raise Unauthorized()
+
+
   @require_scope(scopes.ORG_ADMIN)
   @nickname('removeOrganizationMember')
   def delete(self, orgname, membername):

endpoints/api/repositorynotification.py

@@ -26,7 +26,8 @@ def notification_view(note):
     'uuid': note.uuid,
     'event': note.event.name,
     'method': note.method.name,
-    'config': config
+    'config': config,
+    'title': note.title,
   }
 
 
@@ -55,7 +56,11 @@ class RepositoryNotificationList(RepositoryParamResource):
         'config': {
           'type': 'object',
           'description': 'JSON config information for the specific method of notification'
-        }
+        },
+        'title': {
+          'type': 'string',
+          'description': 'The human-readable title of the notification',
+        },
       }
     },
   }
@@ -78,7 +83,8 @@ class RepositoryNotificationList(RepositoryParamResource):
       raise request_error(message=ex.message)
 
     new_notification = model.notification.create_repo_notification(repo, parsed['event'],
-                                                             parsed['method'], parsed['config'])
+                                                             parsed['method'], parsed['config'],
+                                                             parsed.get('title', None))
 
     resp = notification_view(new_notification)
     log_action('add_repo_notification', namespace,

endpoints/api/trigger.py

@@ -461,6 +461,7 @@ class TriggerBuildList(RepositoryParamResource):
     }
 
 
+FIELD_VALUE_LIMIT = 30
 
 @resource('/v1/repository/<repopath:repository>/trigger/<trigger_uuid>/fields/<field_name>')
 @internal_only
@@ -479,7 +480,7 @@ class BuildTriggerFieldValues(RepositoryParamResource):
     user_permission = UserAdminPermission(trigger.connected_user.username)
     if user_permission.can():
       handler = BuildTriggerHandler.get_handler(trigger, config)
-      values = handler.list_field_values(field_name)
+      values = handler.list_field_values(field_name, limit=FIELD_VALUE_LIMIT)
 
       if values is None:
         raise NotFound()

endpoints/building.py

@@ -28,7 +28,8 @@ def start_build(repository, prepared_build, pull_robot_name=None):
     'build_subdir': prepared_build.subdirectory,
     'trigger_metadata': prepared_build.metadata or {},
     'is_manual': prepared_build.is_manual,
-    'manual_user': get_authenticated_user().username if get_authenticated_user() else None
+    'manual_user': get_authenticated_user().username if get_authenticated_user() else None,
+    'archive_url': prepared_build.archive_url
   }
 
   with app.config['DB_TRANSACTION_FACTORY'](db):
@@ -83,6 +84,7 @@ class PreparedBuild(object):
   """
   def __init__(self, trigger=None):
     self._dockerfile_id = None
+    self._archive_url = None
     self._tags = None
     self._build_name = None
     self._subdirectory = None
@@ -124,6 +126,17 @@ class PreparedBuild(object):
   def trigger(self):
     return self._trigger
 
+  @property
+  def archive_url(self):
+    return self._archive_url
+
+  @archive_url.setter
+  def archive_url(self, value):
+    if self._archive_url:
+      raise Exception('Property archive_url already set')
+
+    self._archive_url = value
+
   @property
   def dockerfile_id(self):
     return self._dockerfile_id

endpoints/notificationmethod.py

@@ -4,8 +4,9 @@ import requests
 import re
 
 from flask.ext.mail import Message
-from app import mail, app
+from app import mail, app, OVERRIDE_CONFIG_DIRECTORY
 from data import model
+from util.config.validator import SSL_FILENAMES
 from workers.queueworker import JobException
 
 logger = logging.getLogger(__name__)
@@ -20,6 +21,11 @@ class NotificationMethodPerformException(JobException):
   pass
 
 
+SSLClientCert = None
+if app.config['PREFERRED_URL_SCHEME'] == 'https':
+  # TODO(jschorr): move this into the config provider library
+  SSLClientCert = [OVERRIDE_CONFIG_DIRECTORY + f for f in SSL_FILENAMES]
+
 class NotificationMethod(object):
   def __init__(self):
     pass
@@ -177,7 +183,7 @@ class WebhookMethod(NotificationMethod):
     headers = {'Content-type': 'application/json'}
 
     try:
-      resp = requests.post(url, data=json.dumps(payload), headers=headers)
+      resp = requests.post(url, data=json.dumps(payload), headers=headers, cert=SSLClientCert)
       if resp.status_code/100 != 2:
         error_message = '%s response for webhook to url: %s' % (resp.status_code, url)
         logger.error(error_message)

endpoints/trigger.py

@@ -197,7 +197,7 @@ class BuildTriggerHandler(object):
     """
     raise NotImplementedError
 
-  def list_field_values(self, field_name):
+  def list_field_values(self, field_name, limit=None):
     """
     Lists all values for the given custom trigger field. For example, a trigger might have a
     field named "branches", and this method would return all branches.
@@ -434,7 +434,7 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
 
     return data
 
-  def list_field_values(self, field_name):
+  def list_field_values(self, field_name, limit=None):
     source = self.config['build_source']
     (namespace, name) = source.split('/')
 
@@ -457,14 +457,22 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
       if not result:
         return None
 
-      return data.keys()
+      tags = list(data.keys())
+      if limit:
+        tags = tags[0:limit]
+
+      return tags
 
     if field_name == 'branch_name':
       (result, data, _) = repository.get_branches()
       if not result:
         return None
 
-      return data.keys()
+      branches = list(data.keys())
+      if limit:
+        branches = branches[0:limit]
+
+      return branches
 
     return None
 
@@ -548,7 +556,7 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
 
   def handle_trigger_request(self, request):
     payload = request.get_json()
-    if not 'push' in payload:
+    if not payload or not 'push' in payload:
       logger.debug('Skipping BitBucket request due to missing push data in payload')
       raise SkipRequestException()
 
@@ -1039,7 +1047,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
     return self._prepare_build(ref, commit_sha, True, repo=repo)
 
 
-  def list_field_values(self, field_name):
+  def list_field_values(self, field_name, limit=None):
     if field_name == 'refs':
       branches = self.list_field_values('branch_name')
       tags = self.list_field_values('tag_name')
@@ -1053,7 +1061,11 @@ class GithubBuildTrigger(BuildTriggerHandler):
         gh_client = self._get_client()
         source = config['build_source']
         repo = gh_client.get_repo(source)
-        return [tag.name for tag in repo.get_tags()]
+        gh_tags = repo.get_tags()
+        if limit:
+          gh_tags = repo.get_tags()[0:limit]
+
+        return [tag.name for tag in gh_tags]
       except GitHubBadCredentialsException:
         return []
       except GithubException:
@@ -1066,7 +1078,11 @@ class GithubBuildTrigger(BuildTriggerHandler):
         gh_client = self._get_client()
         source = config['build_source']
         repo = gh_client.get_repo(source)
-        branches = [branch.name for branch in repo.get_branches()]
+        gh_branches = repo.get_branches()
+        if limit:
+          gh_branches = repo.get_branches()[0:limit]
+
+        branches = [branch.name for branch in gh_branches]
 
         if not repo.default_branch in branches:
           branches.insert(0, repo.default_branch)
@@ -1417,7 +1433,7 @@ class GitLabBuildTrigger(BuildTriggerHandler):
 
     return contents
 
-  def list_field_values(self, field_name):
+  def list_field_values(self, field_name, limit=None):
     if field_name == 'refs':
       branches = self.list_field_values('branch_name')
       tags = self.list_field_values('tag_name')
@@ -1434,12 +1450,20 @@ class GitLabBuildTrigger(BuildTriggerHandler):
       tags = gl_client.getrepositorytags(repo['id'])
       if tags is False:
         return []
+
+      if limit:
+        tags = tags[0:limit]
+
       return [tag['name'] for tag in tags]
 
     if field_name == 'branch_name':
       branches = gl_client.getbranches(repo['id'])
       if branches is False:
         return []
+
+      if limit:
+        branches = branches[0:limit]
+
       return [branch['name'] for branch in branches]
 
     return None

endpoints/v1/__init__.py

@@ -1,10 +1,13 @@
 from flask import Blueprint, make_response
 
+from app import metric_queue
 from endpoints.decorators import anon_protect, anon_allowed
+from util.saas.metricqueue import time_blueprint
 
 
 v1_bp = Blueprint('v1', __name__)
 
+time_blueprint(v1_bp, metric_queue)
 
 # Note: This is *not* part of the Docker index spec. This is here for our own health check,
 # since we have nginx handle the _ping below.

endpoints/v1/registry.py

@@ -1,12 +1,13 @@
 import logging
 import json
+import features
 
 from flask import make_response, request, session, Response, redirect, abort as flask_abort
 from functools import wraps
 from datetime import datetime
 from time import time
 
-from app import storage as store, image_diff_queue, app
+from app import storage as store, image_diff_queue, image_replication_queue, app
 from auth.auth import process_auth, extract_namespace_repo_from_session
 from auth.auth_context import get_authenticated_user, get_grant_user_context
 from digest import checksums
@@ -36,6 +37,30 @@ def set_uploading_flag(repo_image, is_image_uploading):
   repo_image.storage.save()
 
 
+def _finish_image(namespace, repository, repo_image):
+  # Checksum is ok, we remove the marker
+  set_uploading_flag(repo_image, False)
+
+  image_id = repo_image.docker_image_id
+
+  # The layer is ready for download, send a job to the work queue to
+  # process it.
+  logger.debug('Adding layer to diff queue')
+  repo = model.repository.get_repository(namespace, repository)
+  image_diff_queue.put([repo.namespace_user.username, repository, image_id], json.dumps({
+    'namespace_user_id': repo.namespace_user.id,
+    'repository': repository,
+    'image_id': image_id,
+  }))
+
+  # Send a job to the work queue to replicate the image layer.
+  if features.STORAGE_REPLICATION:
+    image_replication_queue.put([repo_image.storage.uuid], json.dumps({
+      'namespace_user_id': repo.namespace_user.id,
+      'storage_id': repo_image.storage.uuid,
+    }))
+
+
 def require_completion(f):
   """This make sure that the image push correctly finished."""
   @wraps(f)
@@ -210,7 +235,11 @@ def put_image_layer(namespace, repository, image_id):
 
   # Stream write the data to storage.
   with database.CloseForLongOperation(app.config):
-    store.stream_write(repo_image.storage.locations, layer_path, sr)
+    try:
+      store.stream_write(repo_image.storage.locations, layer_path, sr)
+    except IOError:
+      logger.exception('Exception when writing image data')
+      abort(520, 'Image %(image_id)s could not be written. Please try again.', image_id=image_id)
 
   # Append the computed checksum.
   csums = []
@@ -243,18 +272,8 @@ def put_image_layer(namespace, repository, image_id):
     abort(400, 'Checksum mismatch; ignoring the layer for image %(image_id)s',
           issue='checksum-mismatch', image_id=image_id)
 
-  # Checksum is ok, we remove the marker
-  set_uploading_flag(repo_image, False)
-
-  # The layer is ready for download, send a job to the work queue to
-  # process it.
-  logger.debug('Adding layer to diff queue')
-  repo = model.repository.get_repository(namespace, repository)
-  image_diff_queue.put([repo.namespace_user.username, repository, image_id], json.dumps({
-    'namespace_user_id': repo.namespace_user.id,
-    'repository': repository,
-    'image_id': image_id,
-  }))
+  # Mark the image as uploaded.
+  _finish_image(namespace, repository, repo_image)
 
   return make_response('true', 200)
 
@@ -316,18 +335,8 @@ def put_image_checksum(namespace, repository, image_id):
     abort(400, 'Checksum mismatch for image: %(image_id)s',
           issue='checksum-mismatch', image_id=image_id)
 
-  # Checksum is ok, we remove the marker
-  set_uploading_flag(repo_image, False)
-
-  # The layer is ready for download, send a job to the work queue to
-  # process it.
-  logger.debug('Adding layer to diff queue')
-  repo = model.repository.get_repository(namespace, repository)
-  image_diff_queue.put([repo.namespace_user.username, repository, image_id], json.dumps({
-    'namespace_user_id': repo.namespace_user.id,
-    'repository': repository,
-    'image_id': image_id,
-  }))
+  # Mark the image as uploaded.
+  _finish_image(namespace, repository, repo_image)
 
   return make_response('true', 200)
 

endpoints/v2/__init__.py

@@ -7,6 +7,7 @@ from flask import Blueprint, make_response, url_for, request, jsonify
 from functools import wraps
 from urlparse import urlparse
 
+from app import metric_queue
 from endpoints.decorators import anon_protect, anon_allowed
 from endpoints.v2.errors import V2RegistryException
 from auth.jwt_auth import process_jwt_auth
@@ -15,13 +16,14 @@ from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermissi
                               AdministerRepositoryPermission)
 from data import model
 from util.http import abort
+from util.saas.metricqueue import time_blueprint
 from app import app
 
 
-
 logger = logging.getLogger(__name__)
 v2_bp = Blueprint('v2', __name__)
 
+time_blueprint(v2_bp, metric_queue)
 
 @v2_bp.app_errorhandler(V2RegistryException)
 def handle_registry_v2_exception(error):

initdb.py

@@ -20,6 +20,7 @@ from data.database import (db, all_models, Role, TeamRole, Visibility, LoginServ
                            ExternalNotificationEvent, ExternalNotificationMethod, NotificationKind)
 from data import model
 from app import app, storage as store
+from storage.basestorage import StoragePaths
 
 from workers import repositoryactioncounter
 
@@ -85,6 +86,17 @@ def __create_subtree(repo, structure, creator_username, parent, tag_map):
     new_image.storage.checksum = checksum
     new_image.storage.save()
 
+    # Write some data for the storage.
+    if os.environ.get('WRITE_STORAGE_FILES'):
+      storage_paths = StoragePaths()
+      paths = [storage_paths.image_json_path,
+               storage_paths.image_ancestry_path,
+               storage_paths.image_layer_path]
+
+      for path_builder in paths:
+        path = path_builder(new_image.storage.uuid)
+        store.put_content('local_us', path, checksum)
+
     creation_time = REFERENCE_DATE + timedelta(weeks=image_num) + timedelta(days=model_num)
     command_list = SAMPLE_CMDS[image_num % len(SAMPLE_CMDS)]
     command = json.dumps(command_list) if command_list else None

local-docker.sh

@@ -1,11 +1,30 @@
-#!/bin/sh
-
-# Run this from the quay directory to start a quay development instance in
-# docker on port 5000.
+#!/bin/bash
 
 set -e
 
 REPO=quay.io/quay/quay-dev
 
-docker build -t $REPO -f dev.df .
-docker run -it -p 5000:5000 -v $(pwd)/..:/src $REPO bash /src/quay/local-run.sh
+d ()
+{
+	docker build -t $REPO -f dev.df .
+	docker -- run --rm -it --net=host -v $(pwd)/..:/src $REPO $*
+}
+
+case $1 in
+buildman)
+	d /venv/bin/python -m buildman.builder
+	;;
+dev)
+	d bash /src/quay/local-run.sh
+	;;
+notifications)
+	d /venv/bin/python -m workers.notificationworker
+	;;
+test)
+	d bash /src/quay/local-test.sh
+	;;
+*)
+	echo "unknown option"
+	exit 1
+	;;
+esac

requirements-nover.txt

@@ -54,3 +54,4 @@ Flask-Testing
 pyjwt
 toposort
 pyjwkest
+rfc3987

requirements.txt

@@ -58,7 +58,7 @@ pycparser==2.14
 pycrypto==2.6.1
 pygpgme==0.3
 pyjwkest==1.0.3
-PyJWT==1.3.0
+PyJWT==1.4.0
 PyMySQL==0.6.6
 pyOpenSSL==0.15.1
 PyPDF2==1.24
@@ -74,6 +74,7 @@ redis==2.10.3
 reportlab==2.7
 requests==2.7.0
 requests-oauthlib==0.5.0
+rfc3987==1.3.4
 simplejson==3.7.3
 six==1.9.0
 SQLAlchemy==1.0.6

static/css/core-ui.css

@@ -1254,3 +1254,14 @@ a:focus {
   color: white;
   z-index: 2;
 }
+
+.co-alert.thin {
+  padding: 6px;
+  padding-left: 38px;
+  margin-bottom: 0px;
+}
+
+.co-alert.thin:before {
+  top: 5px;
+  font-size: 18px;
+}

static/css/directives/ui/dockerfile-build-dialog.css

@@ -0,0 +1,13 @@
+.dockerfile-build-dialog-element .btn-group {
+    margin-bottom: 20px;
+}
+
+.dockerfile-build-dialog-element button i {
+  margin-right: 6px;
+}
+
+.dockerfile-build-dialog-element .trigger-list {
+  margin: 0px;
+  width: 100%;
+}
+

static/css/directives/ui/dockerfile-build-form.css

@@ -3,6 +3,10 @@
   white-space: nowrap;
 }
 
+.dockerfile-build-form .file-drop {
+  padding: 0px;
+}
+
 .dockerfile-build-form input[type="file"] {
   margin: 0px;
 }
@@ -10,6 +14,11 @@
 .dockerfile-build-form .help-text {
   font-size: 13px;
   color: #aaa;
-  margin-bottom: 20px;
-  padding-left: 22px;
+  margin-top: 10px;
+  margin-bottom: 16px;
+}
+
+.dockerfile-build-form dd {
+  padding-left: 20px;
+  padding-top: 14px;
 }

static/css/pages/new-organization.css

@@ -2,3 +2,15 @@
   padding: 30px;
   position: relative;
 }
+
+.new-organization .field-container {
+   display: inline-block;
+   width: 400px;
+   margin-right: 10px;
+}
+
+.new-organization .field-row .co-alert {
+  display: inline-block;
+  margin-left: 10px;
+  margin-top: 10px;
+}

static/directives/build-logs-view.html

@@ -20,7 +20,8 @@
   </div>
 
   <div class="co-alert co-alert-danger" ng-if="loadError == 'request-failed'">
-    Failed to log builds logs. Please reload and try again.
+    Failed to load builds logs. Please reload and try again. If this problem persists,
+    please check for JavaScript or networking issues and contact support.
   </div>
 
   <span class="no-logs" ng-if="!logEntries.length && currentBuild.phase == 'waiting'">

static/directives/config/config-setup-tool.html

@@ -226,6 +226,12 @@
                          ng-model="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]">
                   <label for="dsc-{{ field.name }}">{{ field.placeholder }}</label>
                 </div>
+                <div ng-if="field.kind == 'option'">
+                  <select ng-model="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]">
+                    <option ng-repeat="value in field.values" value="{{ value }}"
+                            ng-selected="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name] == value">{{ value }}</option>
+                  </select>
+                </div>
                 <div class="help-text" ng-if="field.help_url">
                   See <a href="{{ field.help_url }}" target="_blank">Documentation</a> for more information
                 </div>

static/directives/create-external-notification-dialog.html

@@ -11,14 +11,14 @@
         </div>
         <div class="modal-body">
           <!-- Creating spinner -->
-          <div class="quay-spinner" ng-show="status == 'creating' || status == 'authorizing-email'"></div>
+          <div class="cor-loader" ng-show="status == 'creating' || status == 'authorizing-email'"></div>
 
           <!-- Authorize e-mail view -->
           <div ng-show="status == 'authorizing-email-sent'">
             An e-mail has been sent to <code>{{ currentConfig.email }}</code>. Please click the link contained
             in the e-mail.
             <br><br>
-            Waiting... <span class="quay-spinner"></span>
+            <span class="cor-loader-inline"></span>
           </div>
 
           <!-- Authorize e-mail view -->
@@ -30,6 +30,14 @@
 
           <!-- Create View -->
           <table style="width: 100%" ng-show="status == ''">
+            <tr>
+              <td style="width: 120px">Notification title:</td>
+              <td style="padding-right: 21px;">
+                <input class="form-control" type="text" placeholder="(Optional Title)" ng-model="currentTitle"
+                       style="margin: 10px;">
+              </td>
+            </tr>
+
             <tr>
               <td style="width: 120px">When this occurs:</td>
               <td>

static/directives/dockerfile-build-dialog.html

@@ -1,27 +1,57 @@
 <div class="dockerfile-build-dialog-element">
   <!-- Modal message dialog -->
-  <div class="modal fade" id="dockerfilebuildModal">
+  <div class="modal fade dockerfilebuildModal">
     <div class="modal-dialog">
-      <div class="modal-content">
+      <div class="modal-content" ng-show="triggersResource && triggersResource.loading">
+        <div class="cor-loader"></div>
+      </div>
+
+      <div class="modal-content" ng-show="!triggersResource || !triggersResource.loading">
         <div class="modal-header">
           <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
           <h4 class="modal-title">
             Start new Dockerfile build
           </h4>
         </div>
-        <div class="modal-body token-dialog-body">
-          <div class="alert alert-danger" ng-show="errorMessage">
+        <div class="modal-body">
+          <div class="btn-group btn-group-sm" ng-show="triggers.length > 0">
+            <button class="btn" ng-class="viewTriggers ? 'btn-default' : 'btn-info active'" ng-click="showTriggers(false)">
+              <i class="fa fa-upload"></i>Upload Dockerfile
+            </button>
+            <button class="btn" ng-class="viewTriggers ? 'btn-info active' : 'btn-default'" ng-click="showTriggers(true)">
+              <i class="fa fa-flash"></i>Start Build Trigger
+            </button>
+          </div>
+
+          <div class="co-alert co-alert-danger" ng-show="errorMessage">
             {{ errorMessage }}
           </div>
-          <div class="dockerfile-build-form" repository="repository" upload-failed="handleBuildFailed(message)"
-               build-started="handleBuildStarted(build)" build-failed="handleBuildFailed(message)" start-now="startCounter"
-               is-ready="hasDockerfile" uploading="uploading" building="building"></div>
+
+          <!-- Upload Dockerfile -->
+          <div ng-show="!viewTriggers">
+            <div class="dockerfile-build-form" repository="repository" upload-failed="handleBuildFailed(message)"
+                 build-started="handleBuildStarted(build)" build-failed="handleBuildFailed(message)" start-now="startCounter"
+                 is-ready="hasDockerfile" uploading="uploading" building="building"></div>
+          </div>
+
+          <!-- Start Build Trigger -->
+          <div ng-show="viewTriggers">
+            <table class="trigger-list">
+              <tr ng-repeat="trigger in triggers">
+                <td><span class="trigger-description" trigger="trigger"></span></td>
+                <td><button class="btn btn-primary" ng-click="runTriggerNow(trigger)">Run Trigger</button></td>
+              </tr>
+            </table>
+          </div>
         </div>
         <div class="modal-footer">
-          <button type="button" class="btn btn-primary" ng-click="startBuild()" ng-disabled="building || uploading || !hasDockerfile">Start Build</button>
+          <button type="button" class="btn btn-primary" ng-click="startBuild()" ng-disabled="building || uploading || !hasDockerfile" ng-show="!viewTriggers">Start Build</button>
           <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
         </div>
       </div><!-- /.modal-content -->
     </div><!-- /.modal-dialog -->
   </div><!-- /.modal -->
+  <div class="manual-trigger-build-dialog" repository="repository" counter="startTriggerCounter"
+       trigger="startTrigger"
+       build-started="handleBuildStarted(build)"></div>
 </div>

static/directives/dockerfile-build-form.html

@@ -12,35 +12,35 @@
     <dl>
       <dt>Dockerfile or <code>.tar.gz</code> or <code>.zip</code>:</dt>
       <dd>
-        <input id="file-drop" class="file-drop" type="file" file-present="internal.hasDockerfile">
-        <div class="help-text">If an archive, the Dockerfile must be at the root</div>
+        <div class="co-alert co-alert-danger" ng-if="dockerfileState == 'error'">
+          {{ dockerfileError }}
+        </div>
+
+        <input id="file-drop" class="file-drop" type="file" files-changed="handleFilesChanged(files)">
+        <div class="help-text">Note: If an archive, the Dockerfile must be in the root directory.</div>
+        <div ng-if="dockerfileState == 'loading'">
+         Reading Dockerfile: <span class="cor-loader-inline"></span>
+        </div>
       </dd>
     </dl>
 
-    <dl>
+    <dl ng-show="privateBaseRepository">
       <dt>Base Image Pull Credentials:</dt>
-      <dd style="margin: 20px;">
-        <!-- Select credentials -->
-        <div class="btn-group btn-group-sm">
-          <button type="button" class="btn btn-default"
-                  ng-class="is_public ? 'active btn-info' : ''"
-                  ng-click="is_public = true">
-            None
-          </button>
-          <button type="button" class="btn btn-default"
-                  ng-class="is_public ? '' : 'active btn-info'"
-                  ng-click="is_public = false">
-            <i class="fa ci-robot"></i>
-            Robot account
-          </button>
+      <dd>
+        <div class="co-alert co-alert-warning"
+             ng-if="currentRobotHasPermission === false">
+          Warning: Robot account <strong>{{ pullEntity.name }}</strong> does not have
+          read permission on repository <strong>{{ privateBaseRepository }}</strong>, so
+          this build will fail with an authorization error.
         </div>
-
-        <!-- Robot Select -->
-        <div ng-show="!is_public" style="margin-top: 10px">
-          <div class="entity-search" namespace="repository.namespace"
-               placeholder="'Select robot account for pulling...'"
-               current-entity="pull_entity"
-               allowed-entities="['robot']"></div>
+        <div class="entity-search" namespace="repository.namespace"
+             placeholder="'Select robot account for pulling'"
+             current-entity="pullEntity"
+             allowed-entities="['robot']"></div>
+        <div class="help-text">
+          The selected Dockerfile contains a <code>FROM</code> that refers to the private
+          <span class="registry-name"></span> repository <strong>{{ privateBaseRepository }}</strong>.
+          A robot account with read access to that repository is required for the build.
         </div>
       </dd>
     </dl>

static/directives/manual-trigger-build-dialog.html

@@ -1,5 +1,5 @@
 <!-- Modal message dialog -->
-<div class="modal fade" id="startTriggerDialog">
+<div class="modal fade startTriggerDialog">
   <div class="modal-dialog">
     <div class="modal-content">
       <div class="modal-header">

static/directives/repo-view/repo-panel-builds.html

@@ -84,7 +84,7 @@
       <i class="fa fa-flash"></i>
       Build Triggers
 
-      <div class="heading-controls hidden-sm hidden-xs">
+      <div class="heading-controls hidden-xs">
         <!-- Add Build Trigger -->
         <div class="dropdown" id="addBuildTrigger">
           <button class="btn btn-primary dropdown-toggle" data-toggle="dropdown">
@@ -198,7 +198,7 @@
        repository="repository"
        trigger="currentStartTrigger"
        counter="showTriggerStartDialogCounter"
-       start-build="startTrigger(trigger, parameters)"></div>
+       build-started="handleBuildStarted(build)"></div>
 
   <!-- /Dialogs -->
 

static/directives/repository-events-table.html

@@ -26,6 +26,7 @@
           <table class="co-table permissions" ng-if="notifications.length">
             <thead>
               <tr>
+                <td>Title</td>
                 <td>Event</td>
                 <td>Notification</td>
                 <td class="options-col"></td>
@@ -34,6 +35,10 @@
 
             <tbody>
               <tr class="notification-row" ng-repeat="notification in notifications">
+                <td>
+                  {{ notification.title || '(Untitled)' }}
+                </td>
+
                 <td>
                   <span class="notification-event">
                     <i class="fa fa-lg" ng-class="getEventInfo(notification).icon"></i>
@@ -53,6 +58,16 @@
                     <span class="cor-option" option-click="testNotification(notification)">
                       <i class="fa fa-send"></i> Test Notification
                     </span>
+                    <span class="cor-option" option-click="showNotifyInfo(notification, 'url')"
+                          ng-if="getMethodInfo(notification).id == 'webhook'">
+                        <i class="fa fa-link"></i>
+                        View Webhook URL
+                    </span>
+                    <span class="cor-option" option-click="showNotifyInfo(notification, 'email')"
+                          ng-if="getMethodInfo(notification).id == 'email'">
+                        <i class="fa fa-envelope"></i>
+                        View E-mail Address
+                    </span>
                     <span class="cor-option" option-click="showWebhookInfo(notification)"
                           ng-if="getMethodInfo(notification).id == 'webhook'">
                         <i class="fa fa-book"></i>

static/directives/tour-content.html

@@ -70,17 +70,17 @@
     </div>
   </div>
 
-<div class="row testimonial" quay-require="['BILLING']">
-  <div class="tour-action" quay-require="['BILLING']">
-    <a href="/plans?trial-plan=personal">
-      <button class="btn btn-success">
-        Start free trial
-      </button>
-    </a>
+  <div class="row testimonial" quay-require="['BILLING']">
+    <div class="tour-action" quay-require="['BILLING']">
+      <a href="/plans?trial-plan=personal">
+        <button class="btn btn-success">
+          Start free trial
+        </button>
+      </a>
+    </div>
   </div>
 </div>
 
-
 <!-- Organizations -->
 <div class="product-tour" ng-if="kind == 'organizations'">
   <div class="tour-section row tour-header">