diff --git a/CHANGELOG.md b/CHANGELOG.md index 94c318e99..f59fc5333 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +### v1.11.2 + +- Fixed security bug with LDAP login (#376) + +### 1.11.1 + +- Loosened the check for mounted volumes bug (#353) +- Strengthened HTTPS configuration (#329) +- Disabled password change for non-DB auth (#347) +- Added support for custom favicon (#343) +- Fixed tarfile support for non-unicode pax fields (#328) +- Fixed permissions on tag history API requiring READ instead of WRITE tokens (#316) +- Added public access to time machine (#334) +- Added missing JSON schema for 'refs' and 'branch_name' (#330) +- Always create a new connection to Swift (#336) +- Minor UI Fixes (#356, #341, #338, #337) +- Minor trigger fixes (#357, #349) +- Refactored and fixed internal code (#331) + ### 1.11.0 - Changed user pages to display public repositories (#321) diff --git a/Dockerfile b/Dockerfile index 371abd262..ef834f2b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # vim:ft=dockerfile -FROM phusion/baseimage:0.9.16 +FROM phusion/baseimage:0.9.17 ENV DEBIAN_FRONTEND noninteractive ENV HOME /root @@ -42,6 +42,7 @@ ADD conf/init/copy_config_files.sh /etc/my_init.d/ ADD conf/init/doupdatelimits.sh /etc/my_init.d/ ADD conf/init/copy_syslog_config.sh /etc/my_init.d/ ADD conf/init/runmigration.sh /etc/my_init.d/ +ADD conf/init/syslog-ng.conf /etc/syslog-ng/ ADD conf/init/service/ /etc/service/ diff --git a/README.md b/README.md index c395801f4..e73ec0cfd 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Quay.io - container image registry +`master` branch build status: ![Docker Repository on Quay.io](https://quay.io/repository/quay/quay/status?token=7bffbc13-8bb0-4fb4-8a70-684a0cf485d3 "Docker Repository on Quay.io") + Quay.io is a container image registry with managements APIs, a Docker registry API, a container build system. The application is implemented as a set of API endpoints written in python and an Angular.js frontend. @@ -7,6 +9,42 @@ The application is implemented as a set of API endpoints written in python and a If you are doing local development on your workstation against the code base follow these instructions. +### Docker + +Quay and its parts can run inside of docker containers. +This method requires no installation of any python packages on your host machine. +The `local-docker.sh` script is provided to prepare and run parts of quay. +First, start redis: + + +``` +docker run -d -p 6379:6379 redis +``` + +And clone the configuration repo: + +``` +git clone git@github.com:coreos-inc/quay-config.git ../quay-config +ln -s ../../quay-config/local conf/stack +``` + +To build and run a docker container, pass one argument to local-docker.sh: + +- `dev`: run quay on port 5000 +- `buildman`: run the buildmanager +- `notifications`: run the notification worker +- `test`: run the unit tests + +For example: + +``` +./local-docker.sh dev +```` + +will start quay in a docker container. +Now quay will be running on: http://127.0.0.1:5000 +The username is `devtable` and the password is `password`. + ### OS X ``` @@ -15,8 +53,6 @@ cd quay ./contrib/osx/local-setup.sh ``` -## Running Development Environment - Now run the server; it will use sqlite as the SQL server. ``` diff --git a/app.py b/app.py index 15eb0a27b..5f54a9752 100644 --- a/app.py +++ b/app.py @@ -2,6 +2,7 @@ import logging import os import json +from functools import partial from flask import Flask, request, Request, _request_ctx_stack from flask.ext.principal import Principal from flask.ext.login import LoginManager, UserMixin @@ -20,17 +21,18 @@ from data.billing import Billing from data.buildlogs import BuildLogs from data.archivedlogs import LogArchive from data.userevent import UserEventsBuilderModule -from data.queue import WorkQueue +from data.queue import WorkQueue, MetricQueueReporter +from util import get_app_url from util.saas.analytics import Analytics from util.saas.exceptionlog import Sentry from util.names import urn_generator from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig from util.security.signing import Signer -from util.saas.queuemetrics import QueueMetrics +from util.saas.cloudwatch import start_cloudwatch_sender +from util.saas.metricqueue import MetricQueue from util.config.provider import FileConfigProvider, TestConfigProvider from util.config.configutil import generate_secret_key from util.config.superusermanager import SuperUserManager -from buildman.jobutil.buildreporter import BuildMetrics OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/' OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml' @@ -129,8 +131,8 @@ authentication = UserAuthentication(app, OVERRIDE_CONFIG_DIRECTORY) userevents = UserEventsBuilderModule(app) superusers = SuperUserManager(app) signer = Signer(app, OVERRIDE_CONFIG_DIRECTORY) -queue_metrics = QueueMetrics(app) -build_metrics = BuildMetrics(app) +metric_queue = MetricQueue() +start_cloudwatch_sender(metric_queue, app) tf = app.config['DB_TRANSACTION_FACTORY'] @@ -141,8 +143,9 @@ google_login = GoogleOAuthConfig(app.config, 'GOOGLE_LOGIN_CONFIG') oauth_apps = [github_login, github_trigger, gitlab_trigger, google_login] image_diff_queue = WorkQueue(app.config['DIFFS_QUEUE_NAME'], tf) +image_replication_queue = WorkQueue(app.config['REPLICATION_QUEUE_NAME'], tf) dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf, - reporter=queue_metrics.report) + reporter=MetricQueueReporter(metric_queue)) notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf) database.configure(app.config) @@ -173,5 +176,4 @@ class LoginWrappedDBUser(UserMixin): def get_id(self): return unicode(self._uuid) -def get_app_url(): - return '%s://%s' % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME']) +get_app_url = partial(get_app_url, app.config) diff --git a/auth/jwt_auth.py b/auth/jwt_auth.py index cd1a6ca31..9a4aa1bbe 100644 --- a/auth/jwt_auth.py +++ b/auth/jwt_auth.py @@ -1,5 +1,4 @@ import logging -import jwt import re from datetime import datetime, timedelta @@ -11,10 +10,11 @@ from cryptography.hazmat.backends import default_backend from cachetools import lru_cache from app import app -from auth_context import set_grant_user_context -from permissions import repository_read_grant, repository_write_grant +from .auth_context import set_grant_user_context +from .permissions import repository_read_grant, repository_write_grant from util.names import parse_namespace_repository from util.http import abort +from util.security import strictjwt logger = logging.getLogger(__name__) @@ -44,17 +44,14 @@ def identity_from_bearer_token(bearer_token, max_signed_s, public_key): # Load the JWT returned. try: - payload = jwt.decode(encoded, public_key, algorithms=['RS256'], audience='quay', - issuer='token-issuer') - except jwt.InvalidTokenError: + payload = strictjwt.decode(encoded, public_key, algorithms=['RS256'], audience='quay', + issuer='token-issuer') + except strictjwt.InvalidTokenError: raise InvalidJWTException('Invalid token') if not 'sub' in payload: raise InvalidJWTException('Missing sub field in JWT') - if not 'exp' in payload: - raise InvalidJWTException('Missing exp field in JWT') - # Verify that the expiration is no more than 300 seconds in the future. if datetime.fromtimestamp(payload['exp']) > datetime.utcnow() + timedelta(seconds=max_signed_s): raise InvalidJWTException('Token was signed for more than %s seconds' % max_signed_s) diff --git a/buildman/component/buildcomponent.py b/buildman/component/buildcomponent.py index a728020dd..a567910c6 100644 --- a/buildman/component/buildcomponent.py +++ b/buildman/component/buildcomponent.py @@ -116,14 +116,10 @@ class BuildComponent(BaseComponent): # push_token: The token to use to push the built image. # tag_names: The name(s) of the tag(s) for the newly built image. # base_image: The image name and credentials to use to conduct the base image pull. - # repository: The repository to pull (DEPRECATED 0.2) - # tag: The tag to pull (DEPRECATED in 0.2) # username: The username for pulling the base image (if any). # password: The password for pulling the base image (if any). build_arguments = { - 'build_package': self.user_files.get_file_url(build_job.repo_build.resource_key, - requires_cors=False) - if build_job.repo_build.resource_key is not None else "", + 'build_package': build_job.get_build_package_url(self.user_files), 'sub_directory': build_config.get('build_subdir', ''), 'repository': repository_name, 'registry': self.registry_hostname, diff --git a/buildman/jobutil/buildjob.py b/buildman/jobutil/buildjob.py index 5635f0622..f6291f62b 100644 --- a/buildman/jobutil/buildjob.py +++ b/buildman/jobutil/buildjob.py @@ -62,6 +62,17 @@ class BuildJob(object): def repo_build(self): return self._load_repo_build() + def get_build_package_url(self, user_files): + """ Returns the URL of the build package for this build, if any or empty string if none. """ + archive_url = self.build_config.get('archive_url', None) + if archive_url: + return archive_url + + if not self.repo_build.resource_key: + return '' + + return user_files.get_file_url(self.repo_build.resource_key, requires_cors=False) + @property def pull_credentials(self): """ Returns the pull credentials for this job, or None if none. """ diff --git a/buildman/jobutil/buildreporter.py b/buildman/jobutil/buildreporter.py deleted file mode 100644 index 553f62ee7..000000000 --- a/buildman/jobutil/buildreporter.py +++ /dev/null @@ -1,70 +0,0 @@ -from buildman.enums import BuildJobResult -from util.saas.cloudwatch import get_queue - - -class BuildReporter(object): - """ - Base class for reporting build statuses to a metrics service. - """ - def report_completion_status(self, status): - """ - Method to invoke the recording of build's completion status to a metric service. - """ - raise NotImplementedError - - -class NullReporter(BuildReporter): - """ - The /dev/null of BuildReporters. - """ - def report_completion_status(self, *args): - pass - - -class CloudWatchBuildReporter(BuildReporter): - """ - Implements a BuildReporter for Amazon's CloudWatch. - """ - def __init__(self, queue, namespace_name, completed_name, failed_name, incompleted_name): - self._queue = queue - self._namespace_name = namespace_name - self._completed_name = completed_name - self._failed_name = failed_name - self._incompleted_name = incompleted_name - - def _send_to_queue(self, *args, **kwargs): - self._queue.put((args, kwargs)) - - def report_completion_status(self, status): - if status == BuildJobResult.COMPLETE: - status_name = self._completed_name - elif status == BuildJobResult.ERROR: - status_name = self._failed_name - elif status == BuildJobResult.INCOMPLETE: - status_name = self._incompleted_name - else: - return - - self._send_to_queue(self._namespace_name, status_name, 1, unit='Count') - - -class BuildMetrics(object): - """ - BuildMetrics initializes a reporter for recording the status of build completions. - """ - def __init__(self, app=None): - self._app = app - self._reporter = NullReporter() - if app is not None: - reporter_type = app.config.get('BUILD_METRICS_TYPE', 'Null') - if reporter_type == 'CloudWatch': - namespace = app.config['BUILD_METRICS_NAMESPACE'] - completed_name = app.config['BUILD_METRICS_COMPLETED_NAME'] - failed_name = app.config['BUILD_METRICS_FAILED_NAME'] - incompleted_name = app.config['BUILD_METRICS_INCOMPLETED_NAME'] - request_queue = get_queue(app) - self._reporter = CloudWatchBuildReporter(request_queue, namespace, completed_name, - failed_name, incompleted_name) - - def __getattr__(self, name): - return getattr(self._reporter, name, None) diff --git a/buildman/server.py b/buildman/server.py index 655b78495..f2a4feb92 100644 --- a/buildman/server.py +++ b/buildman/server.py @@ -16,7 +16,7 @@ from buildman.enums import BuildJobResult, BuildServerStatus from buildman.jobutil.buildstatus import StatusHandler from buildman.jobutil.buildjob import BuildJob, BuildJobLoadException from data import database -from app import app, build_metrics +from app import app, metric_queue logger = logging.getLogger(__name__) @@ -151,7 +151,7 @@ class BuilderServer(object): if self._current_status == BuildServerStatus.SHUTDOWN and not self._job_count: self._shutdown_event.set() - build_metrics.report_completion_status(job_status) + report_completion_status(job_status) @trollius.coroutine def _work_checker(self): @@ -225,3 +225,15 @@ class BuilderServer(object): # Initialize the work queue checker. yield From(self._work_checker()) + +def report_completion_status(status): + if status == BuildJobResult.COMPLETE: + status_name = 'CompleteBuilds' + elif status == BuildJobResult.ERROR: + status_name = 'FailedBuilds' + elif status == BuildJobResult.INCOMPLETE: + status_name = 'IncompletedBuilds' + else: + return + + metric_queue.put(status_name, 1, unit='Count') diff --git a/buildman/templates/cloudconfig.yaml b/buildman/templates/cloudconfig.yaml index bb459d048..5a9946659 100644 --- a/buildman/templates/cloudconfig.yaml +++ b/buildman/templates/cloudconfig.yaml @@ -4,6 +4,9 @@ ssh_authorized_keys: - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCC0m+hVmyR3vn/xoxJe9+atRWBxSK+YXgyufNVDMcb7H00Jfnc341QH3kDVYZamUbhVh/nyc2RP7YbnZR5zORFtgOaNSdkMYrPozzBvxjnvSUokkCCWbLqXDHvIKiR12r+UTSijPJE/Yk702Mb2ejAFuae1C3Ec+qKAoOCagDjpQ3THyb5oaKE7VPHdwCWjWIQLRhC+plu77ObhoXIFJLD13gCi01L/rp4mYVCxIc2lX5A8rkK+bZHnIZwWUQ4t8SIjWxIaUo0FE7oZ83nKuNkYj5ngmLHQLY23Nx2WhE9H6NBthUpik9SmqQPtVYbhIG+bISPoH9Xs8CLrFb0VRjz Joey's Mac - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCo6FhAP7mFFOAzM91gtaKW7saahtaN4lur42FMMztz6aqUycIltCmvxo+3FmrXgCG30maMNU36Vm1+9QRtVQEd+eRuoIWP28t+8MT01Fh4zPuE2Wca3pOHSNo3X81FfWJLzmwEHiQKs9HPQqUhezR9PcVWVkbMyAzw85c0UycGmHGFNb0UiRd9HFY6XbgbxhZv/mvKLZ99xE3xkOzS1PNsdSNvjUKwZR7pSUPqNS5S/1NXyR4GhFTU24VPH/bTATOv2ATH+PSzsZ7Qyz9UHj38tKC+ALJHEDJ4HXGzobyOUP78cHGZOfCB5FYubq0zmOudAjKIAhwI8XTFvJ2DX1P3 jimmyzelinskie - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNvw8qo9m8np7yQ/Smv/oklM8bo8VyNRZriGYBDuolWDL/mZpYCQnZJXphQo7RFdNABYistikjJlBuuwUohLf2uSq0iKoFa2TgwI43wViWzvuzU4nA02/ITD5BZdmWAFNyIoqeB50Ol4qUgDwLAZ+7Kv7uCi6chcgr9gTi99jY3GHyZjrMiXMHGVGi+FExFuzhVC2drKjbz5q6oRfQeLtNfG4psl5GU3MQU6FkX4fgoCx0r9R48/b7l4+TT7pWblJQiRfeldixu6308vyoTUEHasdkU3/X0OTaGz/h5XqTKnGQc6stvvoED3w+L3QFp0H5Z8sZ9stSsitmCBrmbcKZ jakemoshenko +- ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAgEAo/JkbGO6R7g1ZxARi0xWVM7FOfN02snRAcIO6vT9M7xMUkWVLgD+hM/o91lk+UFiYdql0CATobpFWncRL36KaUqsbw9/1BlI40wg296XHXSSnxhxZ4L7ytf6G1tyN319HXlI2kh9vAf/fy++yDvkH8dI3k1oLoW+mZPET6Pff04/6AXXrRlS5mhmGv9irGwiDHtVKpj6lU8DN/UtOrv1tiQ0pgwEJq05fLGoQfgPNaBCnW2z4Ubpn2gyMcMBMpSwo4hCqJePd349e4bLmFcT+gXYg7Mnup1DoTDlowFFN56wpxQbdp96IxWzU+jYPaIAuRo+BJzCyOS8qBv0Z4RZrgop0qp2JYiVwmViO6TZhIDz6loQJXUOIleQmNgTbiZx8Bwv5GY2jMYoVwlBp7yy5bRjxfbFsJ0vU7TVzNAG7oEJy/74HmHmWzRQlSlQjesr8gRbm9zgR8wqc/L107UOWFg7Cgh8ZNjKuADbXqYuda1Y9m2upcfS26UPz5l5PW5uFRMHZSi8pb1XV6/0Z8H8vwsh37Ur6aLi/5jruRmKhdlsNrB1IiDicBsPW3yg7HHSIdPU4oBNPC77yDCT3l4CKr4el81RrZt7FbJPfY+Ig9Q5O+05f6I8+ZOlJGyZ/Qfyl2aVm1HnlJKuBqPxeic8tMng/9B5N7uZL6Y3k5jFU8c= quentin +- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDI7LtxLItapmUbt3Gs+4Oxa1i22fkx1+aJDkAjiRWPSX3+cxOzuPfHX9uFzr+qj5hy4J7ErrPp8q9alu+il9lE26GQuUxOZiaUrXu4dRCXXdCqTHARWBxGUXjkxdMp2HIzFpBxmVqcRubrgM36LBzKapdDOqQdz7XnNm5Jmf0tH/N0+TgV60P0WVY1CxmTya+JHNFVgazhd+oIGEhTyW/eszMGcFUgZet7DQFytYIQXYSwwGpGdJ+0InKAJ2SzCt/yuUlSrhrVM8vSGeami1XYmgQiyth1zjteMd8uTrc9NREH7bZTNcMFBqVYE3BYQWGRrv8pMMgP9gxgLbxtVsUl barakmich-titania +- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDUWB4aSjSRHCz5/6H9/EJhJVvRmPThvEzyHinaWPsuM9prBSLci9NF9WneVl30nczkvllA+w34kycdrS3fKpjTbODaEOLHBobWl3bccY0I6kr86q5z67NZffjCm/P/RL+dBaOiBWS8PV8oiDF1P6YdMo8Jk46n9fozmLCXHUuCw5BJ8PGjQqbsEzA3qFMeKZYdJHOizOfeIfKfCWYrrumVRY9v6SAUDoFOl4PZEM7QdGp9EoRYb9MNLgKLnZ4RjbcLoFwiqxY4KEM4zfjZPNOECiLCuJqvHM2QawwuO1klJ16HpJk+FzOTWQoZtT47LoE/XNSOcNtAOiD+OQ449ia1EArhm7+1DnLXvHXKIl1JtuqJz+wFCsbNSdB7P562OHAGRIxYK3DfE+0CZH1BeHYl7xiRBeCtZ+OZMIocqeJtq8taIS7Un5wnGcQWxFtQnr/f65EgbIi7G2dxPcjhr6K+GWYezsiReVVKnIClq2MHhABG9QOncKDIa47L3nyx3pm4ZfMbC2jmnK2pFgGGSfYDy4487JnAUOG1mzZ9vm4gDhatT+vZFSBOwv1e4CErBh/wYXooF5I0nGmE6y6zkKFqP+ZolJ6iXmXQ7Ea2oaGeyaprweBjkhHgghi4KbwKbClope4Zo9X9JJYBLQSW33sEEuy8MlSBpdZAbz9t/FvJaw== mjibson write_files: - path: /root/overrides.list diff --git a/conf/init/service/storagereplication/log/run b/conf/init/service/storagereplication/log/run new file mode 100755 index 000000000..adcd2b63f --- /dev/null +++ b/conf/init/service/storagereplication/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec logger -i -t storagereplication \ No newline at end of file diff --git a/conf/init/service/storagereplication/run b/conf/init/service/storagereplication/run new file mode 100755 index 000000000..ed62731f8 --- /dev/null +++ b/conf/init/service/storagereplication/run @@ -0,0 +1,8 @@ +#! /bin/bash + +echo 'Starting storage replication worker' + +cd / +venv/bin/python -m workers.storagereplication 2>&1 + +echo 'Repository storage replication exited' \ No newline at end of file diff --git a/conf/init/syslog-ng.conf b/conf/init/syslog-ng.conf new file mode 100644 index 000000000..678d89366 --- /dev/null +++ b/conf/init/syslog-ng.conf @@ -0,0 +1,143 @@ +@version: 3.5 +@include "scl.conf" +@include "`scl-root`/system/tty10.conf" + +# Syslog-ng configuration file, compatible with default Debian syslogd +# installation. + +# First, set some global options. +options { chain_hostnames(off); flush_lines(0); use_dns(no); use_fqdn(no); + owner("root"); group("adm"); perm(0640); stats_freq(0); + bad_hostname("^gconfd$"); +}; + +######################## +# Sources +######################## +# This is the default behavior of sysklogd package +# Logs may come from unix stream, but not from another machine. +# +source s_src { + unix-stream("/dev/log"); + internal(); +}; + +# If you wish to get logs from remote machine you should uncomment +# this and comment the above source line. +# +#source s_net { tcp(ip(127.0.0.1) port(1000)); }; + +######################## +# Destinations +######################## +# First some standard logfile +# +destination d_auth { file("/var/log/auth.log"); }; +destination d_cron { file("/var/log/cron.log"); }; +destination d_daemon { file("/var/log/daemon.log"); }; +destination d_kern { file("/var/log/kern.log"); }; +destination d_lpr { file("/var/log/lpr.log"); }; +destination d_mail { file("/var/log/mail.log"); }; +destination d_syslog { file("/var/log/syslog"); }; +destination d_user { file("/var/log/user.log"); }; +destination d_uucp { file("/var/log/uucp.log"); }; + +# This files are the log come from the mail subsystem. +# +destination d_mailinfo { file("/var/log/mail.info"); }; +destination d_mailwarn { file("/var/log/mail.warn"); }; +destination d_mailerr { file("/var/log/mail.err"); }; + +# Logging for INN news system +# +destination d_newscrit { file("/var/log/news/news.crit"); }; +destination d_newserr { file("/var/log/news/news.err"); }; +destination d_newsnotice { file("/var/log/news/news.notice"); }; + +# Some `catch-all' logfiles. +# +destination d_debug { file("/var/log/debug"); }; +destination d_error { file("/var/log/error"); }; +destination d_messages { file("/var/log/messages"); }; + +# The named pipe /dev/xconsole is for the nsole' utility. To use it, +# you must invoke nsole' with the -file' option: +# +# $ xconsole -file /dev/xconsole [...] +# +destination d_xconsole { pipe("/dev/xconsole"); }; + +# Send the messages to an other host +# +#destination d_net { tcp("127.0.0.1" port(1000) log_fifo_size(1000)); }; + +# Debian only +destination d_ppp { file("/var/log/ppp.log"); }; + +######################## +# Filters +######################## +# Here's come the filter options. With this rules, we can set which +# message go where. + +filter f_dbg { level(debug); }; +filter f_info { level(info); }; +filter f_notice { level(notice); }; +filter f_warn { level(warn); }; +filter f_err { level(err); }; +filter f_crit { level(crit .. emerg); }; + +filter f_debug { level(debug) and not facility(auth, authpriv, news, mail); }; +filter f_error { level(err .. emerg) ; }; + +filter f_auth { facility(auth, authpriv) and not filter(f_debug); }; +filter f_cron { facility(cron) and not filter(f_debug); }; +filter f_daemon { facility(daemon) and not filter(f_debug); }; +filter f_kern { facility(kern) and not filter(f_debug); }; +filter f_lpr { facility(lpr) and not filter(f_debug); }; +filter f_local { facility(local0, local1, local3, local4, local5, + local6, local7) and not filter(f_debug); }; +filter f_mail { facility(mail) and not filter(f_debug); }; +filter f_news { facility(news) and not filter(f_debug); }; +filter f_syslog3 { not facility(auth, authpriv, mail) and not filter(f_debug); }; +filter f_uucp { facility(uucp) and not filter(f_debug); }; + +filter f_cnews { level(notice, err, crit) and facility(news); }; +filter f_cother { level(debug, info, notice, warn) or facility(daemon, mail); }; + +filter f_ppp { facility(local2) and not filter(f_debug); }; +filter f_console { level(warn .. emerg); }; + +######################## +# Log paths +######################## +log { source(s_src); filter(f_auth); destination(d_auth); }; +log { source(s_src); filter(f_cron); destination(d_cron); }; +log { source(s_src); filter(f_daemon); destination(d_daemon); }; +log { source(s_src); filter(f_kern); destination(d_kern); }; +log { source(s_src); filter(f_lpr); destination(d_lpr); }; +log { source(s_src); filter(f_syslog3); destination(d_syslog); }; +log { source(s_src); filter(f_uucp); destination(d_uucp); }; + +log { source(s_src); filter(f_mail); destination(d_mail); }; +#log { source(s_src); filter(f_mail); filter(f_info); destination(d_mailinfo); }; +#log { source(s_src); filter(f_mail); filter(f_warn); destination(d_mailwarn); }; +#log { source(s_src); filter(f_mail); filter(f_err); destination(d_mailerr); }; + +log { source(s_src); filter(f_news); filter(f_crit); destination(d_newscrit); }; +log { source(s_src); filter(f_news); filter(f_err); destination(d_newserr); }; +log { source(s_src); filter(f_news); filter(f_notice); destination(d_newsnotice); }; + +#log { source(s_src); filter(f_ppp); destination(d_ppp); }; + +log { source(s_src); filter(f_debug); destination(d_debug); }; +log { source(s_src); filter(f_error); destination(d_error); }; + +# All messages send to a remote site +# +#log { source(s_src); destination(d_net); }; + +### +# Include all config files in /etc/syslog-ng/conf.d/ +### +@include "/etc/syslog-ng/conf.d/*.conf" diff --git a/config.py b/config.py index 7163b3cd6..cd199a031 100644 --- a/config.py +++ b/config.py @@ -20,7 +20,7 @@ CLIENT_WHITELIST = ['SERVER_HOSTNAME', 'PREFERRED_URL_SCHEME', 'MIXPANEL_KEY', 'STRIPE_PUBLISHABLE_KEY', 'ENTERPRISE_LOGO_URL', 'SENTRY_PUBLIC_DSN', 'AUTHENTICATION_TYPE', 'REGISTRY_TITLE', 'REGISTRY_TITLE_SHORT', 'CONTACT_INFO', 'AVATAR_KIND', 'LOCAL_OAUTH_HANDLER', 'DOCUMENTATION_LOCATION', - 'DOCUMENTATION_METADATA'] + 'DOCUMENTATION_METADATA', 'SETUP_COMPLETE'] def frontend_visible_config(config_dict): @@ -129,6 +129,7 @@ class DefaultConfig(object): NOTIFICATION_QUEUE_NAME = 'notification' DIFFS_QUEUE_NAME = 'imagediff' DOCKERFILE_BUILD_QUEUE_NAME = 'dockerfilebuild' + REPLICATION_QUEUE_NAME = 'imagestoragereplication' # Super user config. Note: This MUST BE an empty list for the default config. SUPER_USERS = [] @@ -179,6 +180,9 @@ class DefaultConfig(object): # basic auth. FEATURE_REQUIRE_ENCRYPTED_BASIC_AUTH = False + # Feature Flag: Whether to automatically replicate between storage engines. + FEATURE_STORAGE_REPLICATION = False + BUILD_MANAGER = ('enterprise', {}) DISTRIBUTED_STORAGE_CONFIG = { @@ -187,6 +191,7 @@ class DefaultConfig(object): } DISTRIBUTED_STORAGE_PREFERENCE = ['local_us'] + DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS = ['local_us'] # Health checker. HEALTH_CHECKER = ('LocalHealthCheck', {}) diff --git a/data/archivedlogs.py b/data/archivedlogs.py index e190b9782..fd532c3ca 100644 --- a/data/archivedlogs.py +++ b/data/archivedlogs.py @@ -1,8 +1,7 @@ import logging -from gzip import GzipFile +from util.registry.gzipinputstream import GzipInputStream from flask import send_file, abort -from cStringIO import StringIO from data.userfiles import DelegateUserfiles, UserfilesHandlers @@ -17,10 +16,8 @@ class LogArchiveHandlers(UserfilesHandlers): def get(self, file_id): path = self._files.get_file_id_path(file_id) try: - with self._storage.stream_read_file(self._locations, path) as gzip_stream: - with GzipFile(fileobj=gzip_stream) as unzipped: - unzipped_buffer = StringIO(unzipped.read()) - return send_file(unzipped_buffer, mimetype=JSON_MIMETYPE) + data_stream = self._storage.stream_read_file(self._locations, path) + return send_file(GzipInputStream(data_stream), mimetype=JSON_MIMETYPE) except IOError: abort(404) diff --git a/data/billing.py b/data/billing.py index 26f47ed73..7ac7b9f45 100644 --- a/data/billing.py +++ b/data/billing.py @@ -17,6 +17,7 @@ PLANS = [ 'deprecated': True, 'free_trial_days': 14, 'superseded_by': None, + 'plans_page_hidden': False, }, { 'title': 'Basic', @@ -28,6 +29,7 @@ PLANS = [ 'deprecated': True, 'free_trial_days': 14, 'superseded_by': None, + 'plans_page_hidden': False, }, { 'title': 'Yacht', @@ -39,6 +41,7 @@ PLANS = [ 'deprecated': True, 'free_trial_days': 180, 'superseded_by': 'bus-small-30', + 'plans_page_hidden': False, }, { 'title': 'Personal', @@ -50,6 +53,7 @@ PLANS = [ 'deprecated': True, 'free_trial_days': 14, 'superseded_by': 'personal-30', + 'plans_page_hidden': False, }, { 'title': 'Skiff', @@ -61,6 +65,7 @@ PLANS = [ 'deprecated': True, 'free_trial_days': 14, 'superseded_by': 'bus-micro-30', + 'plans_page_hidden': False, }, { 'title': 'Yacht', @@ -72,6 +77,7 @@ PLANS = [ 'deprecated': True, 'free_trial_days': 14, 'superseded_by': 'bus-small-30', + 'plans_page_hidden': False, }, { 'title': 'Freighter', @@ -83,6 +89,7 @@ PLANS = [ 'deprecated': True, 'free_trial_days': 14, 'superseded_by': 'bus-medium-30', + 'plans_page_hidden': False, }, { 'title': 'Tanker', @@ -94,6 +101,7 @@ PLANS = [ 'deprecated': True, 'free_trial_days': 14, 'superseded_by': 'bus-large-30', + 'plans_page_hidden': False, }, # Active plans @@ -107,6 +115,7 @@ PLANS = [ 'deprecated': False, 'free_trial_days': 30, 'superseded_by': None, + 'plans_page_hidden': False, }, { 'title': 'Personal', @@ -118,6 +127,7 @@ PLANS = [ 'deprecated': False, 'free_trial_days': 30, 'superseded_by': None, + 'plans_page_hidden': False, }, { 'title': 'Skiff', @@ -129,6 +139,7 @@ PLANS = [ 'deprecated': False, 'free_trial_days': 30, 'superseded_by': None, + 'plans_page_hidden': False, }, { 'title': 'Yacht', @@ -140,6 +151,7 @@ PLANS = [ 'deprecated': False, 'free_trial_days': 30, 'superseded_by': None, + 'plans_page_hidden': False, }, { 'title': 'Freighter', @@ -151,6 +163,7 @@ PLANS = [ 'deprecated': False, 'free_trial_days': 30, 'superseded_by': None, + 'plans_page_hidden': False, }, { 'title': 'Tanker', @@ -162,6 +175,19 @@ PLANS = [ 'deprecated': False, 'free_trial_days': 30, 'superseded_by': None, + 'plans_page_hidden': False, + }, + { + 'title': 'Carrier', + 'price': 35000, + 'privateRepos': 250, + 'stripeId': 'bus-xlarge-30', + 'audience': 'For extra large businesses', + 'bus_features': True, + 'deprecated': False, + 'free_trial_days': 30, + 'superseded_by': None, + 'plans_page_hidden': True, }, ] diff --git a/data/database.py b/data/database.py index cad01b4ac..369649db3 100644 --- a/data/database.py +++ b/data/database.py @@ -544,6 +544,15 @@ class ImageStoragePlacement(BaseModel): ) +class UserRegion(BaseModel): + user = QuayUserField(index=True, allows_robots=False) + location = ForeignKeyField(ImageStorageLocation) + + indexes = ( + (('user', 'location'), True), + ) + + class Image(BaseModel): # This class is intentionally denormalized. Even though images are supposed # to be globally unique we can't treat them as such for permissions and @@ -733,6 +742,7 @@ class RepositoryNotification(BaseModel): repository = ForeignKeyField(Repository, index=True) event = ForeignKeyField(ExternalNotificationEvent) method = ForeignKeyField(ExternalNotificationMethod) + title = CharField(null=True) config_json = TextField() @@ -777,4 +787,4 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification, RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage, TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind, - AccessTokenKind, Star, RepositoryActionCount, TagManifest, BlobUpload] + AccessTokenKind, Star, RepositoryActionCount, TagManifest, BlobUpload, UserRegion] diff --git a/data/migrations/versions/499f6f08de3_add_title_field_to_notification.py b/data/migrations/versions/499f6f08de3_add_title_field_to_notification.py new file mode 100644 index 000000000..7d203b176 --- /dev/null +++ b/data/migrations/versions/499f6f08de3_add_title_field_to_notification.py @@ -0,0 +1,26 @@ +"""Add title field to notification + +Revision ID: 499f6f08de3 +Revises: 246df01a6d51 +Create Date: 2015-08-21 14:18:07.287743 + +""" + +# revision identifiers, used by Alembic. +revision = '499f6f08de3' +down_revision = '246df01a6d51' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('repositorynotification', sa.Column('title', sa.String(length=255), nullable=True)) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('repositorynotification', 'title') + ### end Alembic commands ### diff --git a/data/migrations/versions/9512773a4a2_add_userregion_table.py b/data/migrations/versions/9512773a4a2_add_userregion_table.py new file mode 100644 index 000000000..212110054 --- /dev/null +++ b/data/migrations/versions/9512773a4a2_add_userregion_table.py @@ -0,0 +1,35 @@ +"""Add UserRegion table + +Revision ID: 9512773a4a2 +Revises: 499f6f08de3 +Create Date: 2015-09-01 14:17:08.628052 + +""" + +# revision identifiers, used by Alembic. +revision = '9512773a4a2' +down_revision = '499f6f08de3' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('userregion', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('location_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['location_id'], ['imagestoragelocation.id'], name=op.f('fk_userregion_location_id_imagestoragelocation')), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], name=op.f('fk_userregion_user_id_user')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_userregion')) + ) + op.create_index('userregion_location_id', 'userregion', ['location_id'], unique=False) + op.create_index('userregion_user_id', 'userregion', ['user_id'], unique=False) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('userregion') + ### end Alembic commands ### diff --git a/data/model/notification.py b/data/model/notification.py index b894a1cf7..87ae7f7ca 100644 --- a/data/model/notification.py +++ b/data/model/notification.py @@ -113,12 +113,12 @@ def delete_matching_notifications(target, kind_name, **kwargs): notification.delete_instance() -def create_repo_notification(repo, event_name, method_name, config): +def create_repo_notification(repo, event_name, method_name, config, title=None): event = ExternalNotificationEvent.get(ExternalNotificationEvent.name == event_name) method = ExternalNotificationMethod.get(ExternalNotificationMethod.name == method_name) return RepositoryNotification.create(repository=repo, event=event, method=method, - config_json=json.dumps(config)) + config_json=json.dumps(config), title=title) def get_repo_notification(uuid): diff --git a/data/model/oauth.py b/data/model/oauth.py index 8c3fb5624..bdec2f7a1 100644 --- a/data/model/oauth.py +++ b/data/model/oauth.py @@ -8,8 +8,9 @@ from oauth2lib import utils from data.database import (OAuthApplication, OAuthAuthorizationCode, OAuthAccessToken, User, AccessToken, random_string_generator) -from data.model import user +from data.model import user, config from auth import scopes +from util import get_app_url logger = logging.getLogger(__name__) @@ -45,7 +46,10 @@ class DatabaseAuthorizationProvider(AuthorizationProvider): return False def validate_redirect_uri(self, client_id, redirect_uri): - if redirect_uri == url_for('web.oauth_local_handler', _external=True): + internal_redirect_url = '%s%s' % (get_app_url(config.app_config), + url_for('web.oauth_local_handler')) + + if redirect_uri == internal_redirect_url: return True try: diff --git a/data/model/permission.py b/data/model/permission.py index 52dcf40f1..340ab3835 100644 --- a/data/model/permission.py +++ b/data/model/permission.py @@ -17,14 +17,19 @@ def list_robot_permissions(robot_name): .where(User.username == robot_name, User.robot == True)) -def list_organization_member_permissions(organization): +def list_organization_member_permissions(organization, limit_to_user=None): query = (RepositoryPermission .select(RepositoryPermission, Repository, User) .join(Repository) .switch(RepositoryPermission) .join(User) - .where(Repository.namespace_user == organization) - .where(User.robot == False)) + .where(Repository.namespace_user == organization)) + + if limit_to_user is not None: + query = query.where(RepositoryPermission.user == limit_to_user) + else: + query = query.where(User.robot == False) + return query diff --git a/data/model/storage.py b/data/model/storage.py index c697c5de8..4b44a2005 100644 --- a/data/model/storage.py +++ b/data/model/storage.py @@ -11,6 +11,12 @@ from data.database import (ImageStorage, Image, DerivedImageStorage, ImageStorag logger = logging.getLogger(__name__) +def add_storage_placement(storage, location_name): + """ Adds a storage placement for the given storage at the given location. """ + location = ImageStorageLocation.get(name=location_name) + ImageStoragePlacement.create(location=location, storage=storage) + + def find_or_create_derived_storage(source, transformation_name, preferred_location): existing = find_derived_storage(source, transformation_name) if existing is not None: diff --git a/data/model/user.py b/data/model/user.py index e5b34a099..1a7709ec7 100644 --- a/data/model/user.py +++ b/data/model/user.py @@ -8,7 +8,8 @@ from datetime import datetime, timedelta from data.database import (User, LoginService, FederatedLogin, RepositoryPermission, TeamMember, Team, Repository, TupleSelector, TeamRole, Namespace, Visibility, - EmailConfirmation, Role, db_for_update, random_string_generator) + EmailConfirmation, Role, db_for_update, random_string_generator, + UserRegion, ImageStorageLocation) from data.model import (DataModelException, InvalidPasswordException, InvalidRobotException, InvalidUsernameException, InvalidEmailAddressException, TooManyUsersException, TooManyLoginAttemptsException, db_transaction, @@ -463,6 +464,13 @@ def get_user_by_id(user_db_id): return None +def get_namespace_user_by_user_id(namespace_user_db_id): + try: + return User.get(User.id == namespace_user_db_id, User.robot == False) + except User.DoesNotExist: + raise InvalidUsernameException('User with id does not exist: %s' % namespace_user_db_id) + + def get_namespace_by_user_id(namespace_user_db_id): try: return User.get(User.id == namespace_user_db_id, User.robot == False).username @@ -664,3 +672,8 @@ def get_pull_credentials(robotname): 'registry': '%s://%s/v1/' % (config.app_config['PREFERRED_URL_SCHEME'], config.app_config['SERVER_HOSTNAME']), } + +def get_region_locations(user): + """ Returns the locations defined as preferred storage for the given user. """ + query = UserRegion.select().join(ImageStorageLocation).where(UserRegion.user == user) + return set([region.location.name for region in query]) diff --git a/data/queue.py b/data/queue.py index 60632f5b1..289b99ada 100644 --- a/data/queue.py +++ b/data/queue.py @@ -13,6 +13,17 @@ class NoopWith: def __exit__(self, type, value, traceback): pass +class MetricQueueReporter(object): + def __init__(self, metric_queue): + self._metric_queue = metric_queue + + def __call__(self, currently_processing, running_count, total_count): + need_capacity_count = total_count - running_count + self._metric_queue.put('BuildCapacityShortage', need_capacity_count, unit='Count') + + building_percent = 100 if currently_processing else 0 + self._metric_queue.put('PercentBuilding', building_percent, unit='Percent') + class WorkQueue(object): def __init__(self, queue_name, transaction_factory, canonical_name_match_list=None, reporter=None): diff --git a/data/users/externaljwt.py b/data/users/externaljwt.py index 241cfa947..ac29f22a1 100644 --- a/data/users/externaljwt.py +++ b/data/users/externaljwt.py @@ -1,13 +1,15 @@ import logging import json import os -import jwt from datetime import datetime, timedelta from data.users.federated import FederatedUsers, VerifiedCredentials +from util.security import strictjwt + logger = logging.getLogger(__name__) + class ExternalJWTAuthN(FederatedUsers): """ Delegates authentication to a REST endpoint that returns JWTs. """ PUBLIC_KEY_FILENAME = 'jwt-authn.cert' @@ -45,9 +47,9 @@ class ExternalJWTAuthN(FederatedUsers): # Load the JWT returned. encoded = result_data.get('token', '') try: - payload = jwt.decode(encoded, self.public_key, algorithms=['RS256'], - audience='quay.io/jwtauthn', issuer=self.issuer) - except jwt.InvalidTokenError: + payload = strictjwt.decode(encoded, self.public_key, algorithms=['RS256'], + audience='quay.io/jwtauthn', issuer=self.issuer) + except strictjwt.InvalidTokenError: logger.exception('Exception when decoding returned JWT') return (None, 'Invalid username or password') diff --git a/data/users/externalldap.py b/data/users/externalldap.py index 9a488b283..1fef02396 100644 --- a/data/users/externalldap.py +++ b/data/users/externalldap.py @@ -9,6 +9,16 @@ from data.users.federated import FederatedUsers, VerifiedCredentials logger = logging.getLogger(__name__) +class LDAPConnectionBuilder(object): + def __init__(self, ldap_uri, user_dn, user_pw): + self._ldap_uri = ldap_uri + self._user_dn = user_dn + self._user_pw = user_pw + + def get_connection(self): + return LDAPConnection(self._ldap_uri, self._user_dn, self._user_pw) + + class LDAPConnection(object): def __init__(self, ldap_uri, user_dn, user_pw): self._ldap_uri = ldap_uri @@ -20,13 +30,7 @@ class LDAPConnection(object): trace_level = 2 if os.environ.get('USERS_DEBUG') == '1' else 0 self._conn = ldap.initialize(self._ldap_uri, trace_level=trace_level) self._conn.set_option(ldap.OPT_REFERRALS, 1) - - try: - self._conn.simple_bind_s(self._user_dn, self._user_pw) - except ldap.INVALID_CREDENTIALS: - logger.exception('LDAP admin dn or password are invalid') - return None - + self._conn.simple_bind_s(self._user_dn, self._user_pw) return self._conn def __exit__(self, exc_type, value, tb): @@ -38,7 +42,7 @@ class LDAPUsers(FederatedUsers): def __init__(self, ldap_uri, base_dn, admin_dn, admin_passwd, user_rdn, uid_attr, email_attr): super(LDAPUsers, self).__init__('ldap') - self._ldap_conn = LDAPConnection(ldap_uri, admin_dn, admin_passwd) + self._ldap = LDAPConnectionBuilder(ldap_uri, admin_dn, admin_passwd) self._ldap_uri = ldap_uri self._base_dn = base_dn self._user_rdn = user_rdn @@ -65,10 +69,15 @@ class LDAPUsers(FederatedUsers): return referral_dn def _ldap_user_search(self, username_or_email): - with self._ldap_conn as conn: - if conn is None: - return (None, 'LDAP Admin dn or password is invalid') + # Verify the admin connection works first. We do this here to avoid wrapping + # the entire block in the INVALID CREDENTIALS check. + try: + with self._ldap.get_connection(): + pass + except ldap.INVALID_CREDENTIALS: + return (None, 'LDAP Admin dn or password is invalid') + with self._ldap.get_connection() as conn: logger.debug('Incoming username or email param: %s', username_or_email.__repr__()) user_search_dn = ','.join(self._user_rdn + self._base_dn) query = u'(|({0}={2})({1}={2}))'.format(self._uid_attr, self._email_attr, diff --git a/dev.df b/dev.df index f99c4cc61..683018080 100644 --- a/dev.df +++ b/dev.df @@ -18,4 +18,4 @@ RUN venv/bin/pip install -r requirements.txt WORKDIR /src/quay ENV PYTHONPATH=/ -ENV PATH=$PATH:/venv/bin +ENV PATH=/venv/bin:$PATH diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index fc16723d1..d8c2a9e66 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -1,7 +1,7 @@ import logging import datetime -from app import app +from app import app, metric_queue from flask import Blueprint, request, make_response, jsonify, session from flask.ext.restful import Resource, abort, Api, reqparse from flask.ext.restful.utils.cors import crossdomain @@ -20,6 +20,7 @@ from auth.auth_context import get_authenticated_user, get_validated_oauth_token from auth.auth import process_oauth from endpoints.csrf import csrf_protect from endpoints.decorators import check_anon_protection +from util.saas.metricqueue import time_decorator logger = logging.getLogger(__name__) @@ -28,7 +29,7 @@ api = Api() api.init_app(api_bp) api.decorators = [csrf_protect, crossdomain(origin='*', headers=['Authorization', 'Content-Type']), - process_oauth] + process_oauth, time_decorator(api_bp.name, metric_queue)] class ApiException(Exception): diff --git a/endpoints/api/build.py b/endpoints/api/build.py index 21002f4a3..397e40ac2 100644 --- a/endpoints/api/build.py +++ b/endpoints/api/build.py @@ -3,8 +3,10 @@ import logging import json import datetime +import hashlib from flask import request +from rfc3987 import parse as uri_parse from app import app, userfiles as user_files, build_logs, log_archive, dockerfile_build_queue from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource, @@ -134,8 +136,11 @@ def build_status_view(build_obj): } } - if can_write and build_obj.resource_key is not None: - resp['archive_url'] = user_files.get_file_url(build_obj.resource_key, requires_cors=True) + if can_write: + if build_obj.resource_key is not None: + resp['archive_url'] = user_files.get_file_url(build_obj.resource_key, requires_cors=True) + elif job_config.get('archive_url', None): + resp['archive_url'] = job_config['archive_url'] return resp @@ -148,14 +153,15 @@ class RepositoryBuildList(RepositoryParamResource): 'RepositoryBuildRequest': { 'type': 'object', 'description': 'Description of a new repository build.', - 'required': [ - 'file_id', - ], 'properties': { 'file_id': { 'type': 'string', 'description': 'The file id that was generated when the build spec was uploaded', }, + 'archive_url': { + 'type': 'string', + 'description': 'The URL of the .tar.gz to build. Must start with "http" or "https".', + }, 'subdirectory': { 'type': 'string', 'description': 'Subdirectory in which the Dockerfile can be found', @@ -204,7 +210,26 @@ class RepositoryBuildList(RepositoryParamResource): logger.debug('User requested repository initialization.') request_json = request.get_json() - dockerfile_id = request_json['file_id'] + dockerfile_id = request_json.get('file_id', None) + archive_url = request_json.get('archive_url', None) + + if not dockerfile_id and not archive_url: + raise InvalidRequest('file_id or archive_url required') + + if archive_url: + archive_match = None + try: + archive_match = uri_parse(archive_url, 'URI') + except ValueError: + pass + + if not archive_match: + raise InvalidRequest('Invalid Archive URL: Must be a valid URI') + + scheme = archive_match.get('scheme', None) + if scheme != 'http' and scheme != 'https': + raise InvalidRequest('Invalid Archive URL: Must be http or https') + subdir = request_json['subdirectory'] if 'subdirectory' in request_json else '' tags = request_json.get('docker_tags', ['latest']) pull_robot_name = request_json.get('pull_robot', None) @@ -228,18 +253,24 @@ class RepositoryBuildList(RepositoryParamResource): # Check if the dockerfile resource has already been used. If so, then it # can only be reused if the user has access to the repository in which the # dockerfile was previously built. - associated_repository = model.build.get_repository_for_resource(dockerfile_id) - if associated_repository: - if not ModifyRepositoryPermission(associated_repository.namespace_user.username, - associated_repository.name): - raise Unauthorized() + if dockerfile_id: + associated_repository = model.build.get_repository_for_resource(dockerfile_id) + if associated_repository: + if not ModifyRepositoryPermission(associated_repository.namespace_user.username, + associated_repository.name): + raise Unauthorized() # Start the build. repo = model.repository.get_repository(namespace, repository) + build_name = (user_files.get_file_checksum(dockerfile_id) + if dockerfile_id + else hashlib.sha224(archive_url).hexdigest()[0:7]) + prepared = PreparedBuild() - prepared.build_name = user_files.get_file_checksum(dockerfile_id) + prepared.build_name = build_name prepared.dockerfile_id = dockerfile_id + prepared.archive_url = archive_url prepared.tags = tags prepared.subdirectory = subdir prepared.is_manual = True diff --git a/endpoints/api/organization.py b/endpoints/api/organization.py index 55da710a0..818851caf 100644 --- a/endpoints/api/organization.py +++ b/endpoints/api/organization.py @@ -278,6 +278,46 @@ class OrganizationMemberList(ApiResource): class OrganizationMember(ApiResource): """ Resource for managing individual organization members. """ + @require_scope(scopes.ORG_ADMIN) + @nickname('getOrganizationMember') + def get(self, orgname, membername): + """ Retrieves the details of a member of the organization. + """ + permission = AdministerOrganizationPermission(orgname) + if permission.can(): + # Lookup the user. + member = model.user.get_user(membername) + if not member: + raise NotFound() + + organization = model.user.get_user_or_org(orgname) + if not organization: + raise NotFound() + + # Lookup the user's information in the organization. + teams = list(model.team.get_user_teams_within_org(membername, organization)) + if not teams: + raise NotFound() + + repo_permissions = model.permission.list_organization_member_permissions(organization, member) + + def local_team_view(team): + return { + 'name': team.name, + 'avatar': avatar.get_data_for_team(team), + } + + return { + 'name': member.username, + 'kind': 'robot' if member.robot else 'user', + 'avatar': avatar.get_data_for_user(member), + 'teams': [local_team_view(team) for team in teams], + 'repositories': [permission.repository.name for permission in repo_permissions] + } + + raise Unauthorized() + + @require_scope(scopes.ORG_ADMIN) @nickname('removeOrganizationMember') def delete(self, orgname, membername): diff --git a/endpoints/api/repositorynotification.py b/endpoints/api/repositorynotification.py index 54643ccc3..832328cbe 100644 --- a/endpoints/api/repositorynotification.py +++ b/endpoints/api/repositorynotification.py @@ -26,7 +26,8 @@ def notification_view(note): 'uuid': note.uuid, 'event': note.event.name, 'method': note.method.name, - 'config': config + 'config': config, + 'title': note.title, } @@ -55,7 +56,11 @@ class RepositoryNotificationList(RepositoryParamResource): 'config': { 'type': 'object', 'description': 'JSON config information for the specific method of notification' - } + }, + 'title': { + 'type': 'string', + 'description': 'The human-readable title of the notification', + }, } }, } @@ -78,7 +83,8 @@ class RepositoryNotificationList(RepositoryParamResource): raise request_error(message=ex.message) new_notification = model.notification.create_repo_notification(repo, parsed['event'], - parsed['method'], parsed['config']) + parsed['method'], parsed['config'], + parsed.get('title', None)) resp = notification_view(new_notification) log_action('add_repo_notification', namespace, diff --git a/endpoints/api/trigger.py b/endpoints/api/trigger.py index 72d1ee7a2..1582f3890 100644 --- a/endpoints/api/trigger.py +++ b/endpoints/api/trigger.py @@ -461,6 +461,7 @@ class TriggerBuildList(RepositoryParamResource): } +FIELD_VALUE_LIMIT = 30 @resource('/v1/repository//trigger//fields/') @internal_only @@ -479,7 +480,7 @@ class BuildTriggerFieldValues(RepositoryParamResource): user_permission = UserAdminPermission(trigger.connected_user.username) if user_permission.can(): handler = BuildTriggerHandler.get_handler(trigger, config) - values = handler.list_field_values(field_name) + values = handler.list_field_values(field_name, limit=FIELD_VALUE_LIMIT) if values is None: raise NotFound() diff --git a/endpoints/building.py b/endpoints/building.py index 02d89a323..93d76be7b 100644 --- a/endpoints/building.py +++ b/endpoints/building.py @@ -28,7 +28,8 @@ def start_build(repository, prepared_build, pull_robot_name=None): 'build_subdir': prepared_build.subdirectory, 'trigger_metadata': prepared_build.metadata or {}, 'is_manual': prepared_build.is_manual, - 'manual_user': get_authenticated_user().username if get_authenticated_user() else None + 'manual_user': get_authenticated_user().username if get_authenticated_user() else None, + 'archive_url': prepared_build.archive_url } with app.config['DB_TRANSACTION_FACTORY'](db): @@ -83,6 +84,7 @@ class PreparedBuild(object): """ def __init__(self, trigger=None): self._dockerfile_id = None + self._archive_url = None self._tags = None self._build_name = None self._subdirectory = None @@ -124,6 +126,17 @@ class PreparedBuild(object): def trigger(self): return self._trigger + @property + def archive_url(self): + return self._archive_url + + @archive_url.setter + def archive_url(self, value): + if self._archive_url: + raise Exception('Property archive_url already set') + + self._archive_url = value + @property def dockerfile_id(self): return self._dockerfile_id diff --git a/endpoints/notificationmethod.py b/endpoints/notificationmethod.py index 526a25199..d3e444fc1 100644 --- a/endpoints/notificationmethod.py +++ b/endpoints/notificationmethod.py @@ -4,8 +4,9 @@ import requests import re from flask.ext.mail import Message -from app import mail, app +from app import mail, app, OVERRIDE_CONFIG_DIRECTORY from data import model +from util.config.validator import SSL_FILENAMES from workers.queueworker import JobException logger = logging.getLogger(__name__) @@ -20,6 +21,11 @@ class NotificationMethodPerformException(JobException): pass +SSLClientCert = None +if app.config['PREFERRED_URL_SCHEME'] == 'https': + # TODO(jschorr): move this into the config provider library + SSLClientCert = [OVERRIDE_CONFIG_DIRECTORY + f for f in SSL_FILENAMES] + class NotificationMethod(object): def __init__(self): pass @@ -177,7 +183,7 @@ class WebhookMethod(NotificationMethod): headers = {'Content-type': 'application/json'} try: - resp = requests.post(url, data=json.dumps(payload), headers=headers) + resp = requests.post(url, data=json.dumps(payload), headers=headers, cert=SSLClientCert) if resp.status_code/100 != 2: error_message = '%s response for webhook to url: %s' % (resp.status_code, url) logger.error(error_message) diff --git a/endpoints/trigger.py b/endpoints/trigger.py index c4bc63912..5e174cdb9 100644 --- a/endpoints/trigger.py +++ b/endpoints/trigger.py @@ -197,7 +197,7 @@ class BuildTriggerHandler(object): """ raise NotImplementedError - def list_field_values(self, field_name): + def list_field_values(self, field_name, limit=None): """ Lists all values for the given custom trigger field. For example, a trigger might have a field named "branches", and this method would return all branches. @@ -434,7 +434,7 @@ class BitbucketBuildTrigger(BuildTriggerHandler): return data - def list_field_values(self, field_name): + def list_field_values(self, field_name, limit=None): source = self.config['build_source'] (namespace, name) = source.split('/') @@ -457,14 +457,22 @@ class BitbucketBuildTrigger(BuildTriggerHandler): if not result: return None - return data.keys() + tags = list(data.keys()) + if limit: + tags = tags[0:limit] + + return tags if field_name == 'branch_name': (result, data, _) = repository.get_branches() if not result: return None - return data.keys() + branches = list(data.keys()) + if limit: + branches = branches[0:limit] + + return branches return None @@ -548,7 +556,7 @@ class BitbucketBuildTrigger(BuildTriggerHandler): def handle_trigger_request(self, request): payload = request.get_json() - if not 'push' in payload: + if not payload or not 'push' in payload: logger.debug('Skipping BitBucket request due to missing push data in payload') raise SkipRequestException() @@ -1039,7 +1047,7 @@ class GithubBuildTrigger(BuildTriggerHandler): return self._prepare_build(ref, commit_sha, True, repo=repo) - def list_field_values(self, field_name): + def list_field_values(self, field_name, limit=None): if field_name == 'refs': branches = self.list_field_values('branch_name') tags = self.list_field_values('tag_name') @@ -1053,7 +1061,11 @@ class GithubBuildTrigger(BuildTriggerHandler): gh_client = self._get_client() source = config['build_source'] repo = gh_client.get_repo(source) - return [tag.name for tag in repo.get_tags()] + gh_tags = repo.get_tags() + if limit: + gh_tags = repo.get_tags()[0:limit] + + return [tag.name for tag in gh_tags] except GitHubBadCredentialsException: return [] except GithubException: @@ -1066,7 +1078,11 @@ class GithubBuildTrigger(BuildTriggerHandler): gh_client = self._get_client() source = config['build_source'] repo = gh_client.get_repo(source) - branches = [branch.name for branch in repo.get_branches()] + gh_branches = repo.get_branches() + if limit: + gh_branches = repo.get_branches()[0:limit] + + branches = [branch.name for branch in gh_branches] if not repo.default_branch in branches: branches.insert(0, repo.default_branch) @@ -1417,7 +1433,7 @@ class GitLabBuildTrigger(BuildTriggerHandler): return contents - def list_field_values(self, field_name): + def list_field_values(self, field_name, limit=None): if field_name == 'refs': branches = self.list_field_values('branch_name') tags = self.list_field_values('tag_name') @@ -1434,12 +1450,20 @@ class GitLabBuildTrigger(BuildTriggerHandler): tags = gl_client.getrepositorytags(repo['id']) if tags is False: return [] + + if limit: + tags = tags[0:limit] + return [tag['name'] for tag in tags] if field_name == 'branch_name': branches = gl_client.getbranches(repo['id']) if branches is False: return [] + + if limit: + branches = branches[0:limit] + return [branch['name'] for branch in branches] return None diff --git a/endpoints/v1/__init__.py b/endpoints/v1/__init__.py index 587d0dbec..3eabbe338 100644 --- a/endpoints/v1/__init__.py +++ b/endpoints/v1/__init__.py @@ -1,10 +1,13 @@ from flask import Blueprint, make_response +from app import metric_queue from endpoints.decorators import anon_protect, anon_allowed +from util.saas.metricqueue import time_blueprint v1_bp = Blueprint('v1', __name__) +time_blueprint(v1_bp, metric_queue) # Note: This is *not* part of the Docker index spec. This is here for our own health check, # since we have nginx handle the _ping below. diff --git a/endpoints/v1/registry.py b/endpoints/v1/registry.py index 484cacb7c..f38e2f66d 100644 --- a/endpoints/v1/registry.py +++ b/endpoints/v1/registry.py @@ -1,12 +1,13 @@ import logging import json +import features from flask import make_response, request, session, Response, redirect, abort as flask_abort from functools import wraps from datetime import datetime from time import time -from app import storage as store, image_diff_queue, app +from app import storage as store, image_diff_queue, image_replication_queue, app from auth.auth import process_auth, extract_namespace_repo_from_session from auth.auth_context import get_authenticated_user, get_grant_user_context from digest import checksums @@ -36,6 +37,30 @@ def set_uploading_flag(repo_image, is_image_uploading): repo_image.storage.save() +def _finish_image(namespace, repository, repo_image): + # Checksum is ok, we remove the marker + set_uploading_flag(repo_image, False) + + image_id = repo_image.docker_image_id + + # The layer is ready for download, send a job to the work queue to + # process it. + logger.debug('Adding layer to diff queue') + repo = model.repository.get_repository(namespace, repository) + image_diff_queue.put([repo.namespace_user.username, repository, image_id], json.dumps({ + 'namespace_user_id': repo.namespace_user.id, + 'repository': repository, + 'image_id': image_id, + })) + + # Send a job to the work queue to replicate the image layer. + if features.STORAGE_REPLICATION: + image_replication_queue.put([repo_image.storage.uuid], json.dumps({ + 'namespace_user_id': repo.namespace_user.id, + 'storage_id': repo_image.storage.uuid, + })) + + def require_completion(f): """This make sure that the image push correctly finished.""" @wraps(f) @@ -210,7 +235,11 @@ def put_image_layer(namespace, repository, image_id): # Stream write the data to storage. with database.CloseForLongOperation(app.config): - store.stream_write(repo_image.storage.locations, layer_path, sr) + try: + store.stream_write(repo_image.storage.locations, layer_path, sr) + except IOError: + logger.exception('Exception when writing image data') + abort(520, 'Image %(image_id)s could not be written. Please try again.', image_id=image_id) # Append the computed checksum. csums = [] @@ -243,18 +272,8 @@ def put_image_layer(namespace, repository, image_id): abort(400, 'Checksum mismatch; ignoring the layer for image %(image_id)s', issue='checksum-mismatch', image_id=image_id) - # Checksum is ok, we remove the marker - set_uploading_flag(repo_image, False) - - # The layer is ready for download, send a job to the work queue to - # process it. - logger.debug('Adding layer to diff queue') - repo = model.repository.get_repository(namespace, repository) - image_diff_queue.put([repo.namespace_user.username, repository, image_id], json.dumps({ - 'namespace_user_id': repo.namespace_user.id, - 'repository': repository, - 'image_id': image_id, - })) + # Mark the image as uploaded. + _finish_image(namespace, repository, repo_image) return make_response('true', 200) @@ -316,18 +335,8 @@ def put_image_checksum(namespace, repository, image_id): abort(400, 'Checksum mismatch for image: %(image_id)s', issue='checksum-mismatch', image_id=image_id) - # Checksum is ok, we remove the marker - set_uploading_flag(repo_image, False) - - # The layer is ready for download, send a job to the work queue to - # process it. - logger.debug('Adding layer to diff queue') - repo = model.repository.get_repository(namespace, repository) - image_diff_queue.put([repo.namespace_user.username, repository, image_id], json.dumps({ - 'namespace_user_id': repo.namespace_user.id, - 'repository': repository, - 'image_id': image_id, - })) + # Mark the image as uploaded. + _finish_image(namespace, repository, repo_image) return make_response('true', 200) diff --git a/endpoints/v2/__init__.py b/endpoints/v2/__init__.py index 66e4a6cbd..b22af3ec2 100644 --- a/endpoints/v2/__init__.py +++ b/endpoints/v2/__init__.py @@ -7,6 +7,7 @@ from flask import Blueprint, make_response, url_for, request, jsonify from functools import wraps from urlparse import urlparse +from app import metric_queue from endpoints.decorators import anon_protect, anon_allowed from endpoints.v2.errors import V2RegistryException from auth.jwt_auth import process_jwt_auth @@ -15,13 +16,14 @@ from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermissi AdministerRepositoryPermission) from data import model from util.http import abort +from util.saas.metricqueue import time_blueprint from app import app - logger = logging.getLogger(__name__) v2_bp = Blueprint('v2', __name__) +time_blueprint(v2_bp, metric_queue) @v2_bp.app_errorhandler(V2RegistryException) def handle_registry_v2_exception(error): diff --git a/initdb.py b/initdb.py index 5c0e86f0a..c8c814832 100644 --- a/initdb.py +++ b/initdb.py @@ -20,6 +20,7 @@ from data.database import (db, all_models, Role, TeamRole, Visibility, LoginServ ExternalNotificationEvent, ExternalNotificationMethod, NotificationKind) from data import model from app import app, storage as store +from storage.basestorage import StoragePaths from workers import repositoryactioncounter @@ -85,6 +86,17 @@ def __create_subtree(repo, structure, creator_username, parent, tag_map): new_image.storage.checksum = checksum new_image.storage.save() + # Write some data for the storage. + if os.environ.get('WRITE_STORAGE_FILES'): + storage_paths = StoragePaths() + paths = [storage_paths.image_json_path, + storage_paths.image_ancestry_path, + storage_paths.image_layer_path] + + for path_builder in paths: + path = path_builder(new_image.storage.uuid) + store.put_content('local_us', path, checksum) + creation_time = REFERENCE_DATE + timedelta(weeks=image_num) + timedelta(days=model_num) command_list = SAMPLE_CMDS[image_num % len(SAMPLE_CMDS)] command = json.dumps(command_list) if command_list else None diff --git a/local-docker.sh b/local-docker.sh index 7dc6d447e..530ff2dae 100755 --- a/local-docker.sh +++ b/local-docker.sh @@ -1,11 +1,30 @@ -#!/bin/sh - -# Run this from the quay directory to start a quay development instance in -# docker on port 5000. +#!/bin/bash set -e REPO=quay.io/quay/quay-dev -docker build -t $REPO -f dev.df . -docker run -it -p 5000:5000 -v $(pwd)/..:/src $REPO bash /src/quay/local-run.sh +d () +{ + docker build -t $REPO -f dev.df . + docker -- run --rm -it --net=host -v $(pwd)/..:/src $REPO $* +} + +case $1 in +buildman) + d /venv/bin/python -m buildman.builder + ;; +dev) + d bash /src/quay/local-run.sh + ;; +notifications) + d /venv/bin/python -m workers.notificationworker + ;; +test) + d bash /src/quay/local-test.sh + ;; +*) + echo "unknown option" + exit 1 + ;; +esac diff --git a/requirements-nover.txt b/requirements-nover.txt index 41483e4e2..31da0c5b9 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -54,3 +54,4 @@ Flask-Testing pyjwt toposort pyjwkest +rfc3987 diff --git a/requirements.txt b/requirements.txt index ae7a83313..598558965 100644 --- a/requirements.txt +++ b/requirements.txt @@ -58,7 +58,7 @@ pycparser==2.14 pycrypto==2.6.1 pygpgme==0.3 pyjwkest==1.0.3 -PyJWT==1.3.0 +PyJWT==1.4.0 PyMySQL==0.6.6 pyOpenSSL==0.15.1 PyPDF2==1.24 @@ -74,6 +74,7 @@ redis==2.10.3 reportlab==2.7 requests==2.7.0 requests-oauthlib==0.5.0 +rfc3987==1.3.4 simplejson==3.7.3 six==1.9.0 SQLAlchemy==1.0.6 diff --git a/static/css/core-ui.css b/static/css/core-ui.css index b2213d91f..bf92cd6ae 100644 --- a/static/css/core-ui.css +++ b/static/css/core-ui.css @@ -1254,3 +1254,14 @@ a:focus { color: white; z-index: 2; } + +.co-alert.thin { + padding: 6px; + padding-left: 38px; + margin-bottom: 0px; +} + +.co-alert.thin:before { + top: 5px; + font-size: 18px; +} diff --git a/static/css/directives/ui/dockerfile-build-dialog.css b/static/css/directives/ui/dockerfile-build-dialog.css new file mode 100644 index 000000000..e31d6a7f9 --- /dev/null +++ b/static/css/directives/ui/dockerfile-build-dialog.css @@ -0,0 +1,13 @@ +.dockerfile-build-dialog-element .btn-group { + margin-bottom: 20px; +} + +.dockerfile-build-dialog-element button i { + margin-right: 6px; +} + +.dockerfile-build-dialog-element .trigger-list { + margin: 0px; + width: 100%; +} + diff --git a/static/css/directives/ui/dockerfile-build-form.css b/static/css/directives/ui/dockerfile-build-form.css index 825a6a717..2b0b95299 100644 --- a/static/css/directives/ui/dockerfile-build-form.css +++ b/static/css/directives/ui/dockerfile-build-form.css @@ -3,6 +3,10 @@ white-space: nowrap; } +.dockerfile-build-form .file-drop { + padding: 0px; +} + .dockerfile-build-form input[type="file"] { margin: 0px; } @@ -10,6 +14,11 @@ .dockerfile-build-form .help-text { font-size: 13px; color: #aaa; - margin-bottom: 20px; - padding-left: 22px; + margin-top: 10px; + margin-bottom: 16px; +} + +.dockerfile-build-form dd { + padding-left: 20px; + padding-top: 14px; } \ No newline at end of file diff --git a/static/css/pages/new-organization.css b/static/css/pages/new-organization.css index 6e6a1ef7b..bf4caeda3 100644 --- a/static/css/pages/new-organization.css +++ b/static/css/pages/new-organization.css @@ -1,4 +1,16 @@ .new-organization .co-main-content-panel { padding: 30px; position: relative; -} \ No newline at end of file +} + +.new-organization .field-container { + display: inline-block; + width: 400px; + margin-right: 10px; +} + +.new-organization .field-row .co-alert { + display: inline-block; + margin-left: 10px; + margin-top: 10px; +} diff --git a/static/directives/build-logs-view.html b/static/directives/build-logs-view.html index 5aacf0a99..a26f3eafc 100644 --- a/static/directives/build-logs-view.html +++ b/static/directives/build-logs-view.html @@ -20,7 +20,8 @@
- Failed to log builds logs. Please reload and try again. + Failed to load builds logs. Please reload and try again. If this problem persists, + please check for JavaScript or networking issues and contact support.
diff --git a/static/directives/config/config-setup-tool.html b/static/directives/config/config-setup-tool.html index 43bff96f3..ae85f6d48 100644 --- a/static/directives/config/config-setup-tool.html +++ b/static/directives/config/config-setup-tool.html @@ -226,6 +226,12 @@ ng-model="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]"> +
+ +
See Documentation for more information
diff --git a/static/directives/create-external-notification-dialog.html b/static/directives/create-external-notification-dialog.html index 0b7401c30..592efc322 100644 --- a/static/directives/create-external-notification-dialog.html +++ b/static/directives/create-external-notification-dialog.html @@ -11,14 +11,14 @@
- +
An e-mail has been sent to {{ currentConfig.email }}. Please click the link contained in the e-mail.

- Waiting... +
@@ -30,6 +30,14 @@ + + + + +
Notification title: + +
When this occurs: diff --git a/static/directives/dockerfile-build-dialog.html b/static/directives/dockerfile-build-dialog.html index 5ba1f475f..f49e0c353 100644 --- a/static/directives/dockerfile-build-dialog.html +++ b/static/directives/dockerfile-build-dialog.html @@ -1,27 +1,57 @@
-