vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'master' into star

Browse source
This commit is contained in:
Jimmy Zelinskie 2015-02-18 17:36:58 -05:00
commit 917dd6b674
229 changed files with 10807 additions and 3003 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
Bobfile
View file

@ -9,14 +9,8 @@ version = 1
[[container]]
name = "quay"
Dockerfile = "Dockerfile.web"
Dockerfile = "Dockerfile"
project = "quay"
tags = ["git:short"]
[[container]]
name = "builder"
Dockerfile = "Dockerfile.buildworker"
project = "builder"
tags = ["git:short"]
# vim:ft=toml

27
Dockerfile.web → Dockerfile
View file

@ -1,27 +1,21 @@
# vim:ft=dockerfile
FROM phusion/baseimage:0.9.15
FROM phusion/baseimage:0.9.16
ENV DEBIAN_FRONTEND noninteractive
ENV HOME /root
# Install the dependencies.
RUN apt-get update # 10SEP2014
RUN apt-get update # 29JAN2015
# New ubuntu packages should be added as their own apt-get install lines below the existing install commands
RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev libfreetype6-dev
RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev libfreetype6-dev libffi-dev libgpgme11 libgpgme11-dev
# Build the python dependencies
ADD requirements.txt requirements.txt
RUN virtualenv --distribute venv
RUN venv/bin/pip install -r requirements.txt
RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev
### End common section ###
# Remove SSH.
RUN rm -rf /etc/service/sshd /etc/my_init.d/00_regen_ssh_host_keys.sh
# Install the binary dependencies
ADD binary_dependencies binary_dependencies
RUN gdebi --n binary_dependencies/*.deb
@ -34,6 +28,10 @@ RUN npm install -g grunt-cli
ADD grunt grunt
RUN cd grunt && npm install
RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev libgpgme11-dev
RUN apt-get autoremove -y
RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# Add all of the files!
ADD . .
@ -58,14 +56,9 @@ ADD conf/init/buildmanager /etc/service/buildmanager
RUN mkdir static/fonts static/ldn
RUN venv/bin/python -m external_libraries
RUN apt-get autoremove -y
RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# Run the tests
RUN TEST=true venv/bin/python -m unittest discover
VOLUME ["/conf/stack", "/var/log", "/datastorage", "/tmp"]
VOLUME ["/conf/stack", "/var/log", "/datastorage", "/tmp", "/conf/etcd"]
EXPOSE 443 80
CMD ["/sbin/my_init"]
EXPOSE 443 8443 80

42
Dockerfile.buildworker
View file

@ -1,42 +0,0 @@
# vim:ft=dockerfile
FROM phusion/baseimage:0.9.15
ENV DEBIAN_FRONTEND noninteractive
ENV HOME /root
# Install the dependencies.
RUN apt-get update # 20NOV2014
# New ubuntu packages should be added as their own apt-get install lines below the existing install commands
RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev
# Build the python dependencies
ADD requirements.txt requirements.txt
RUN virtualenv --distribute venv
RUN venv/bin/pip install -r requirements.txt
RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev
### End common section ###
RUN apt-get install -y lxc aufs-tools
RUN usermod -v 100000-200000 -w 100000-200000 root
ADD binary_dependencies/builder binary_dependencies/builder
RUN gdebi --n binary_dependencies/builder/*.deb
ADD . .
ADD conf/init/svlogd_config /svlogd_config
ADD conf/init/preplogsdir.sh /etc/my_init.d/
ADD conf/init/tutumdocker /etc/service/tutumdocker
ADD conf/init/dockerfilebuild /etc/service/dockerfilebuild
RUN apt-get remove -y --auto-remove nodejs npm git phantomjs
RUN apt-get autoremove -y
RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
VOLUME ["/var/lib/docker", "/var/lib/lxc", "/conf/stack", "/var/log"]
CMD ["/sbin/my_init"]

8
README.md
View file

@ -3,7 +3,7 @@ to build and upload quay to quay:
```
curl -s https://get.docker.io/ubuntu/ | sudo sh
sudo apt-get update && sudo apt-get install -y git
git clone https://bitbucket.org/yackob03/quay.git
git clone https://github.com/coreos-inc/quay.git
cd quay
rm Dockerfile
ln -s Dockerfile.web Dockerfile
@ -33,7 +33,7 @@ start the quay processes:
```
cd ~
git clone https://bitbucket.org/yackob03/quayconfig.git
git clone https://github.com/coreos-inc/quay.git
sudo docker pull staging.quay.io/quay/quay
cd ~/gantryd
sudo venv/bin/python gantry.py ../quayconfig/production/gantry.json update quay
@ -44,7 +44,7 @@ to build and upload the builder to quay
```
curl -s https://get.docker.io/ubuntu/ | sudo sh
sudo apt-get update && sudo apt-get install -y git
git clone git clone https://bitbucket.org/yackob03/quay.git
git clone git clone https://github.com/coreos-inc/quay.git
cd quay
rm Dockerfile
ln -s Dockerfile.buildworker Dockerfile
@ -74,7 +74,7 @@ start the worker
```
cd ~
git clone https://bitbucket.org/yackob03/quayconfig.git
git clone https://github.com/coreos-inc/quay.git
sudo docker pull quay.io/quay/builder
cd ~/gantryd
sudo venv/bin/python gantry.py ../quayconfig/production/gantry.json update builder

137
app.py
View file

@ -1,71 +1,55 @@
import logging
import os
import json
import yaml
from flask import Flask as BaseFlask, Config as BaseConfig, request, Request
from flask import Flask, Config, request, Request, _request_ctx_stack
from flask.ext.principal import Principal
from flask.ext.login import LoginManager
from flask.ext.login import LoginManager, UserMixin
from flask.ext.mail import Mail
import features
from avatars.avatars import Avatar
from storage import Storage
from avatars.avatars import Avatar
from data import model
from data import database
from data.userfiles import Userfiles
from data.users import UserAuthentication
from util.analytics import Analytics
from util.exceptionlog import Sentry
from util.queuemetrics import QueueMetrics
from util.names import urn_generator
from util.oauth import GoogleOAuthConfig, GithubOAuthConfig
from data.billing import Billing
from data.buildlogs import BuildLogs
from data.archivedlogs import LogArchive
from data.queue import WorkQueue
from data.userevent import UserEventsBuilderModule
from avatars.avatars import Avatar
class Config(BaseConfig):
""" Flask config enhanced with a `from_yamlfile` method """
def from_yamlfile(self, config_file):
with open(config_file) as f:
c = yaml.load(f)
if not c:
logger.debug('Empty YAML config file')
return
if isinstance(c, str):
raise Exception('Invalid YAML config file: ' + str(c))
for key in c.iterkeys():
if key.isupper():
self[key] = c[key]
class Flask(BaseFlask):
""" Extends the Flask class to implement our custom Config class. """
def make_config(self, instance_relative=False):
root_path = self.instance_path if instance_relative else self.root_path
return Config(root_path, self.default_config)
from data.queue import WorkQueue
from util.analytics import Analytics
from util.exceptionlog import Sentry
from util.names import urn_generator
from util.oauth import GoogleOAuthConfig, GithubOAuthConfig
from util.signing import Signer
from util.queuemetrics import QueueMetrics
from util.config.provider import FileConfigProvider, TestConfigProvider
from util.config.configutil import generate_secret_key
from util.config.superusermanager import SuperUserManager
from buildman.jobutil.buildreporter import BuildMetrics
OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/'
OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml'
OVERRIDE_CONFIG_PY_FILENAME = 'conf/stack/config.py'
OVERRIDE_CONFIG_KEY = 'QUAY_OVERRIDE_CONFIG'
LICENSE_FILENAME = 'conf/stack/license.enc'
CONFIG_PROVIDER = FileConfigProvider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py')
app = Flask(__name__)
logger = logging.getLogger(__name__)
profile = logging.getLogger('profile')
# Instantiate the default configuration (for test or for normal operation).
if 'TEST' in os.environ:
CONFIG_PROVIDER = TestConfigProvider()
from test.testconfig import TestConfig
logger.debug('Loading test config.')
app.config.from_object(TestConfig())
@ -73,20 +57,17 @@ else:
from config import DefaultConfig
logger.debug('Loading default config.')
app.config.from_object(DefaultConfig())
if os.path.exists(OVERRIDE_CONFIG_PY_FILENAME):
logger.debug('Applying config file: %s', OVERRIDE_CONFIG_PY_FILENAME)
app.config.from_pyfile(OVERRIDE_CONFIG_PY_FILENAME)
if os.path.exists(OVERRIDE_CONFIG_YAML_FILENAME):
logger.debug('Applying config file: %s', OVERRIDE_CONFIG_YAML_FILENAME)
app.config.from_yamlfile(OVERRIDE_CONFIG_YAML_FILENAME)
environ_config = json.loads(os.environ.get(OVERRIDE_CONFIG_KEY, '{}'))
app.config.update(environ_config)
app.teardown_request(database.close_db_filter)
# Load the override config via the provider.
CONFIG_PROVIDER.update_app_config(app.config)
# Update any configuration found in the override environment variable.
OVERRIDE_CONFIG_KEY = 'QUAY_OVERRIDE_CONFIG'
environ_config = json.loads(os.environ.get(OVERRIDE_CONFIG_KEY, '{}'))
app.config.update(environ_config)
class RequestWithId(Request):
request_gen = staticmethod(urn_generator(['request']))
@ -98,21 +79,24 @@ class RequestWithId(Request):
@app.before_request
def _request_start():
profile.debug('Starting request: %s', request.path)
logger.debug('Starting request: %s', request.path)
@app.after_request
def _request_end(r):
profile.debug('Ending request: %s', request.path)
logger.debug('Ending request: %s', request.path)
return r
class InjectingFilter(logging.Filter):
def filter(self, record):
record.msg = '[%s] %s' % (request.request_id, record.msg)
if _request_ctx_stack.top is not None:
record.msg = '[%s] %s' % (request.request_id, record.msg)
return True
profile.addFilter(InjectingFilter())
# Add the request id filter to all handlers of the root logger
for handler in logging.getLogger().handlers:
handler.addFilter(InjectingFilter())
app.request_class = RequestWithId
@ -130,16 +114,20 @@ analytics = Analytics(app)
billing = Billing(app)
sentry = Sentry(app)
build_logs = BuildLogs(app)
queue_metrics = QueueMetrics(app)
authentication = UserAuthentication(app)
userevents = UserEventsBuilderModule(app)
github_login = GithubOAuthConfig(app, 'GITHUB_LOGIN_CONFIG')
github_trigger = GithubOAuthConfig(app, 'GITHUB_TRIGGER_CONFIG')
google_login = GoogleOAuthConfig(app, 'GOOGLE_LOGIN_CONFIG')
oauth_apps = [github_login, github_trigger, google_login]
superusers = SuperUserManager(app)
signer = Signer(app, OVERRIDE_CONFIG_DIRECTORY)
queue_metrics = QueueMetrics(app)
build_metrics = BuildMetrics(app)
tf = app.config['DB_TRANSACTION_FACTORY']
github_login = GithubOAuthConfig(app.config, 'GITHUB_LOGIN_CONFIG')
github_trigger = GithubOAuthConfig(app.config, 'GITHUB_TRIGGER_CONFIG')
google_login = GoogleOAuthConfig(app.config, 'GOOGLE_LOGIN_CONFIG')
oauth_apps = [github_login, github_trigger, google_login]
image_diff_queue = WorkQueue(app.config['DIFFS_QUEUE_NAME'], tf)
dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf,
reporter=queue_metrics.report)
@ -149,5 +137,34 @@ database.configure(app.config)
model.config.app_config = app.config
model.config.store = storage
# Generate a secret key if none was specified.
if app.config['SECRET_KEY'] is None:
logger.debug('Generating in-memory secret key')
app.config['SECRET_KEY'] = generate_secret_key()
@login_manager.user_loader
def load_user(user_uuid):
logger.debug('User loader loading deferred user with uuid: %s' % user_uuid)
return LoginWrappedDBUser(user_uuid)
class LoginWrappedDBUser(UserMixin):
def __init__(self, user_uuid, db_user=None):
self._uuid = user_uuid
self._db_user = db_user
def db_user(self):
if not self._db_user:
self._db_user = model.get_user_by_uuid(self._uuid)
return self._db_user
def is_authenticated(self):
return self.db_user() is not None
def is_active(self):
return self.db_user().verified
def get_id(self):
return unicode(self._uuid)
def get_app_url():
return '%s://%s' % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'])

2
application.py
View file

@ -11,5 +11,5 @@ import registry
if __name__ == '__main__':
logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False)
logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False)
application.run(port=5000, debug=True, threaded=True, host='0.0.0.0')

10
auth/permissions.py
View file

@ -7,7 +7,7 @@ from functools import partial
import scopes
from data import model
from app import app
from app import app, superusers
logger = logging.getLogger(__name__)
@ -89,10 +89,14 @@ class QuayDeferredPermissionUser(Identity):
if not self._permissions_loaded:
logger.debug('Loading user permissions after deferring.')
user_object = model.get_user_by_uuid(self.id)
if user_object is None:
return super(QuayDeferredPermissionUser, self).can(permission)
if user_object is None:
return super(QuayDeferredPermissionUser, self).can(permission)
# Add the superuser need, if applicable.
if (user_object.username is not None and
user_object.username in app.config.get('SUPER_USERS', [])):
if superusers.is_superuser(user_object.username):
self.provides.add(_SuperUserNeed())
# Add the user specific permissions, only for non-oauth permission

BIN
binary_dependencies/builder/lxc-docker-1.3.2-userns_1.3.2-userns-20141124223407-72690d2-dirty_amd64.deb
View file

Binary file not shown.

BIN
binary_dependencies/nginx_1.4.2-nobuffer-3_amd64.deb
View file

Binary file not shown.

BIN
binary_dependencies/tengine_2.1.0-1_amd64.deb Normal file
View file

Binary file not shown.

2
build.sh Executable file
View file

@ -0,0 +1,2 @@
docker build -t quay.io/quay/quay:`git rev-parse --short HEAD` .
echo quay.io/quay/quay:`git rev-parse --short HEAD`

27
buildman/asyncutil.py Normal file
View file

@ -0,0 +1,27 @@
from functools import partial, wraps
from trollius import get_event_loop
class AsyncWrapper(object):
""" Wrapper class which will transform a syncronous library to one that can be used with
trollius coroutines.
"""
def __init__(self, delegate, loop=None, executor=None):
self._loop = loop if loop is not None else get_event_loop()
self._delegate = delegate
self._executor = executor
def __getattr__(self, attrib):
delegate_attr = getattr(self._delegate, attrib)
if not callable(delegate_attr):
return delegate_attr
def wrapper(*args, **kwargs):
""" Wraps the delegate_attr with primitives that will transform sync calls to ones shelled
out to a thread pool.
"""
callable_delegate_attr = partial(delegate_attr, *args, **kwargs)
return self._loop.run_in_executor(self._executor, callable_delegate_attr)
return wrapper

35
buildman/builder.py
View file

@ -6,6 +6,7 @@ import time
from app import app, userfiles as user_files, build_logs, dockerfile_build_queue
from buildman.manager.enterprise import EnterpriseManager
from buildman.manager.ephemeral import EphemeralBuilderManager
from buildman.server import BuilderServer
from trollius import SSLContext
@ -13,14 +14,22 @@ from trollius import SSLContext
logger = logging.getLogger(__name__)
BUILD_MANAGERS = {
'enterprise': EnterpriseManager
'enterprise': EnterpriseManager,
'ephemeral': EphemeralBuilderManager,
}
EXTERNALLY_MANAGED = 'external'
DEFAULT_WEBSOCKET_PORT = 8787
DEFAULT_CONTROLLER_PORT = 8686
LOG_FORMAT = "%(asctime)s [%(process)d] [%(levelname)s] [%(name)s] %(message)s"
def run_build_manager():
if not features.BUILD_SUPPORT:
logger.debug('Building is disabled. Please enable the feature flag')
while True:
time.sleep(1000)
return
build_manager_config = app.config.get('BUILD_MANAGER')
@ -39,18 +48,32 @@ def run_build_manager():
if manager_klass is None:
return
manager_hostname = os.environ.get('BUILDMAN_HOSTNAME',
app.config.get('BUILDMAN_HOSTNAME',
app.config['SERVER_HOSTNAME']))
websocket_port = int(os.environ.get('BUILDMAN_WEBSOCKET_PORT',
app.config.get('BUILDMAN_WEBSOCKET_PORT',
DEFAULT_WEBSOCKET_PORT)))
controller_port = int(os.environ.get('BUILDMAN_CONTROLLER_PORT',
app.config.get('BUILDMAN_CONTROLLER_PORT',
DEFAULT_CONTROLLER_PORT)))
logger.debug('Will pass buildman hostname %s to builders for websocket connection',
manager_hostname)
logger.debug('Starting build manager with lifecycle "%s"', build_manager_config[0])
ssl_context = None
if os.environ.get('SSL_CONFIG'):
logger.debug('Loading SSL cert and key')
ssl_context = SSLContext()
ssl_context.load_cert_chain(os.environ.get('SSL_CONFIG') + '/ssl.cert',
os.environ.get('SSL_CONFIG') + '/ssl.key')
ssl_context.load_cert_chain(os.path.join(os.environ.get('SSL_CONFIG'), 'ssl.cert'),
os.path.join(os.environ.get('SSL_CONFIG'), 'ssl.key'))
server = BuilderServer(app.config['SERVER_HOSTNAME'], dockerfile_build_queue, build_logs,
user_files, manager_klass)
server.run('0.0.0.0', ssl=ssl_context)
user_files, manager_klass, build_manager_config[1], manager_hostname)
server.run('0.0.0.0', websocket_port, controller_port, ssl=ssl_context)
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
logging.getLogger('peewee').setLevel(logging.WARN)
run_build_manager()

3
buildman/component/basecomponent.py
View file

@ -8,3 +8,6 @@ class BaseComponent(ApplicationSession):
self.parent_manager = None
self.build_logs = None
self.user_files = None
def kind(self):
raise NotImplementedError

201
buildman/component/buildcomponent.py
View file

@ -6,11 +6,10 @@ import trollius
import re
from autobahn.wamp.exception import ApplicationError
from trollius.coroutines import From
from buildman.server import BuildJobResult
from buildman.component.basecomponent import BaseComponent
from buildman.jobutil.buildpack import BuildPackage, BuildPackageException
from buildman.jobutil.buildjob import BuildJobLoadException
from buildman.jobutil.buildstatus import StatusHandler
from buildman.jobutil.workererror import WorkerError
@ -20,7 +19,7 @@ HEARTBEAT_DELTA = datetime.timedelta(seconds=30)
HEARTBEAT_TIMEOUT = 10
INITIAL_TIMEOUT = 25
SUPPORTED_WORKER_VERSIONS = ['0.1-beta']
SUPPORTED_WORKER_VERSIONS = ['0.3']
logger = logging.getLogger(__name__)
@ -39,84 +38,71 @@ class BuildComponent(BaseComponent):
self.builder_realm = realm
self.parent_manager = None
self.server_hostname = None
self.registry_hostname = None
self._component_status = ComponentStatus.JOINING
self._last_heartbeat = None
self._current_job = None
self._build_status = None
self._image_info = None
self._worker_version = None
BaseComponent.__init__(self, config, **kwargs)
def kind(self):
return 'builder'
def onConnect(self):
self.join(self.builder_realm)
def onJoin(self, details):
logger.debug('Registering methods and listeners for component %s', self.builder_realm)
yield From(self.register(self._on_ready, u'io.quay.buildworker.ready'))
yield From(self.register(self._ping, u'io.quay.buildworker.ping'))
yield From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat'))
yield From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage'))
yield trollius.From(self.register(self._on_ready, u'io.quay.buildworker.ready'))
yield trollius.From(self.register(self._determine_cache_tag,
u'io.quay.buildworker.determinecachetag'))
yield trollius.From(self.register(self._ping, u'io.quay.buildworker.ping'))
self._set_status(ComponentStatus.WAITING)
yield trollius.From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat'))
yield trollius.From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage'))
yield trollius.From(self._set_status(ComponentStatus.WAITING))
def is_ready(self):
""" Determines whether a build component is ready to begin a build. """
return self._component_status == ComponentStatus.RUNNING
@trollius.coroutine
def start_build(self, build_job):
""" Starts a build. """
logger.debug('Starting build for component %s (worker version: %s)',
self.builder_realm, self._worker_version)
self._current_job = build_job
self._build_status = StatusHandler(self.build_logs, build_job.repo_build())
self._build_status = StatusHandler(self.build_logs, build_job.repo_build.uuid)
self._image_info = {}
self._set_status(ComponentStatus.BUILDING)
yield trollius.From(self._set_status(ComponentStatus.BUILDING))
# Retrieve the job's buildpack.
buildpack_url = self.user_files.get_file_url(build_job.repo_build().resource_key,
# Send the notification that the build has started.
build_job.send_notification('build_start')
# Parse the build configuration.
try:
build_config = build_job.build_config
except BuildJobLoadException as irbe:
self._build_failure('Could not load build job information', irbe)
base_image_information = {}
buildpack_url = self.user_files.get_file_url(build_job.repo_build.resource_key,
requires_cors=False)
logger.debug('Retreiving build package: %s', buildpack_url)
buildpack = None
try:
buildpack = BuildPackage.from_url(buildpack_url)
except BuildPackageException as bpe:
self._build_failure('Could not retrieve build package', bpe)
return
# Extract the base image information from the Dockerfile.
parsed_dockerfile = None
logger.debug('Parsing dockerfile')
build_config = build_job.build_config()
try:
parsed_dockerfile = buildpack.parse_dockerfile(build_config.get('build_subdir'))
except BuildPackageException as bpe:
self._build_failure('Could not find Dockerfile in build package', bpe)
return
image_and_tag_tuple = parsed_dockerfile.get_image_and_tag()
if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
self._build_failure('Missing FROM line in Dockerfile')
return
base_image_information = {
'repository': image_and_tag_tuple[0],
'tag': image_and_tag_tuple[1]
}
# Extract the number of steps from the Dockerfile.
with self._build_status as status_dict:
status_dict['total_commands'] = len(parsed_dockerfile.commands)
# Add the pull robot information, if any.
if build_config.get('pull_credentials') is not None:
base_image_information['username'] = build_config['pull_credentials'].get('username', '')
base_image_information['password'] = build_config['pull_credentials'].get('password', '')
if build_job.pull_credentials:
base_image_information['username'] = build_job.pull_credentials.get('username', '')
base_image_information['password'] = build_job.pull_credentials.get('password', '')
# Retrieve the repository's fully qualified name.
repo = build_job.repo_build().repository
repo = build_job.repo_build.repository
repository_name = repo.namespace_user.username + '/' + repo.name
# Parse the build queue item into build arguments.
@ -128,29 +114,26 @@ class BuildComponent(BaseComponent):
# push_token: The token to use to push the built image.
# tag_names: The name(s) of the tag(s) for the newly built image.
# base_image: The image name and credentials to use to conduct the base image pull.
# repository: The repository to pull.
# tag: The tag to pull.
# repository: The repository to pull (DEPRECATED 0.2)
# tag: The tag to pull (DEPRECATED in 0.2)
# username: The username for pulling the base image (if any).
# password: The password for pulling the base image (if any).
build_arguments = {
'build_package': buildpack_url,
'sub_directory': build_config.get('build_subdir', ''),
'repository': repository_name,
'registry': self.server_hostname,
'pull_token': build_job.repo_build().access_token.code,
'push_token': build_job.repo_build().access_token.code,
'registry': self.registry_hostname,
'pull_token': build_job.repo_build.access_token.code,
'push_token': build_job.repo_build.access_token.code,
'tag_names': build_config.get('docker_tags', ['latest']),
'base_image': base_image_information,
'cached_tag': build_job.determine_cached_tag() or ''
'base_image': base_image_information
}
# Invoke the build.
logger.debug('Invoking build: %s', self.builder_realm)
logger.debug('With Arguments: %s', build_arguments)
return (self
.call("io.quay.builder.build", **build_arguments)
.add_done_callback(self._build_complete))
self.call("io.quay.builder.build", **build_arguments).add_done_callback(self._build_complete)
@staticmethod
def _total_completion(statuses, total_images):
@ -237,18 +220,28 @@ class BuildComponent(BaseComponent):
elif phase == BUILD_PHASE.BUILDING:
self._build_status.append_log(current_status_string)
@trollius.coroutine
def _determine_cache_tag(self, command_comments, base_image_name, base_image_tag, base_image_id):
with self._build_status as status_dict:
status_dict['total_commands'] = len(command_comments) + 1
logger.debug('Checking cache on realm %s. Base image: %s:%s (%s)', self.builder_realm,
base_image_name, base_image_tag, base_image_id)
tag_found = self._current_job.determine_cached_tag(base_image_id, command_comments)
raise trollius.Return(tag_found or '')
def _build_failure(self, error_message, exception=None):
""" Handles and logs a failed build. """
self._build_status.set_error(error_message, {
'internal_error': exception.message if exception else None
'internal_error': str(exception) if exception else None
})
build_id = self._current_job.repo_build().uuid
build_id = self._current_job.repo_build.uuid
logger.warning('Build %s failed with message: %s', build_id, error_message)
# Mark that the build has finished (in an error state)
self._build_finished(BuildJobResult.ERROR)
trollius.async(self._build_finished(BuildJobResult.ERROR))
def _build_complete(self, result):
""" Wraps up a completed build. Handles any errors and calls self._build_finished. """
@ -256,60 +249,78 @@ class BuildComponent(BaseComponent):
# Retrieve the result. This will raise an ApplicationError on any error that occurred.
result.result()
self._build_status.set_phase(BUILD_PHASE.COMPLETE)
self._build_finished(BuildJobResult.COMPLETE)
trollius.async(self._build_finished(BuildJobResult.COMPLETE))
# Send the notification that the build has completed successfully.
self._current_job.send_notification('build_success')
except ApplicationError as aex:
worker_error = WorkerError(aex.error, aex.kwargs.get('base_error'))
# Write the error to the log.
self._build_status.set_error(worker_error.public_message(), worker_error.extra_data(),
internal_error=worker_error.is_internal_error())
internal_error=worker_error.is_internal_error(),
requeued=self._current_job.has_retries_remaining())
# Send the notification that the build has failed.
self._current_job.send_notification('build_failure',
error_message=worker_error.public_message())
# Mark the build as completed.
if worker_error.is_internal_error():
self._build_finished(BuildJobResult.INCOMPLETE)
trollius.async(self._build_finished(BuildJobResult.INCOMPLETE))
else:
self._build_finished(BuildJobResult.ERROR)
trollius.async(self._build_finished(BuildJobResult.ERROR))
@trollius.coroutine
def _build_finished(self, job_status):
""" Alerts the parent that a build has completed and sets the status back to running. """
self.parent_manager.job_completed(self._current_job, job_status, self)
yield trollius.From(self.parent_manager.job_completed(self._current_job, job_status, self))
self._current_job = None
# Set the component back to a running state.
self._set_status(ComponentStatus.RUNNING)
yield trollius.From(self._set_status(ComponentStatus.RUNNING))
@staticmethod
def _ping():
""" Ping pong. """
return 'pong'
@trollius.coroutine
def _on_ready(self, token, version):
if not version in SUPPORTED_WORKER_VERSIONS:
logger.warning('Build component (token "%s") is running an out-of-date version: %s', version)
return False
self._worker_version = version
if self._component_status != 'waiting':
if not version in SUPPORTED_WORKER_VERSIONS:
logger.warning('Build component (token "%s") is running an out-of-date version: %s', token,
version)
raise trollius.Return(False)
if self._component_status != ComponentStatus.WAITING:
logger.warning('Build component (token "%s") is already connected', self.expected_token)
return False
raise trollius.Return(False)
if token != self.expected_token:
logger.warning('Builder token mismatch. Expected: "%s". Found: "%s"', self.expected_token, token)
return False
logger.warning('Builder token mismatch. Expected: "%s". Found: "%s"', self.expected_token,
token)
raise trollius.Return(False)
self._set_status(ComponentStatus.RUNNING)
yield trollius.From(self._set_status(ComponentStatus.RUNNING))
# Start the heartbeat check and updating loop.
loop = trollius.get_event_loop()
loop.create_task(self._heartbeat())
logger.debug('Build worker %s is connected and ready', self.builder_realm)
return True
raise trollius.Return(True)
@trollius.coroutine
def _set_status(self, phase):
if phase == ComponentStatus.RUNNING:
yield trollius.From(self.parent_manager.build_component_ready(self))
self._component_status = phase
def _on_heartbeat(self):
""" Updates the last known heartbeat. """
self._last_heartbeat = datetime.datetime.now()
self._last_heartbeat = datetime.datetime.utcnow()
@trollius.coroutine
def _heartbeat(self):
@ -317,13 +328,13 @@ class BuildComponent(BaseComponent):
and updating the heartbeat in the build status dictionary (if applicable). This allows
the build system to catch crashes from either end.
"""
yield From(trollius.sleep(INITIAL_TIMEOUT))
yield trollius.From(trollius.sleep(INITIAL_TIMEOUT))
while True:
# If the component is no longer running or actively building, nothing more to do.
if (self._component_status != ComponentStatus.RUNNING and
self._component_status != ComponentStatus.BUILDING):
return
raise trollius.Return()
# If there is an active build, write the heartbeat to its status.
build_status = self._build_status
@ -331,35 +342,37 @@ class BuildComponent(BaseComponent):
with build_status as status_dict:
status_dict['heartbeat'] = int(time.time())
# Mark the build item.
current_job = self._current_job
if current_job is not None:
self.parent_manager.job_heartbeat(current_job)
yield trollius.From(self.parent_manager.job_heartbeat(current_job))
# Check the heartbeat from the worker.
logger.debug('Checking heartbeat on realm %s', self.builder_realm)
if self._last_heartbeat and self._last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
self._timeout()
return
if (self._last_heartbeat and
self._last_heartbeat < datetime.datetime.utcnow() - HEARTBEAT_DELTA):
yield trollius.From(self._timeout())
raise trollius.Return()
yield From(trollius.sleep(HEARTBEAT_TIMEOUT))
yield trollius.From(trollius.sleep(HEARTBEAT_TIMEOUT))
@trollius.coroutine
def _timeout(self):
self._set_status(ComponentStatus.TIMED_OUT)
logger.warning('Build component with realm %s has timed out', self.builder_realm)
self._dispose(timed_out=True)
if self._component_status == ComponentStatus.TIMED_OUT:
raise trollius.Return()
yield trollius.From(self._set_status(ComponentStatus.TIMED_OUT))
logger.warning('Build component with realm %s has timed out', self.builder_realm)
def _dispose(self, timed_out=False):
# If we still have a running job, then it has not completed and we need to tell the parent
# manager.
if self._current_job is not None:
if timed_out:
self._build_status.set_error('Build worker timed out', internal_error=True)
self._build_status.set_error('Build worker timed out', internal_error=True,
requeued=self._current_job.has_retries_remaining())
self.parent_manager.job_completed(self._current_job, BuildJobResult.INCOMPLETE, self)
self._build_status = None
self._current_job = None
# Unregister the current component so that it cannot be invoked again.
self.parent_manager.build_component_disposed(self, timed_out)
self.parent_manager.build_component_disposed(self, True)

12
buildman/enums.py Normal file
View file

@ -0,0 +1,12 @@
class BuildJobResult(object):
""" Build job result enum """
INCOMPLETE = 'incomplete'
COMPLETE = 'complete'
ERROR = 'error'
class BuildServerStatus(object):
""" Build server status enum """
STARTING = 'starting'
RUNNING = 'running'
SHUTDOWN = 'shutting_down'

136
buildman/jobutil/buildjob.py
View file

@ -1,6 +1,13 @@
from data import model
import json
import logging
from cachetools import lru_cache
from endpoints.notificationhelper import spawn_notification
from data import model
from util.imagetree import ImageTree
logger = logging.getLogger(__name__)
class BuildJobLoadException(Exception):
""" Exception raised if a build job could not be instantiated for some reason. """
@ -9,52 +16,123 @@ class BuildJobLoadException(Exception):
class BuildJob(object):
""" Represents a single in-progress build job. """
def __init__(self, job_item):
self._job_item = job_item
self.job_item = job_item
try:
self._job_details = json.loads(job_item.body)
self.job_details = json.loads(job_item.body)
except ValueError:
raise BuildJobLoadException(
'Could not parse build queue item config with ID %s' % self._job_details['build_uuid']
'Could not parse build queue item config with ID %s' % self.job_details['build_uuid']
)
def has_retries_remaining(self):
return self.job_item.retries_remaining > 0
def send_notification(self, kind, error_message=None):
tags = self.build_config.get('docker_tags', ['latest'])
event_data = {
'build_id': self.repo_build.uuid,
'build_name': self.repo_build.display_name,
'docker_tags': tags,
'trigger_id': self.repo_build.trigger.uuid,
'trigger_kind': self.repo_build.trigger.service.name
}
if error_message is not None:
event_data['error_message'] = error_message
spawn_notification(self.repo_build.repository, kind, event_data,
subpage='build?current=%s' % self.repo_build.uuid,
pathargs=['build', self.repo_build.uuid])
@lru_cache(maxsize=1)
def _load_repo_build(self):
try:
self._repo_build = model.get_repository_build(self._job_details['namespace'],
self._job_details['repository'],
self._job_details['build_uuid'])
return model.get_repository_build(self.job_details['build_uuid'])
except model.InvalidRepositoryBuildException:
raise BuildJobLoadException(
'Could not load repository build with ID %s' % self._job_details['build_uuid'])
'Could not load repository build with ID %s' % self.job_details['build_uuid'])
@property
def repo_build(self):
return self._load_repo_build()
@property
def pull_credentials(self):
""" Returns the pull credentials for this job, or None if none. """
return self.job_details.get('pull_credentials')
@property
def build_config(self):
try:
self._build_config = json.loads(self._repo_build.job_config)
return json.loads(self.repo_build.job_config)
except ValueError:
raise BuildJobLoadException(
'Could not parse repository build job config with ID %s' % self._job_details['build_uuid']
'Could not parse repository build job config with ID %s' % self.job_details['build_uuid']
)
def determine_cached_tag(self):
def determine_cached_tag(self, base_image_id=None, cache_comments=None):
""" Returns the tag to pull to prime the cache or None if none. """
# TODO(jschorr): Change this to use the more complicated caching rules, once we have caching
# be a pull of things besides the constructed tags.
tags = self._build_config.get('docker_tags', ['latest'])
existing_tags = model.list_repository_tags(self._job_details['namespace'],
self._job_details['repository'])
cached_tag = None
if base_image_id and cache_comments:
cached_tag = self._determine_cached_tag_by_comments(base_image_id, cache_comments)
if not cached_tag:
cached_tag = self._determine_cached_tag_by_tag()
logger.debug('Determined cached tag %s for %s: %s', cached_tag, base_image_id, cache_comments)
return cached_tag
def _determine_cached_tag_by_comments(self, base_image_id, cache_commands):
""" Determines the tag to use for priming the cache for this build job, by matching commands
starting at the given base_image_id. This mimics the Docker cache checking, so it should,
in theory, provide "perfect" caching.
"""
# Lookup the base image in the repository. If it doesn't exist, nothing more to do.
repo_build = self.repo_build
repo_namespace = repo_build.repository.namespace_user.username
repo_name = repo_build.repository.name
base_image = model.get_image(repo_build.repository, base_image_id)
if base_image is None:
return None
# Build an in-memory tree of the full heirarchy of images in the repository.
all_images = model.get_repository_images(repo_namespace, repo_name)
all_tags = model.list_repository_tags(repo_namespace, repo_name)
tree = ImageTree(all_images, all_tags, base_filter=base_image.id)
# Find a path in the tree, starting at the base image, that matches the cache comments
# or some subset thereof.
def checker(step, image):
if step >= len(cache_commands):
return False
full_command = '["/bin/sh", "-c", "%s"]' % cache_commands[step]
logger.debug('Checking step #%s: %s, %s == %s', step, image.id,
image.storage.command, full_command)
return image.storage.command == full_command
path = tree.find_longest_path(base_image.id, checker)
if not path:
return None
# Find any tag associated with the last image in the path.
return tree.tag_containing_image(path[-1])
def _determine_cached_tag_by_tag(self):
""" Determines the cached tag by looking for one of the tags being built, and seeing if it
exists in the repository. This is a fallback for when no comment information is available.
"""
tags = self.build_config.get('docker_tags', ['latest'])
repository = self.repo_build.repository
existing_tags = model.list_repository_tags(repository.namespace_user.username, repository.name)
cached_tags = set(tags) & set([tag.name for tag in existing_tags])
if cached_tags:
return list(cached_tags)[0]
return None
def job_item(self):
""" Returns the job's queue item. """
return self._job_item
def repo_build(self):
""" Returns the repository build DB row for the job. """
return self._repo_build
def build_config(self):
""" Returns the parsed repository build config for the job. """
return self._build_config

88
buildman/jobutil/buildpack.py
View file

@ -1,88 +0,0 @@
import tarfile
import requests
import os
from tempfile import TemporaryFile, mkdtemp
from zipfile import ZipFile
from util.dockerfileparse import parse_dockerfile
from util.safetar import safe_extractall
class BuildPackageException(Exception):
""" Exception raised when retrieving or parsing a build package. """
pass
class BuildPackage(object):
""" Helper class for easy reading and updating of a Dockerfile build pack. """
def __init__(self, requests_file):
self._mime_processors = {
'application/zip': BuildPackage._prepare_zip,
'application/x-zip-compressed': BuildPackage._prepare_zip,
'text/plain': BuildPackage._prepare_dockerfile,
'application/octet-stream': BuildPackage._prepare_dockerfile,
'application/x-tar': BuildPackage._prepare_tarball,
'application/gzip': BuildPackage._prepare_tarball,
'application/x-gzip': BuildPackage._prepare_tarball,
}
c_type = requests_file.headers['content-type']
c_type = c_type.split(';')[0] if ';' in c_type else c_type
if c_type not in self._mime_processors:
raise BuildPackageException('Unknown build package mime type: %s' % c_type)
self._package_directory = None
try:
self._package_directory = self._mime_processors[c_type](requests_file)
except Exception as ex:
raise BuildPackageException(ex.message)
def parse_dockerfile(self, subdirectory):
dockerfile_path = os.path.join(self._package_directory, subdirectory, 'Dockerfile')
if not os.path.exists(dockerfile_path):
if subdirectory:
message = 'Build package did not contain a Dockerfile at sub directory %s.' % subdirectory
else:
message = 'Build package did not contain a Dockerfile at the root directory.'
raise BuildPackageException(message)
with open(dockerfile_path, 'r') as dockerfileobj:
return parse_dockerfile(dockerfileobj.read())
@staticmethod
def from_url(url):
buildpack_resource = requests.get(url, stream=True)
return BuildPackage(buildpack_resource)
@staticmethod
def _prepare_zip(request_file):
build_dir = mkdtemp(prefix='docker-build-')
# Save the zip file to temp somewhere
with TemporaryFile() as zip_file:
zip_file.write(request_file.content)
to_extract = ZipFile(zip_file)
to_extract.extractall(build_dir)
return build_dir
@staticmethod
def _prepare_dockerfile(request_file):
build_dir = mkdtemp(prefix='docker-build-')
dockerfile_path = os.path.join(build_dir, "Dockerfile")
with open(dockerfile_path, 'w') as dockerfile:
dockerfile.write(request_file.content)
return build_dir
@staticmethod
def _prepare_tarball(request_file):
build_dir = mkdtemp(prefix='docker-build-')
# Save the zip file to temp somewhere
with tarfile.open(mode='r|*', fileobj=request_file.raw) as tar_stream:
safe_extractall(tar_stream, build_dir)
return build_dir

72
buildman/jobutil/buildreporter.py Normal file
View file

@ -0,0 +1,72 @@
from trollius import From
from buildman.enums import BuildJobResult
from util.cloudwatch import get_queue
class BuildReporter(object):
"""
Base class for reporting build statuses to a metrics service.
"""
def report_completion_status(self, status):
"""
Method to invoke the recording of build's completion status to a metric service.
"""
raise NotImplementedError
class NullReporter(BuildReporter):
"""
The /dev/null of BuildReporters.
"""
def report_completion_status(self, *args):
pass
class CloudWatchBuildReporter(BuildReporter):
"""
Implements a BuildReporter for Amazon's CloudWatch.
"""
def __init__(self, queue, namespace_name, completed_name, failed_name, incompleted_name):
self._queue = queue
self._namespace_name = namespace_name
self._completed_name = completed_name
self._failed_name = failed_name
self._incompleted_name = incompleted_name
def _send_to_queue(self, *args, **kwargs):
self._queue.put((args, kwargs))
def report_completion_status(self, status):
if status == BuildJobResult.COMPLETE:
status_name = self._completed_name
elif status == BuildJobResult.ERROR:
status_name = self._failed_name
elif status == BuildJobResult.INCOMPLETE:
status_name = self._incompleted_name
else:
return
self._send_to_queue(self._namespace_name, status_name, 1, unit='Count')
class BuildMetrics(object):
"""
BuildMetrics initializes a reporter for recording the status of build completions.
"""
def __init__(self, app=None):
self._app = app
self._reporter = NullReporter()
if app is not None:
reporter_type = app.config.get('BUILD_METRICS_TYPE', 'Null')
if reporter_type == 'CloudWatch':
namespace = app.config['BUILD_METRICS_NAMESPACE']
completed_name = app.config['BUILD_METRICS_COMPLETED_NAME']
failed_name = app.config['BUILD_METRICS_FAILED_NAME']
incompleted_name = app.config['BUILD_METRICS_INCOMPLETED_NAME']
request_queue = get_queue(app)
self._reporter = CloudWatchBuildReporter(request_queue, namespace, completed_name,
failed_name, incompleted_name)
def __getattr__(self, name):
return getattr(self._reporter, name, None)

31
buildman/jobutil/buildstatus.py
View file

@ -1,16 +1,18 @@
from data.database import BUILD_PHASE
from data import model
import datetime
class StatusHandler(object):
""" Context wrapper for writing status to build logs. """
def __init__(self, build_logs, repository_build):
def __init__(self, build_logs, repository_build_uuid):
self._current_phase = None
self._repository_build = repository_build
self._uuid = repository_build.uuid
self._current_command = None
self._uuid = repository_build_uuid
self._build_logs = build_logs
self._status = {
'total_commands': None,
'total_commands': 0,
'current_command': None,
'push_completion': 0.0,
'pull_completion': 0.0,
@ -20,16 +22,25 @@ class StatusHandler(object):
self.__exit__(None, None, None)
def _append_log_message(self, log_message, log_type=None, log_data=None):
log_data = log_data or {}
log_data['datetime'] = str(datetime.datetime.now())
self._build_logs.append_log_message(self._uuid, log_message, log_type, log_data)
def append_log(self, log_message, extra_data=None):
if log_message is None:
return
self._append_log_message(log_message, log_data=extra_data)
def set_command(self, command, extra_data=None):
if self._current_command == command:
return
self._current_command = command
self._append_log_message(command, self._build_logs.COMMAND, extra_data)
def set_error(self, error_message, extra_data=None, internal_error=False):
self.set_phase(BUILD_PHASE.INTERNAL_ERROR if internal_error else BUILD_PHASE.ERROR)
def set_error(self, error_message, extra_data=None, internal_error=False, requeued=False):
self.set_phase(BUILD_PHASE.INTERNAL_ERROR if internal_error and requeued else BUILD_PHASE.ERROR)
extra_data = extra_data or {}
extra_data['internal_error'] = internal_error
@ -41,8 +52,12 @@ class StatusHandler(object):
self._current_phase = phase
self._append_log_message(phase, self._build_logs.PHASE, extra_data)
self._repository_build.phase = phase
self._repository_build.save()
# Update the repository build with the new phase
repo_build = model.get_repository_build(self._uuid)
repo_build.phase = phase
repo_build.save()
return True
def __enter__(self):

13
buildman/jobutil/workererror.py
View file

@ -19,13 +19,19 @@ class WorkerError(object):
'is_internal': True
},
'io.quay.builder.dockerfileissue': {
'message': 'Could not find or parse Dockerfile',
'show_base_error': True
},
'io.quay.builder.cannotpullbaseimage': {
'message': 'Could not pull base image',
'show_base_error': True
},
'io.quay.builder.internalerror': {
'message': 'An internal error occurred while building. Please submit a ticket.'
'message': 'An internal error occurred while building. Please submit a ticket.',
'is_internal': True
},
'io.quay.builder.buildrunerror': {
@ -57,6 +63,11 @@ class WorkerError(object):
'io.quay.builder.missingorinvalidargument': {
'message': 'Missing required arguments for builder',
'is_internal': True
},
'io.quay.builder.cachelookupissue': {
'message': 'Error checking for a cached tag',
'is_internal': True
}
}

29
buildman/manager/basemanager.py
View file

@ -1,12 +1,17 @@
from trollius import coroutine
class BaseManager(object):
""" Base for all worker managers. """
def __init__(self, register_component, unregister_component, job_heartbeat_callback,
job_complete_callback):
job_complete_callback, manager_hostname, heartbeat_period_sec):
self.register_component = register_component
self.unregister_component = unregister_component
self.job_heartbeat_callback = job_heartbeat_callback
self.job_complete_callback = job_complete_callback
self.manager_hostname = manager_hostname
self.heartbeat_period_sec = heartbeat_period_sec
@coroutine
def job_heartbeat(self, build_job):
""" Method invoked to tell the manager that a job is still running. This method will be called
every few minutes. """
@ -25,25 +30,41 @@ class BaseManager(object):
"""
raise NotImplementedError
def schedule(self, build_job, loop):
@coroutine
def schedule(self, build_job):
""" Schedules a queue item to be built. Returns True if the item was properly scheduled
and False if all workers are busy.
"""
raise NotImplementedError
def initialize(self):
def initialize(self, manager_config):
""" Runs any initialization code for the manager. Called once the server is in a ready state.
"""
raise NotImplementedError
@coroutine
def build_component_ready(self, build_component):
""" Method invoked whenever a build component announces itself as ready.
"""
raise NotImplementedError
def build_component_disposed(self, build_component, timed_out):
""" Method invoked whenever a build component has been disposed. The timed_out boolean indicates
whether the component's heartbeat timed out.
"""
raise NotImplementedError
@coroutine
def job_completed(self, build_job, job_status, build_component):
""" Method invoked once a job_item has completed, in some manner. The job_status will be
one of: incomplete, error, complete. If incomplete, the job should be requeued.
one of: incomplete, error, complete. Implementations of this method should call
self.job_complete_callback with a status of Incomplete if they wish for the job to be
automatically requeued.
"""
raise NotImplementedError
def num_workers(self):
""" Returns the number of active build workers currently registered. This includes those
that are currently busy and awaiting more work.
"""
raise NotImplementedError

49
buildman/manager/enterprise.py
View file

@ -5,7 +5,7 @@ from buildman.component.basecomponent import BaseComponent
from buildman.component.buildcomponent import BuildComponent
from buildman.manager.basemanager import BaseManager
from trollius.coroutines import From
from trollius import From, Return, coroutine
REGISTRATION_REALM = 'registration'
logger = logging.getLogger(__name__)
@ -25,13 +25,21 @@ class DynamicRegistrationComponent(BaseComponent):
logger.debug('Registering new build component+worker with realm %s', realm)
return realm
def kind(self):
return 'registration'
class EnterpriseManager(BaseManager):
""" Build manager implementation for the Enterprise Registry. """
build_components = []
shutting_down = False
def initialize(self):
def __init__(self, *args, **kwargs):
self.ready_components = set()
self.all_components = set()
self.shutting_down = False
super(EnterpriseManager, self).__init__(*args, **kwargs)
def initialize(self, manager_config):
# Add a component which is used by build workers for dynamic registration. Unlike
# production, build workers in enterprise are long-lived and register dynamically.
self.register_component(REGISTRATION_REALM, DynamicRegistrationComponent)
@ -45,28 +53,39 @@ class EnterpriseManager(BaseManager):
""" Adds a new build component for an Enterprise Registry. """
# Generate a new unique realm ID for the build worker.
realm = str(uuid.uuid4())
component = self.register_component(realm, BuildComponent, token="")
self.build_components.append(component)
new_component = self.register_component(realm, BuildComponent, token="")
self.all_components.add(new_component)
return realm
def schedule(self, build_job, loop):
@coroutine
def schedule(self, build_job):
""" Schedules a build for an Enterprise Registry. """
if self.shutting_down:
return False
if self.shutting_down or not self.ready_components:
raise Return(False)
for component in self.build_components:
if component.is_ready():
loop.call_soon(component.start_build, build_job)
return True
component = self.ready_components.pop()
return False
yield From(component.start_build(build_job))
raise Return(True)
@coroutine
def build_component_ready(self, build_component):
self.ready_components.add(build_component)
def shutdown(self):
self.shutting_down = True
@coroutine
def job_completed(self, build_job, job_status, build_component):
self.job_complete_callback(build_job, job_status)
def build_component_disposed(self, build_component, timed_out):
self.build_components.remove(build_component)
self.all_components.remove(build_component)
if build_component in self.ready_components:
self.ready_components.remove(build_component)
self.unregister_component(build_component)
def num_workers(self):
return len(self.all_components)

326
buildman/manager/ephemeral.py Normal file
View file

@ -0,0 +1,326 @@
import logging
import etcd
import uuid
import calendar
import os.path
import json
from datetime import datetime, timedelta
from trollius import From, coroutine, Return, async
from concurrent.futures import ThreadPoolExecutor
from urllib3.exceptions import ReadTimeoutError, ProtocolError
from buildman.manager.basemanager import BaseManager
from buildman.manager.executor import PopenExecutor, EC2Executor
from buildman.component.buildcomponent import BuildComponent
from buildman.jobutil.buildjob import BuildJob
from buildman.asyncutil import AsyncWrapper
from util.morecollections import AttrDict
logger = logging.getLogger(__name__)
ETCD_DISABLE_TIMEOUT = 0
class EtcdAction(object):
GET = 'get'
SET = 'set'
EXPIRE = 'expire'
UPDATE = 'update'
DELETE = 'delete'
CREATE = 'create'
COMPARE_AND_SWAP = 'compareAndSwap'
COMPARE_AND_DELETE = 'compareAndDelete'
class EphemeralBuilderManager(BaseManager):
""" Build manager implementation for the Enterprise Registry. """
_executors = {
'popen': PopenExecutor,
'ec2': EC2Executor,
}
_etcd_client_klass = etcd.Client
def __init__(self, *args, **kwargs):
self._shutting_down = False
self._manager_config = None
self._async_thread_executor = None
self._etcd_client = None
self._etcd_realm_prefix = None
self._etcd_builder_prefix = None
self._component_to_job = {}
self._job_uuid_to_component = {}
self._component_to_builder = {}
self._executor = None
# Map of etcd keys being watched to the tasks watching them
self._watch_tasks = {}
super(EphemeralBuilderManager, self).__init__(*args, **kwargs)
def _watch_etcd(self, etcd_key, change_callback, recursive=True):
watch_task_key = (etcd_key, recursive)
def callback_wrapper(changed_key_future):
if watch_task_key not in self._watch_tasks or self._watch_tasks[watch_task_key].done():
self._watch_etcd(etcd_key, change_callback)
if changed_key_future.cancelled():
# Due to lack of interest, tomorrow has been cancelled
return
try:
etcd_result = changed_key_future.result()
except (ReadTimeoutError, ProtocolError):
return
change_callback(etcd_result)
if not self._shutting_down:
watch_future = self._etcd_client.watch(etcd_key, recursive=recursive,
timeout=ETCD_DISABLE_TIMEOUT)
watch_future.add_done_callback(callback_wrapper)
logger.debug('Scheduling watch of key: %s%s', etcd_key, '/*' if recursive else '')
self._watch_tasks[watch_task_key] = async(watch_future)
def _handle_builder_expiration(self, etcd_result):
if etcd_result.action == EtcdAction.EXPIRE:
# Handle the expiration
logger.debug('Builder expired, clean up the old build node')
job_metadata = json.loads(etcd_result._prev_node.value)
if 'builder_id' in job_metadata:
logger.info('Terminating expired build node.')
async(self._executor.stop_builder(job_metadata['builder_id']))
def _handle_realm_change(self, etcd_result):
if etcd_result.action == EtcdAction.CREATE:
# We must listen on the realm created by ourselves or another worker
realm_spec = json.loads(etcd_result.value)
self._register_realm(realm_spec)
elif etcd_result.action == EtcdAction.DELETE or etcd_result.action == EtcdAction.EXPIRE:
# We must stop listening for new connections on the specified realm, if we did not get the
# connection
realm_spec = json.loads(etcd_result._prev_node.value)
build_job = BuildJob(AttrDict(realm_spec['job_queue_item']))
component = self._job_uuid_to_component.pop(build_job.job_details['build_uuid'], None)
if component is not None:
# We were not the manager which the worker connected to, remove the bookkeeping for it
logger.debug('Unregistering unused component on realm: %s', realm_spec['realm'])
del self._component_to_job[component]
del self._component_to_builder[component]
self.unregister_component(component)
else:
logger.warning('Unexpected action (%s) on realm key: %s', etcd_result.action, etcd_result.key)
def _register_realm(self, realm_spec):
logger.debug('Registering realm with manager: %s', realm_spec['realm'])
component = self.register_component(realm_spec['realm'], BuildComponent,
token=realm_spec['token'])
build_job = BuildJob(AttrDict(realm_spec['job_queue_item']))
self._component_to_job[component] = build_job
self._component_to_builder[component] = realm_spec['builder_id']
self._job_uuid_to_component[build_job.job_details['build_uuid']] = component
@coroutine
def _register_existing_realms(self):
try:
all_realms = yield From(self._etcd_client.read(self._etcd_realm_prefix, recursive=True))
for realm in all_realms.children:
if not realm.dir:
self._register_realm(json.loads(realm.value))
except KeyError:
# no realms have been registered yet
pass
def initialize(self, manager_config):
logger.debug('Calling initialize')
self._manager_config = manager_config
executor_klass = self._executors.get(manager_config.get('EXECUTOR', ''), PopenExecutor)
self._executor = executor_klass(manager_config.get('EXECUTOR_CONFIG', {}),
self.manager_hostname)
etcd_host = self._manager_config.get('ETCD_HOST', '127.0.0.1')
etcd_port = self._manager_config.get('ETCD_PORT', 2379)
etcd_auth = self._manager_config.get('ETCD_CERT_AND_KEY', None)
etcd_ca_cert = self._manager_config.get('ETCD_CA_CERT', None)
etcd_protocol = 'http' if etcd_auth is None else 'https'
logger.debug('Connecting to etcd on %s:%s', etcd_host, etcd_port)
worker_threads = self._manager_config.get('ETCD_WORKER_THREADS', 5)
self._async_thread_executor = ThreadPoolExecutor(worker_threads)
self._etcd_client = AsyncWrapper(self._etcd_client_klass(host=etcd_host, port=etcd_port,
cert=etcd_auth, ca_cert=etcd_ca_cert,
protocol=etcd_protocol),
executor=self._async_thread_executor)
self._etcd_builder_prefix = self._manager_config.get('ETCD_BUILDER_PREFIX', 'building/')
self._watch_etcd(self._etcd_builder_prefix, self._handle_builder_expiration)
self._etcd_realm_prefix = self._manager_config.get('ETCD_REALM_PREFIX', 'realm/')
self._watch_etcd(self._etcd_realm_prefix, self._handle_realm_change)
# Load components for all realms currently known to the cluster
async(self._register_existing_realms())
def setup_time(self):
setup_time = self._manager_config.get('MACHINE_SETUP_TIME', 300)
return setup_time
def shutdown(self):
logger.debug('Shutting down worker.')
self._shutting_down = True
for (etcd_key, _), task in self._watch_tasks.items():
if not task.done():
logger.debug('Canceling watch task for %s', etcd_key)
task.cancel()
if self._async_thread_executor is not None:
logger.debug('Shutting down thread pool executor.')
self._async_thread_executor.shutdown()
@coroutine
def schedule(self, build_job):
build_uuid = build_job.job_details['build_uuid']
logger.debug('Calling schedule with job: %s', build_uuid)
# Check if there are worker slots avialable by checking the number of jobs in etcd
allowed_worker_count = self._manager_config.get('ALLOWED_WORKER_COUNT', 1)
try:
building = yield From(self._etcd_client.read(self._etcd_builder_prefix, recursive=True))
workers_alive = sum(1 for child in building.children if not child.dir)
except KeyError:
workers_alive = 0
logger.debug('Total jobs: %s', workers_alive)
if workers_alive >= allowed_worker_count:
logger.info('Too many workers alive, unable to start new worker. %s >= %s', workers_alive,
allowed_worker_count)
raise Return(False)
job_key = self._etcd_job_key(build_job)
# First try to take a lock for this job, meaning we will be responsible for its lifeline
realm = str(uuid.uuid4())
token = str(uuid.uuid4())
ttl = self.setup_time()
expiration = datetime.utcnow() + timedelta(seconds=ttl)
machine_max_expiration = self._manager_config.get('MACHINE_MAX_TIME', 7200)
max_expiration = datetime.utcnow() + timedelta(seconds=machine_max_expiration)
payload = {
'expiration': calendar.timegm(expiration.timetuple()),
'max_expiration': calendar.timegm(max_expiration.timetuple()),
}
try:
yield From(self._etcd_client.write(job_key, json.dumps(payload), prevExist=False, ttl=ttl))
except KeyError:
# The job was already taken by someone else, we are probably a retry
logger.error('Job already exists in etcd, are timeouts misconfigured or is the queue broken?')
raise Return(False)
logger.debug('Starting builder with executor: %s', self._executor)
builder_id = yield From(self._executor.start_builder(realm, token, build_uuid))
# Store the builder in etcd associated with the job id
payload['builder_id'] = builder_id
yield From(self._etcd_client.write(job_key, json.dumps(payload), prevExist=True, ttl=ttl))
# Store the realm spec which will allow any manager to accept this builder when it connects
realm_spec = json.dumps({
'realm': realm,
'token': token,
'builder_id': builder_id,
'job_queue_item': build_job.job_item,
})
try:
yield From(self._etcd_client.write(self._etcd_realm_key(realm), realm_spec, prevExist=False,
ttl=ttl))
except KeyError:
logger.error('Realm already exists in etcd. UUID collision or something is very very wrong.')
raise Return(False)
raise Return(True)
@coroutine
def build_component_ready(self, build_component):
try:
# Clean up the bookkeeping for allowing any manager to take the job
job = self._component_to_job.pop(build_component)
del self._job_uuid_to_component[job.job_details['build_uuid']]
yield From(self._etcd_client.delete(self._etcd_realm_key(build_component.builder_realm)))
logger.debug('Sending build %s to newly ready component on realm %s',
job.job_details['build_uuid'], build_component.builder_realm)
yield From(build_component.start_build(job))
except KeyError:
logger.debug('Builder is asking for more work, but work already completed')
def build_component_disposed(self, build_component, timed_out):
logger.debug('Calling build_component_disposed.')
self.unregister_component(build_component)
@coroutine
def job_completed(self, build_job, job_status, build_component):
logger.debug('Calling job_completed with status: %s', job_status)
# Kill the ephmeral builder
yield From(self._executor.stop_builder(self._component_to_builder.pop(build_component)))
# Release the lock in etcd
job_key = self._etcd_job_key(build_job)
yield From(self._etcd_client.delete(job_key))
self.job_complete_callback(build_job, job_status)
@coroutine
def job_heartbeat(self, build_job):
# Extend the deadline in etcd
job_key = self._etcd_job_key(build_job)
build_job_metadata_response = yield From(self._etcd_client.read(job_key))
build_job_metadata = json.loads(build_job_metadata_response.value)
max_expiration = datetime.utcfromtimestamp(build_job_metadata['max_expiration'])
max_expiration_remaining = max_expiration - datetime.utcnow()
max_expiration_sec = max(0, int(max_expiration_remaining.total_seconds()))
ttl = min(self.heartbeat_period_sec * 2, max_expiration_sec)
new_expiration = datetime.utcnow() + timedelta(seconds=ttl)
payload = {
'expiration': calendar.timegm(new_expiration.timetuple()),
'builder_id': build_job_metadata['builder_id'],
'max_expiration': build_job_metadata['max_expiration'],
}
yield From(self._etcd_client.write(job_key, json.dumps(payload), ttl=ttl))
self.job_heartbeat_callback(build_job)
def _etcd_job_key(self, build_job):
""" Create a key which is used to track a job in etcd.
"""
return os.path.join(self._etcd_builder_prefix, build_job.job_details['build_uuid'])
def _etcd_realm_key(self, realm):
""" Create a key which is used to track an incoming connection on a realm.
"""
return os.path.join(self._etcd_realm_prefix, realm)
def num_workers(self):
""" Return the number of workers we're managing locally.
"""
return len(self._component_to_builder)

238
buildman/manager/executor.py Normal file
View file

@ -0,0 +1,238 @@
import logging
import os
import uuid
import threading
import boto.ec2
import requests
import cachetools
from jinja2 import FileSystemLoader, Environment
from trollius import coroutine, From, Return, get_event_loop
from functools import partial
from buildman.asyncutil import AsyncWrapper
from container_cloud_config import CloudConfigContext
logger = logging.getLogger(__name__)
ONE_HOUR = 60*60
ENV = Environment(loader=FileSystemLoader('buildman/templates'))
TEMPLATE = ENV.get_template('cloudconfig.yaml')
CloudConfigContext().populate_jinja_environment(ENV)
class ExecutorException(Exception):
""" Exception raised when there is a problem starting or stopping a builder.
"""
pass
class BuilderExecutor(object):
def __init__(self, executor_config, manager_hostname):
self.executor_config = executor_config
self.manager_hostname = manager_hostname
""" Interface which can be plugged into the EphemeralNodeManager to provide a strategy for
starting and stopping builders.
"""
@coroutine
def start_builder(self, realm, token, build_uuid):
""" Create a builder with the specified config. Returns a unique id which can be used to manage
the builder.
"""
raise NotImplementedError
@coroutine
def stop_builder(self, builder_id):
""" Stop a builder which is currently running.
"""
raise NotImplementedError
def get_manager_websocket_url(self):
return 'ws://{0}:'
def generate_cloud_config(self, realm, token, coreos_channel, manager_hostname,
quay_username=None, quay_password=None):
if quay_username is None:
quay_username = self.executor_config['QUAY_USERNAME']
if quay_password is None:
quay_password = self.executor_config['QUAY_PASSWORD']
return TEMPLATE.render(
realm=realm,
token=token,
quay_username=quay_username,
quay_password=quay_password,
manager_hostname=manager_hostname,
coreos_channel=coreos_channel,
worker_tag=self.executor_config['WORKER_TAG'],
)
class EC2Executor(BuilderExecutor):
""" Implementation of BuilderExecutor which uses libcloud to start machines on a variety of cloud
providers.
"""
COREOS_STACK_URL = 'http://%s.release.core-os.net/amd64-usr/current/coreos_production_ami_hvm.txt'
def __init__(self, *args, **kwargs):
self._loop = get_event_loop()
super(EC2Executor, self).__init__(*args, **kwargs)
def _get_conn(self):
""" Creates an ec2 connection which can be used to manage instances.
"""
return AsyncWrapper(boto.ec2.connect_to_region(
self.executor_config['EC2_REGION'],
aws_access_key_id=self.executor_config['AWS_ACCESS_KEY'],
aws_secret_access_key=self.executor_config['AWS_SECRET_KEY'],
))
@classmethod
@cachetools.ttl_cache(ttl=ONE_HOUR)
def _get_coreos_ami(cls, ec2_region, coreos_channel):
""" Retrieve the CoreOS AMI id from the canonical listing.
"""
stack_list_string = requests.get(EC2Executor.COREOS_STACK_URL % coreos_channel).text
stack_amis = dict([stack.split('=') for stack in stack_list_string.split('|')])
return stack_amis[ec2_region]
@coroutine
def start_builder(self, realm, token, build_uuid):
region = self.executor_config['EC2_REGION']
channel = self.executor_config.get('COREOS_CHANNEL', 'stable')
get_ami_callable = partial(self._get_coreos_ami, region, channel)
coreos_ami = yield From(self._loop.run_in_executor(None, get_ami_callable))
user_data = self.generate_cloud_config(realm, token, channel, self.manager_hostname)
logger.debug('Generated cloud config: %s', user_data)
ec2_conn = self._get_conn()
ssd_root_ebs = boto.ec2.blockdevicemapping.BlockDeviceType(
size=32,
volume_type='gp2',
delete_on_termination=True,
)
block_devices = boto.ec2.blockdevicemapping.BlockDeviceMapping()
block_devices['/dev/xvda'] = ssd_root_ebs
interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(
subnet_id=self.executor_config['EC2_VPC_SUBNET_ID'],
groups=self.executor_config['EC2_SECURITY_GROUP_IDS'],
associate_public_ip_address=True,
)
interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(interface)
reservation = yield From(ec2_conn.run_instances(
coreos_ami,
instance_type=self.executor_config['EC2_INSTANCE_TYPE'],
key_name=self.executor_config.get('EC2_KEY_NAME', None),
user_data=user_data,
instance_initiated_shutdown_behavior='terminate',
block_device_map=block_devices,
network_interfaces=interfaces,
))
if not reservation.instances:
raise ExecutorException('Unable to spawn builder instance.')
elif len(reservation.instances) != 1:
raise ExecutorException('EC2 started wrong number of instances!')
launched = AsyncWrapper(reservation.instances[0])
yield From(launched.add_tags({
'Name': 'Quay Ephemeral Builder',
'Realm': realm,
'Token': token,
'BuildUUID': build_uuid,
}))
raise Return(launched.id)
@coroutine
def stop_builder(self, builder_id):
ec2_conn = self._get_conn()
terminated_instances = yield From(ec2_conn.terminate_instances([builder_id]))
if builder_id not in [si.id for si in terminated_instances]:
raise ExecutorException('Unable to terminate instance: %s' % builder_id)
class PopenExecutor(BuilderExecutor):
""" Implementation of BuilderExecutor which uses Popen to fork a quay-builder process.
"""
def __init__(self, executor_config, manager_hostname):
self._jobs = {}
super(PopenExecutor, self).__init__(executor_config, manager_hostname)
""" Executor which uses Popen to fork a quay-builder process.
"""
@coroutine
def start_builder(self, realm, token, build_uuid):
# Now start a machine for this job, adding the machine id to the etcd information
logger.debug('Forking process for build')
import subprocess
builder_env = {
'TOKEN': token,
'REALM': realm,
'ENDPOINT': 'ws://localhost:8787',
'DOCKER_TLS_VERIFY': os.environ.get('DOCKER_TLS_VERIFY', ''),
'DOCKER_CERT_PATH': os.environ.get('DOCKER_CERT_PATH', ''),
'DOCKER_HOST': os.environ.get('DOCKER_HOST', ''),
}
logpipe = LogPipe(logging.INFO)
spawned = subprocess.Popen('/Users/jake/bin/quay-builder', stdout=logpipe, stderr=logpipe,
env=builder_env)
builder_id = str(uuid.uuid4())
self._jobs[builder_id] = (spawned, logpipe)
logger.debug('Builder spawned with id: %s', builder_id)
raise Return(builder_id)
@coroutine
def stop_builder(self, builder_id):
if builder_id not in self._jobs:
raise ExecutorException('Builder id not being tracked by executor.')
logger.debug('Killing builder with id: %s', builder_id)
spawned, logpipe = self._jobs[builder_id]
if spawned.poll() is None:
spawned.kill()
logpipe.close()
class LogPipe(threading.Thread):
""" Adapted from http://codereview.stackexchange.com/a/17959
"""
def __init__(self, level):
"""Setup the object with a logger and a loglevel
and start the thread
"""
threading.Thread.__init__(self)
self.daemon = False
self.level = level
self.fd_read, self.fd_write = os.pipe()
self.pipe_reader = os.fdopen(self.fd_read)
self.start()
def fileno(self):
"""Return the write file descriptor of the pipe
"""
return self.fd_write
def run(self):
"""Run the thread, logging everything.
"""
for line in iter(self.pipe_reader.readline, ''):
logging.log(self.level, line.strip('\n'))
self.pipe_reader.close()
def close(self):
"""Close the write end of the pipe.
"""
os.close(self.fd_write)

113
buildman/server.py
View file

@ -1,5 +1,6 @@
import logging
import trollius
import json
from autobahn.asyncio.wamp import RouterFactory, RouterSessionFactory
from autobahn.asyncio.websocket import WampWebSocketServerFactory
@ -8,11 +9,15 @@ from autobahn.wamp import types
from aiowsgi import create_server as create_wsgi_server
from flask import Flask
from threading import Event
from trollius.tasks import Task
from trollius.coroutines import From
from datetime import datetime, timedelta
from datetime import timedelta
from buildman.enums import BuildJobResult, BuildServerStatus
from buildman.jobutil.buildstatus import StatusHandler
from buildman.jobutil.buildjob import BuildJob, BuildJobLoadException
from data.queue import WorkQueue
from data import database
from app import app, build_metrics
logger = logging.getLogger(__name__)
@ -21,27 +26,21 @@ TIMEOUT_PERIOD_MINUTES = 20
JOB_TIMEOUT_SECONDS = 300
MINIMUM_JOB_EXTENSION = timedelta(minutes=2)
WEBSOCKET_PORT = 8787
CONTROLLER_PORT = 8686
class BuildJobResult(object):
""" Build job result enum """
INCOMPLETE = 'incomplete'
COMPLETE = 'complete'
ERROR = 'error'
HEARTBEAT_PERIOD_SEC = 30
class BuilderServer(object):
""" Server which handles both HTTP and WAMP requests, managing the full state of the build
controller.
"""
def __init__(self, server_hostname, queue, build_logs, user_files, lifecycle_manager_klass):
def __init__(self, registry_hostname, queue, build_logs, user_files, lifecycle_manager_klass,
lifecycle_manager_config, manager_hostname):
self._loop = None
self._current_status = 'starting'
self._current_status = BuildServerStatus.STARTING
self._current_components = []
self._job_count = 0
self._session_factory = RouterSessionFactory(RouterFactory())
self._server_hostname = server_hostname
self._registry_hostname = registry_hostname
self._queue = queue
self._build_logs = build_logs
self._user_files = user_files
@ -49,11 +48,14 @@ class BuilderServer(object):
self._register_component,
self._unregister_component,
self._job_heartbeat,
self._job_complete
self._job_complete,
manager_hostname,
HEARTBEAT_PERIOD_SEC,
)
self._lifecycle_manager_config = lifecycle_manager_config
self._shutdown_event = Event()
self._current_status = 'running'
self._current_status = BuildServerStatus.RUNNING
self._register_controller()
@ -63,22 +65,41 @@ class BuilderServer(object):
@controller_app.route('/status')
def status():
return server._current_status
metrics = server._queue.get_metrics(require_transaction=False)
(running_count, available_not_running_count, available_count) = metrics
workers = [component for component in server._current_components
if component.kind() == 'builder']
data = {
'status': server._current_status,
'running_local': server._job_count,
'running_total': running_count,
'workers': len(workers),
'job_total': available_count + running_count
}
return json.dumps(data)
self._controller_app = controller_app
def run(self, host, ssl=None):
def run(self, host, websocket_port, controller_port, ssl=None):
logger.debug('Initializing the lifecycle manager')
self._lifecycle_manager.initialize()
self._lifecycle_manager.initialize(self._lifecycle_manager_config)
logger.debug('Initializing all members of the event loop')
loop = trollius.get_event_loop()
trollius.Task(self._initialize(loop, host, ssl))
logger.debug('Starting server on port %s, with controller on port %s', WEBSOCKET_PORT,
CONTROLLER_PORT)
logger.debug('Starting server on port %s, with controller on port %s', websocket_port,
controller_port)
TASKS = [
Task(self._initialize(loop, host, websocket_port, controller_port, ssl)),
Task(self._queue_metrics_updater()),
]
try:
loop.run_forever()
loop.run_until_complete(trollius.wait(TASKS))
except KeyboardInterrupt:
pass
finally:
@ -86,7 +107,7 @@ class BuilderServer(object):
def close(self):
logger.debug('Requested server shutdown')
self._current_status = 'shutting_down'
self._current_status = BuildServerStatus.SHUTDOWN
self._lifecycle_manager.shutdown()
self._shutdown_event.wait()
logger.debug('Shutting down server')
@ -102,7 +123,7 @@ class BuilderServer(object):
component.parent_manager = self._lifecycle_manager
component.build_logs = self._build_logs
component.user_files = self._user_files
component.server_hostname = self._server_hostname
component.registry_hostname = self._registry_hostname
self._current_components.append(component)
self._session_factory.add(component)
@ -116,32 +137,34 @@ class BuilderServer(object):
self._session_factory.remove(component)
def _job_heartbeat(self, build_job):
WorkQueue.extend_processing(build_job.job_item(), seconds_from_now=JOB_TIMEOUT_SECONDS,
retry_count=1, minimum_extension=MINIMUM_JOB_EXTENSION)
self._queue.extend_processing(build_job.job_item, seconds_from_now=JOB_TIMEOUT_SECONDS,
minimum_extension=MINIMUM_JOB_EXTENSION)
def _job_complete(self, build_job, job_status):
if job_status == BuildJobResult.INCOMPLETE:
self._queue.incomplete(build_job.job_item(), restore_retry=True, retry_after=30)
elif job_status == BuildJobResult.ERROR:
self._queue.incomplete(build_job.job_item(), restore_retry=False)
self._queue.incomplete(build_job.job_item, restore_retry=False, retry_after=30)
else:
self._queue.complete(build_job.job_item())
self._queue.complete(build_job.job_item)
self._job_count = self._job_count - 1
if self._current_status == 'shutting_down' and not self._job_count:
if self._current_status == BuildServerStatus.SHUTDOWN and not self._job_count:
self._shutdown_event.set()
# TODO(jschorr): check for work here?
build_metrics.report_completion_status(job_status)
@trollius.coroutine
def _work_checker(self):
while self._current_status == 'running':
logger.debug('Checking for more work')
while self._current_status == BuildServerStatus.RUNNING:
with database.CloseForLongOperation(app.config):
yield From(trollius.sleep(WORK_CHECK_TIMEOUT))
logger.debug('Checking for more work for %d active workers',
self._lifecycle_manager.num_workers())
job_item = self._queue.get(processing_time=self._lifecycle_manager.setup_time())
if job_item is None:
logger.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT)
yield From(trollius.sleep(WORK_CHECK_TIMEOUT))
continue
try:
@ -149,20 +172,28 @@ class BuilderServer(object):
except BuildJobLoadException as irbe:
logger.exception(irbe)
self._queue.incomplete(job_item, restore_retry=False)
continue
logger.debug('Build job found. Checking for an avaliable worker.')
if self._lifecycle_manager.schedule(build_job, self._loop):
scheduled = yield From(self._lifecycle_manager.schedule(build_job))
if scheduled:
status_handler = StatusHandler(self._build_logs, build_job.repo_build.uuid)
status_handler.set_phase('build-scheduled')
self._job_count = self._job_count + 1
logger.debug('Build job scheduled. Running: %s', self._job_count)
else:
logger.debug('All workers are busy. Requeuing.')
self._queue.incomplete(job_item, restore_retry=True, retry_after=0)
yield From(trollius.sleep(WORK_CHECK_TIMEOUT))
@trollius.coroutine
def _queue_metrics_updater(self):
while self._current_status == BuildServerStatus.RUNNING:
yield From(trollius.sleep(30))
self._queue.update_metrics()
@trollius.coroutine
def _initialize(self, loop, host, ssl=None):
def _initialize(self, loop, host, websocket_port, controller_port, ssl=None):
self._loop = loop
# Create the WAMP server.
@ -170,8 +201,8 @@ class BuilderServer(object):
transport_factory.setProtocolOptions(failByDrop=True)
# Initialize the controller server and the WAMP server
create_wsgi_server(self._controller_app, loop=loop, host=host, port=CONTROLLER_PORT, ssl=ssl)
yield From(loop.create_server(transport_factory, host, WEBSOCKET_PORT, ssl=ssl))
create_wsgi_server(self._controller_app, loop=loop, host=host, port=controller_port, ssl=ssl)
yield From(loop.create_server(transport_factory, host, websocket_port, ssl=ssl))
# Initialize the work queue checker.
yield From(self._work_checker())

31
buildman/templates/cloudconfig.yaml Normal file
View file

@ -0,0 +1,31 @@
#cloud-config
ssh_authorized_keys:
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCC0m+hVmyR3vn/xoxJe9+atRWBxSK+YXgyufNVDMcb7H00Jfnc341QH3kDVYZamUbhVh/nyc2RP7YbnZR5zORFtgOaNSdkMYrPozzBvxjnvSUokkCCWbLqXDHvIKiR12r+UTSijPJE/Yk702Mb2ejAFuae1C3Ec+qKAoOCagDjpQ3THyb5oaKE7VPHdwCWjWIQLRhC+plu77ObhoXIFJLD13gCi01L/rp4mYVCxIc2lX5A8rkK+bZHnIZwWUQ4t8SIjWxIaUo0FE7oZ83nKuNkYj5ngmLHQLY23Nx2WhE9H6NBthUpik9SmqQPtVYbhIG+bISPoH9Xs8CLrFb0VRjz Joey's Mac
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCo6FhAP7mFFOAzM91gtaKW7saahtaN4lur42FMMztz6aqUycIltCmvxo+3FmrXgCG30maMNU36Vm1+9QRtVQEd+eRuoIWP28t+8MT01Fh4zPuE2Wca3pOHSNo3X81FfWJLzmwEHiQKs9HPQqUhezR9PcVWVkbMyAzw85c0UycGmHGFNb0UiRd9HFY6XbgbxhZv/mvKLZ99xE3xkOzS1PNsdSNvjUKwZR7pSUPqNS5S/1NXyR4GhFTU24VPH/bTATOv2ATH+PSzsZ7Qyz9UHj38tKC+ALJHEDJ4HXGzobyOUP78cHGZOfCB5FYubq0zmOudAjKIAhwI8XTFvJ2DX1P3 jimmyzelinskie
- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNvw8qo9m8np7yQ/Smv/oklM8bo8VyNRZriGYBDuolWDL/mZpYCQnZJXphQo7RFdNABYistikjJlBuuwUohLf2uSq0iKoFa2TgwI43wViWzvuzU4nA02/ITD5BZdmWAFNyIoqeB50Ol4qUgDwLAZ+7Kv7uCi6chcgr9gTi99jY3GHyZjrMiXMHGVGi+FExFuzhVC2drKjbz5q6oRfQeLtNfG4psl5GU3MQU6FkX4fgoCx0r9R48/b7l4+TT7pWblJQiRfeldixu6308vyoTUEHasdkU3/X0OTaGz/h5XqTKnGQc6stvvoED3w+L3QFp0H5Z8sZ9stSsitmCBrmbcKZ jakemoshenko
write_files:
- path: /root/overrides.list
permission: '0644'
content: |
REALM={{ realm }}
TOKEN={{ token }}
SERVER=wss://{{ manager_hostname }}
coreos:
update:
reboot-strategy: off
group: {{ coreos_channel }}
units:
{{ dockersystemd('quay-builder',
'quay.io/coreos/registry-build-worker',
quay_username,
quay_password,
worker_tag,
extra_args='--net=host --privileged --env-file /root/overrides.list -v /var/run/docker.sock:/var/run/docker.sock -v /usr/share/ca-certificates:/etc/ssl/certs',
exec_stop_post=['/bin/sh -xc "/bin/sleep 120; /usr/bin/systemctl --no-block poweroff"'],
flattened=True,
restart_policy='no'
) | indent(4) }}

4
conf/gunicorn_local.py
View file

@ -1,7 +1,7 @@
bind = '0.0.0.0:5000'
workers = 2
worker_class = 'gevent'
timeout = 2000
daemon = False
logconfig = 'conf/logging.conf'
logconfig = 'conf/logging_debug.conf'
pythonpath = '.'
preload_app = True

1
conf/gunicorn_registry.py
View file

@ -1,7 +1,6 @@
bind = 'unix:/tmp/gunicorn_registry.sock'
workers = 8
worker_class = 'gevent'
timeout = 2000
logconfig = 'conf/logging.conf'
pythonpath = '.'
preload_app = True

1
conf/gunicorn_verbs.py
View file

@ -1,6 +1,5 @@
bind = 'unix:/tmp/gunicorn_verbs.sock'
workers = 4
timeout = 2000
logconfig = 'conf/logging.conf'
pythonpath = '.'
preload_app = True

1
conf/gunicorn_web.py
View file

@ -1,7 +1,6 @@
bind = 'unix:/tmp/gunicorn_web.sock'
workers = 2
worker_class = 'gevent'
timeout = 30
logconfig = 'conf/logging.conf'
pythonpath = '.'
preload_app = True

2
conf/hosted-http-base.conf
View file

@ -1,3 +1,5 @@
# vim: ft=nginx
server {
listen 80 default_server;
server_name _;

2
conf/http-base.conf
View file

@ -1,3 +1,5 @@
# vim: ft=nginx
types_hash_max_size 2048;
include /usr/local/nginx/conf/mime.types.default;

2
conf/init/dockerfilebuild/log/run
View file

@ -1,2 +0,0 @@
#!/bin/sh
exec svlogd /var/log/dockerfilebuild/

6
conf/init/dockerfilebuild/run
View file

@ -1,6 +0,0 @@
#! /bin/bash
sv start tutumdocker || exit 1
cd /
venv/bin/python -m workers.dockerfilebuild

2
conf/init/gunicorn_registry/run
View file

@ -3,6 +3,6 @@
echo 'Starting gunicon'
cd /
venv/bin/gunicorn -c conf/gunicorn_registry.py registry:application
nice -n 10 venv/bin/gunicorn -c conf/gunicorn_registry.py registry:application
echo 'Gunicorn exited'

2
conf/init/gunicorn_verbs/run
View file

@ -3,6 +3,6 @@
echo 'Starting gunicon'
cd /
nice -10 venv/bin/gunicorn -c conf/gunicorn_verbs.py verbs:application
nice -n 10 venv/bin/gunicorn -c conf/gunicorn_verbs.py verbs:application
echo 'Gunicorn exited'

2
conf/init/tutumdocker/log/run
View file

@ -1,2 +0,0 @@
#!/bin/sh
exec svlogd /var/log/tutumdocker/

96
conf/init/tutumdocker/run
View file

@ -1,96 +0,0 @@
#!/bin/bash
# First, make sure that cgroups are mounted correctly.
CGROUP=/sys/fs/cgroup
[ -d $CGROUP ] ||
mkdir $CGROUP
mountpoint -q $CGROUP ||
mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || {
echo "Could not make a tmpfs mount. Did you use -privileged?"
exit 1
}
if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security
then
mount -t securityfs none /sys/kernel/security || {
echo "Could not mount /sys/kernel/security."
echo "AppArmor detection and -privileged mode might break."
}
fi
# Mount the cgroup hierarchies exactly as they are in the parent system.
for SUBSYS in $(cut -d: -f2 /proc/1/cgroup)
do
[ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS
mountpoint -q $CGROUP/$SUBSYS ||
mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS
# The two following sections address a bug which manifests itself
# by a cryptic "lxc-start: no ns_cgroup option specified" when
# trying to start containers withina container.
# The bug seems to appear when the cgroup hierarchies are not
# mounted on the exact same directories in the host, and in the
# container.
# Named, control-less cgroups are mounted with "-o name=foo"
# (and appear as such under /proc/<pid>/cgroup) but are usually
# mounted on a directory named "foo" (without the "name=" prefix).
# Systemd and OpenRC (and possibly others) both create such a
# cgroup. To avoid the aforementioned bug, we symlink "foo" to
# "name=foo". This shouldn't have any adverse effect.
echo $SUBSYS | grep -q ^name= && {
NAME=$(echo $SUBSYS | sed s/^name=//)
ln -s $SUBSYS $CGROUP/$NAME
}
# Likewise, on at least one system, it has been reported that
# systemd would mount the CPU and CPU accounting controllers
# (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu"
# but on a directory called "cpu,cpuacct" (note the inversion
# in the order of the groups). This tries to work around it.
[ $SUBSYS = cpuacct,cpu ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct
done
# Note: as I write those lines, the LXC userland tools cannot setup
# a "sub-container" properly if the "devices" cgroup is not in its
# own hierarchy. Let's detect this and issue a warning.
grep -q :devices: /proc/1/cgroup ||
echo "WARNING: the 'devices' cgroup should be in its own hierarchy."
grep -qw devices /proc/1/cgroup ||
echo "WARNING: it looks like the 'devices' cgroup is not mounted."
# Now, close extraneous file descriptors.
pushd /proc/self/fd >/dev/null
for FD in *
do
case "$FD" in
# Keep stdin/stdout/stderr
[012])
;;
# Nuke everything else
*)
eval exec "$FD>&-"
;;
esac
done
popd >/dev/null
# If a pidfile is still around (for example after a container restart),
# delete it so that docker can start.
rm -rf /var/run/docker.pid
chmod 777 /var/lib/lxc
chmod 777 /var/lib/docker
# If we were given a PORT environment variable, start as a simple daemon;
# otherwise, spawn a shell as well
if [ "$PORT" ]
then
exec docker -d -H 0.0.0.0:$PORT
else
docker -d -D -e lxc 2>&1
fi

32
conf/logging.conf
View file

@ -1,5 +1,5 @@
[loggers]
keys=root, gunicorn.error, gunicorn.access, application.profiler, boto, werkzeug
keys=root
[handlers]
keys=console
@ -7,39 +7,9 @@ keys=console
[formatters]
keys=generic
[logger_application.profiler]
level=DEBUG
handlers=console
propagate=0
qualname=application.profiler
[logger_root]
level=DEBUG
handlers=console
[logger_boto]
level=INFO
handlers=console
propagate=0
qualname=boto
[logger_werkzeug]
level=DEBUG
handlers=console
propagate=0
qualname=werkzeug
[logger_gunicorn.error]
level=INFO
handlers=console
propagate=1
qualname=gunicorn.error
[logger_gunicorn.access]
level=INFO
handlers=console
propagate=0
qualname=gunicorn.access
[handler_console]
class=StreamHandler

21
conf/logging_debug.conf Normal file
View file

@ -0,0 +1,21 @@
[loggers]
keys=root
[handlers]
keys=console
[formatters]
keys=generic
[logger_root]
level=DEBUG
handlers=console
[handler_console]
class=StreamHandler
formatter=generic
args=(sys.stdout, )
[formatter_generic]
format=%(asctime)s [%(process)d] [%(levelname)s] [%(name)s] %(message)s
class=logging.Formatter

10
conf/nginx-nossl.conf
View file

@ -1,14 +1,12 @@
# vim: ft=nginx
include root-base.conf;
worker_processes 2;
user root nogroup;
daemon off;
http {
include http-base.conf;
include rate-limiting.conf;
server {
include server-base.conf;

26
conf/nginx.conf
View file

@ -1,16 +1,14 @@
# vim: ft=nginx
include root-base.conf;
worker_processes 2;
user root nogroup;
daemon off;
http {
include http-base.conf;
include hosted-http-base.conf;
include rate-limiting.conf;
server {
include server-base.conf;
@ -24,4 +22,20 @@ http {
ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP;
ssl_prefer_server_ciphers on;
}
server {
include proxy-protocol.conf;
include proxy-server-base.conf;
listen 8443 default proxy_protocol;
ssl on;
ssl_certificate ./stack/ssl.cert;
ssl_certificate_key ./stack/ssl.key;
ssl_session_timeout 5m;
ssl_protocols SSLv3 TLSv1;
ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP;
ssl_prefer_server_ciphers on;
}
}

8
conf/proxy-protocol.conf Normal file
View file

@ -0,0 +1,8 @@
# vim: ft=nginx
set_real_ip_from 0.0.0.0/0;
real_ip_header proxy_protocol;
log_format elb_pp '$proxy_protocol_addr - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent"';
access_log /var/log/nginx/nginx.access.log elb_pp;

87
conf/proxy-server-base.conf Normal file
View file

@ -0,0 +1,87 @@
# vim: ft=nginx
client_body_temp_path /var/log/nginx/client_body 1 2;
server_name _;
keepalive_timeout 5;
if ($args ~ "_escaped_fragment_") {
rewrite ^ /snapshot$uri;
}
proxy_set_header X-Forwarded-For $proxy_protocol_addr;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
proxy_redirect off;
proxy_set_header Transfer-Encoding $http_transfer_encoding;
location / {
proxy_pass http://web_app_server;
limit_req zone=webapp burst=25 nodelay;
}
location /realtime {
proxy_pass http://web_app_server;
proxy_buffering off;
proxy_request_buffering off;
}
location /v1/repositories/ {
proxy_buffering off;
proxy_request_buffering off;
proxy_pass http://registry_app_server;
proxy_temp_path /var/log/nginx/proxy_temp 1 2;
client_max_body_size 20G;
limit_req zone=repositories burst=5 nodelay;
}
location /v1/ {
proxy_buffering off;
proxy_request_buffering off;
proxy_pass http://registry_app_server;
proxy_temp_path /var/log/nginx/proxy_temp 1 2;
client_max_body_size 20G;
}
location /c1/ {
proxy_buffering off;
proxy_request_buffering off;
proxy_pass http://verbs_app_server;
proxy_temp_path /var/log/nginx/proxy_temp 1 2;
limit_req zone=api burst=5 nodelay;
}
location /static/ {
# checks for static file, if not found proxy to app
alias /static/;
}
location /v1/_ping {
add_header Content-Type text/plain;
add_header X-Docker-Registry-Version 0.6.0;
add_header X-Docker-Registry-Standalone 0;
return 200 'true';
}
location ~ ^/b1/controller(/?)(.*) {
proxy_pass http://build_manager_controller_server/$2;
}
location ~ ^/b1/socket(/?)(.*) {
proxy_pass http://build_manager_websocket_server/$2;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}

7
conf/rate-limiting.conf Normal file
View file

@ -0,0 +1,7 @@
# vim: ft=nginx
limit_req_zone $proxy_protocol_addr zone=webapp:10m rate=25r/s;
limit_req_zone $proxy_protocol_addr zone=repositories:10m rate=1r/s;
limit_req_zone $proxy_protocol_addr zone=api:10m rate=1r/s;
limit_req_status 429;
limit_req_log_level warn;

12
conf/root-base.conf
View file

@ -1,7 +1,17 @@
# vim: ft=nginx
pid /tmp/nginx.pid;
error_log /var/log/nginx/nginx.error.log;
worker_processes 2;
worker_priority -10;
worker_rlimit_nofile 10240;
user root nogroup;
daemon off;
events {
worker_connections 1024;
worker_connections 10240;
accept_mutex off;
}

5
conf/server-base.conf
View file

@ -1,3 +1,5 @@
# vim: ft=nginx
client_body_temp_path /var/log/nginx/client_body 1 2;
server_name _;
@ -33,7 +35,6 @@ location /v1/ {
proxy_request_buffering off;
proxy_pass http://registry_app_server;
proxy_read_timeout 2000;
proxy_temp_path /var/log/nginx/proxy_temp 1 2;
client_max_body_size 20G;
@ -45,7 +46,6 @@ location /c1/ {
proxy_request_buffering off;
proxy_pass http://verbs_app_server;
proxy_read_timeout 2000;
proxy_temp_path /var/log/nginx/proxy_temp 1 2;
}
@ -63,7 +63,6 @@ location /v1/_ping {
location ~ ^/b1/controller(/?)(.*) {
proxy_pass http://build_manager_controller_server/$2;
proxy_read_timeout 2000;
}
location ~ ^/b1/socket(/?)(.*) {

23
config.py
View file

@ -36,7 +36,6 @@ def getFrontendVisibleConfig(config_dict):
class DefaultConfig(object):
# Flask config
SECRET_KEY = 'a36c9d7d-25a9-4d3f-a586-3d2f8dc40a83'
JSONIFY_PRETTYPRINT_REGULAR = False
SESSION_COOKIE_SECURE = False
@ -48,8 +47,9 @@ class DefaultConfig(object):
AVATAR_KIND = 'local'
REGISTRY_TITLE = 'Quay.io'
REGISTRY_TITLE_SHORT = 'Quay.io'
REGISTRY_TITLE = 'CoreOS Enterprise Registry'
REGISTRY_TITLE_SHORT = 'Enterprise Registry'
CONTACT_INFO = [
'mailto:support@quay.io',
'irc://chat.freenode.net:6665/quayio',
@ -132,6 +132,9 @@ class DefaultConfig(object):
# Super user config. Note: This MUST BE an empty list for the default config.
SUPER_USERS = []
# Feature Flag: Whether super users are supported.
FEATURE_SUPER_USERS = True
# Feature Flag: Whether billing is required.
FEATURE_BILLING = False
@ -147,9 +150,6 @@ class DefaultConfig(object):
# Feature flag, whether to enable olark chat
FEATURE_OLARK_CHAT = False
# Feature Flag: Whether super users are supported.
FEATURE_SUPER_USERS = False
# Feature Flag: Whether to support GitHub build triggers.
FEATURE_GITHUB_BUILD = False
@ -187,3 +187,14 @@ class DefaultConfig(object):
# For enterprise:
MAXIMUM_REPOSITORY_USAGE = 20
# System logs.
SYSTEM_LOGS_PATH = "/var/log/"
SYSTEM_SERVICE_LOGS_PATH = "/var/log/%s/current"
SYSTEM_SERVICES_PATH = "conf/init/"
# Services that should not be shown in the logs view.
SYSTEM_SERVICE_BLACKLIST = []
# Temporary tag expiration in seconds, this may actually be longer based on GC policy
PUSH_TEMP_TAG_EXPIRATION_SEC = 60 * 60

108
data/database.py
View file

@ -1,14 +1,15 @@
import string
import logging
import uuid
import time
from random import SystemRandom
from datetime import datetime
from peewee import (Proxy, MySQLDatabase, SqliteDatabase, PostgresqlDatabase, fn, CharField,
BooleanField, IntegerField, DateTimeField, ForeignKeyField, TextField,
BigIntegerField)
from peewee import *
from data.read_slave import ReadSlaveModel
from sqlalchemy.engine.url import make_url
from data.read_slave import ReadSlaveModel
from util.names import urn_generator
@ -31,6 +32,16 @@ SCHEME_RANDOM_FUNCTION = {
'postgresql+psycopg2': fn.Random,
}
def real_for_update(query):
return query.for_update()
def null_for_update(query):
return query
SCHEME_SPECIALIZED_FOR_UPDATE = {
'sqlite': null_for_update,
}
class CallableProxy(Proxy):
def __call__(self, *args, **kwargs):
if self.obj is None:
@ -70,6 +81,15 @@ class UseThenDisconnect(object):
db = Proxy()
read_slave = Proxy()
db_random_func = CallableProxy()
db_for_update = CallableProxy()
def validate_database_url(url, connect_timeout=5):
driver = _db_from_url(url, {
'connect_timeout': connect_timeout
})
driver.connect()
driver.close()
def _db_from_url(url, db_kwargs):
@ -84,6 +104,10 @@ def _db_from_url(url, db_kwargs):
if parsed_url.password:
db_kwargs['password'] = parsed_url.password
# Note: sqlite does not support connect_timeout.
if parsed_url.drivername == 'sqlite' and 'connect_timeout' in db_kwargs:
del db_kwargs['connect_timeout']
return SCHEME_DRIVERS[parsed_url.drivername](parsed_url.database, **db_kwargs)
@ -95,6 +119,8 @@ def configure(config_object):
parsed_write_uri = make_url(write_db_uri)
db_random_func.initialize(SCHEME_RANDOM_FUNCTION[parsed_write_uri.drivername])
db_for_update.initialize(SCHEME_SPECIALIZED_FOR_UPDATE.get(parsed_write_uri.drivername,
real_for_update))
read_slave_uri = config_object.get('DB_READ_SLAVE_URI', None)
if read_slave_uri is not None:
@ -113,6 +139,9 @@ def uuid_generator():
return str(uuid.uuid4())
_get_epoch_timestamp = lambda: int(time.time())
def close_db_filter(_):
if not db.is_closed():
logger.debug('Disconnecting from database.')
@ -124,8 +153,9 @@ def close_db_filter(_):
class QuayUserField(ForeignKeyField):
def __init__(self, allows_robots=False, *args, **kwargs):
def __init__(self, allows_robots=False, robot_null_delete=False, *args, **kwargs):
self.allows_robots = allows_robots
self.robot_null_delete = robot_null_delete
if not 'rel_model' in kwargs:
kwargs['rel_model'] = User
@ -151,6 +181,7 @@ class User(BaseModel):
invoice_email = BooleanField(default=False)
invalid_login_attempts = IntegerField(default=0)
last_invalid_login = DateTimeField(default=datetime.utcnow)
removed_tag_expiration_s = IntegerField(default=1209600) # Two weeks
def delete_instance(self, recursive=False, delete_nullable=False):
# If we are deleting a robot account, only execute the subset of queries necessary.
@ -159,7 +190,11 @@ class User(BaseModel):
for query, fk in self.dependencies(search_nullable=True):
if isinstance(fk, QuayUserField) and fk.allows_robots:
model = fk.model_class
model.delete().where(query).execute()
if fk.robot_null_delete:
model.update(**{fk.name: None}).where(query).execute()
else:
model.delete().where(query).execute()
# Delete the instance itself.
super(User, self).delete_instance(recursive=False, delete_nullable=False)
@ -319,6 +354,10 @@ class PermissionPrototype(BaseModel):
)
class AccessTokenKind(BaseModel):
name = CharField(unique=True, index=True)
class AccessToken(BaseModel):
friendly_name = CharField(null=True)
code = CharField(default=random_string_generator(length=64), unique=True,
@ -327,6 +366,7 @@ class AccessToken(BaseModel):
created = DateTimeField(default=datetime.now)
role = ForeignKeyField(Role)
temporary = BooleanField(default=True)
kind = ForeignKeyField(AccessTokenKind, null=True)
class BuildTriggerService(BaseModel):
@ -368,6 +408,24 @@ class ImageStorageTransformation(BaseModel):
name = CharField(index=True, unique=True)
class ImageStorageSignatureKind(BaseModel):
name = CharField(index=True, unique=True)
class ImageStorageSignature(BaseModel):
storage = ForeignKeyField(ImageStorage, index=True)
kind = ForeignKeyField(ImageStorageSignatureKind)
signature = TextField(null=True)
uploading = BooleanField(default=True, null=True)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('kind', 'storage'), True),
)
class DerivedImageStorage(BaseModel):
source = ForeignKeyField(ImageStorage, null=True, related_name='source')
derivative = ForeignKeyField(ImageStorage, related_name='derivative')
@ -424,12 +482,15 @@ class RepositoryTag(BaseModel):
name = CharField()
image = ForeignKeyField(Image)
repository = ForeignKeyField(Repository)
lifetime_start_ts = IntegerField(default=_get_epoch_timestamp)
lifetime_end_ts = IntegerField(null=True, index=True)
hidden = BooleanField(default=False)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('repository', 'name'), True),
(('repository', 'name'), False),
)
@ -441,23 +502,10 @@ class BUILD_PHASE(object):
PULLING = 'pulling'
BUILDING = 'building'
PUSHING = 'pushing'
WAITING = 'waiting'
COMPLETE = 'complete'
class RepositoryBuild(BaseModel):
uuid = CharField(default=uuid_generator, index=True)
repository = ForeignKeyField(Repository, index=True)
access_token = ForeignKeyField(AccessToken)
resource_key = CharField(index=True)
job_config = TextField()
phase = CharField(default='waiting')
started = DateTimeField(default=datetime.now)
display_name = CharField()
trigger = ForeignKeyField(RepositoryBuildTrigger, null=True, index=True)
pull_robot = QuayUserField(null=True, related_name='buildpullrobot')
logs_archived = BooleanField(default=False)
class QueueItem(BaseModel):
queue_name = CharField(index=True, max_length=1024)
body = TextField()
@ -467,6 +515,21 @@ class QueueItem(BaseModel):
retries_remaining = IntegerField(default=5)
class RepositoryBuild(BaseModel):
uuid = CharField(default=uuid_generator, index=True)
repository = ForeignKeyField(Repository, index=True)
access_token = ForeignKeyField(AccessToken)
resource_key = CharField(index=True)
job_config = TextField()
phase = CharField(default=BUILD_PHASE.WAITING)
started = DateTimeField(default=datetime.now)
display_name = CharField()
trigger = ForeignKeyField(RepositoryBuildTrigger, null=True, index=True)
pull_robot = QuayUserField(null=True, related_name='buildpullrobot')
logs_archived = BooleanField(default=False)
queue_item = ForeignKeyField(QueueItem, null=True, index=True)
class LogEntryKind(BaseModel):
name = CharField(index=True, unique=True)
@ -475,7 +538,7 @@ class LogEntry(BaseModel):
kind = ForeignKeyField(LogEntryKind, index=True)
account = QuayUserField(index=True, related_name='account')
performer = QuayUserField(allows_robots=True, index=True, null=True,
related_name='performer')
related_name='performer', robot_null_delete=True)
repository = ForeignKeyField(Repository, index=True, null=True)
datetime = DateTimeField(default=datetime.now, index=True)
ip = CharField(null=True)
@ -566,4 +629,5 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission,
Notification, ImageStorageLocation, ImageStoragePlacement,
ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification,
RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage,
TeamMemberInvite, Star]
TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind,
AccessTokenKind, Star]

3
data/migrations/env.py
View file

@ -18,7 +18,8 @@ config.set_main_option('sqlalchemy.url', unquote(app.config['DB_URI']))
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
if config.config_file_name:
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support

57
data/migrations/migration.sh
View file

@ -2,13 +2,14 @@ set -e
DOCKER_IP=`echo $DOCKER_HOST | sed 's/tcp:\/\///' | sed 's/:.*//'`
MYSQL_CONFIG_OVERRIDE="{\"DB_URI\":\"mysql+pymysql://root:password@$DOCKER_IP/genschema\"}"
PERCONA_CONFIG_OVERRIDE="{\"DB_URI\":\"mysql+pymysql://root@$DOCKER_IP/genschema\"}"
PGSQL_CONFIG_OVERRIDE="{\"DB_URI\":\"postgresql://postgres@$DOCKER_IP/genschema\"}"
up_mysql() {
# Run a SQL database on port 3306 inside of Docker.
docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mysql
# Sleep for 5s to get MySQL get started.
# Sleep for 10s to get MySQL get started.
echo 'Sleeping for 10...'
sleep 10
@ -21,6 +22,40 @@ down_mysql() {
docker rm mysql
}
up_mariadb() {
# Run a SQL database on port 3306 inside of Docker.
docker run --name mariadb -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mariadb
# Sleep for 10s to get MySQL get started.
echo 'Sleeping for 10...'
sleep 10
# Add the database to mysql.
docker run --rm --link mariadb:mariadb mariadb sh -c 'echo "create database genschema" | mysql -h"$MARIADB_PORT_3306_TCP_ADDR" -P"$MARIADB_PORT_3306_TCP_PORT" -uroot -ppassword'
}
down_mariadb() {
docker kill mariadb
docker rm mariadb
}
up_percona() {
# Run a SQL database on port 3306 inside of Docker.
docker run --name percona -p 3306:3306 -d dockerfile/percona
# Sleep for 10s
echo 'Sleeping for 10...'
sleep 10
# Add the daabase to mysql.
docker run --rm --link percona:percona dockerfile/percona sh -c 'echo "create database genschema" | mysql -h $PERCONA_PORT_3306_TCP_ADDR'
}
down_percona() {
docker kill percona
docker rm percona
}
up_postgres() {
# Run a SQL database on port 5432 inside of Docker.
docker run --name postgres -p 5432:5432 -d postgres
@ -73,6 +108,26 @@ test_migrate $MYSQL_CONFIG_OVERRIDE
set -e
down_mysql
# Test via MariaDB.
echo '> Starting MariaDB'
up_mariadb
echo '> Testing Migration (mariadb)'
set +e
test_migrate $MYSQL_CONFIG_OVERRIDE
set -e
down_mariadb
# Test via Percona.
echo '> Starting Percona'
up_percona
echo '> Testing Migration (percona)'
set +e
test_migrate $PERCONA_CONFIG_OVERRIDE
set -e
down_percona
# Test via Postgres.
echo '> Starting Postgres'
up_postgres

30
data/migrations/versions/14fe12ade3df_add_build_queue_item_reference_to_the_.py Normal file
View file

@ -0,0 +1,30 @@
"""Add build queue item reference to the repositorybuild table
Revision ID: 14fe12ade3df
Revises: 5ad999136045
Create Date: 2015-02-12 16:11:57.814645
"""
# revision identifiers, used by Alembic.
revision = '14fe12ade3df'
down_revision = '5ad999136045'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
def upgrade(tables):
### commands auto generated by Alembic - please adjust! ###
op.add_column('repositorybuild', sa.Column('queue_item_id', sa.Integer(), nullable=True))
op.create_index('repositorybuild_queue_item_id', 'repositorybuild', ['queue_item_id'], unique=False)
op.create_foreign_key(op.f('fk_repositorybuild_queue_item_id_queueitem'), 'repositorybuild', 'queueitem', ['queue_item_id'], ['id'])
### end Alembic commands ###
def downgrade(tables):
### commands auto generated by Alembic - please adjust! ###
op.drop_constraint(op.f('fk_repositorybuild_queue_item_id_queueitem'), 'repositorybuild', type_='foreignkey')
op.drop_index('repositorybuild_queue_item_id', table_name='repositorybuild')
op.drop_column('repositorybuild', 'queue_item_id')
### end Alembic commands ###

37
data/migrations/versions/1d2d86d09fcd_actually_remove_the_column.py Normal file
View file

@ -0,0 +1,37 @@
"""Actually remove the column access_token_id
Revision ID: 1d2d86d09fcd
Revises: 14fe12ade3df
Create Date: 2015-02-12 16:27:30.260797
"""
# revision identifiers, used by Alembic.
revision = '1d2d86d09fcd'
down_revision = '14fe12ade3df'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
from sqlalchemy.exc import InternalError
def upgrade(tables):
### commands auto generated by Alembic - please adjust! ###
try:
op.drop_constraint(u'fk_logentry_access_token_id_accesstoken', 'logentry', type_='foreignkey')
op.drop_index('logentry_access_token_id', table_name='logentry')
op.drop_column('logentry', 'access_token_id')
except InternalError:
pass
### end Alembic commands ###
def downgrade(tables):
### commands auto generated by Alembic - please adjust! ###
try:
op.add_column('logentry', sa.Column('access_token_id', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True))
op.create_foreign_key(u'fk_logentry_access_token_id_accesstoken', 'logentry', 'accesstoken', ['access_token_id'], ['id'])
op.create_index('logentry_access_token_id', 'logentry', ['access_token_id'], unique=False)
except InternalError:
pass
### end Alembic commands ###

25
data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py Normal file
View file

@ -0,0 +1,25 @@
"""mysql max index lengths
Revision ID: 228d1af6af1c
Revises: 5b84373e5db
Create Date: 2015-01-06 14:35:24.651424
"""
# revision identifiers, used by Alembic.
revision = '228d1af6af1c'
down_revision = '5b84373e5db'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
def upgrade(tables):
op.drop_index('queueitem_queue_name', table_name='queueitem')
op.create_index('queueitem_queue_name', 'queueitem', ['queue_name'], unique=False, mysql_length=767)
op.drop_index('image_ancestors', table_name='image')
op.create_index('image_ancestors', 'image', ['ancestors'], unique=False, mysql_length=767)
def downgrade(tables):
pass

44
data/migrations/versions/3e2d38b52a75_add_access_token_kinds_type.py Normal file
View file

@ -0,0 +1,44 @@
"""Add access token kinds type
Revision ID: 3e2d38b52a75
Revises: 1d2d86d09fcd
Create Date: 2015-02-17 12:03:26.422485
"""
# revision identifiers, used by Alembic.
revision = '3e2d38b52a75'
down_revision = '1d2d86d09fcd'
from alembic import op
import sqlalchemy as sa
def upgrade(tables):
### commands auto generated by Alembic - please adjust! ###
op.create_table('accesstokenkind',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_accesstokenkind'))
)
op.create_index('accesstokenkind_name', 'accesstokenkind', ['name'], unique=True)
op.add_column(u'accesstoken', sa.Column('kind_id', sa.Integer(), nullable=True))
op.create_index('accesstoken_kind_id', 'accesstoken', ['kind_id'], unique=False)
op.create_foreign_key(op.f('fk_accesstoken_kind_id_accesstokenkind'), 'accesstoken', 'accesstokenkind', ['kind_id'], ['id'])
### end Alembic commands ###
op.bulk_insert(tables.accesstokenkind,
[
{'id': 1, 'name':'build-worker'},
{'id': 2, 'name':'pushpull-token'},
])
def downgrade(tables):
### commands auto generated by Alembic - please adjust! ###
op.drop_constraint(op.f('fk_accesstoken_kind_id_accesstokenkind'), 'accesstoken', type_='foreignkey')
op.drop_index('accesstoken_kind_id', table_name='accesstoken')
op.drop_column(u'accesstoken', 'kind_id')
op.drop_index('accesstokenkind_name', table_name='accesstokenkind')
op.drop_table('accesstokenkind')
### end Alembic commands ###

26
data/migrations/versions/4ef04c61fcf9_allow_tags_to_be_marked_as_hidden.py Normal file
View file

@ -0,0 +1,26 @@
"""Allow tags to be marked as hidden.
Revision ID: 4ef04c61fcf9
Revises: 509d2857566f
Create Date: 2015-02-18 16:34:16.586129
"""
# revision identifiers, used by Alembic.
revision = '4ef04c61fcf9'
down_revision = '509d2857566f'
from alembic import op
import sqlalchemy as sa
def upgrade(tables):
### commands auto generated by Alembic - please adjust! ###
op.add_column('repositorytag', sa.Column('hidden', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()))
### end Alembic commands ###
def downgrade(tables):
### commands auto generated by Alembic - please adjust! ###
op.drop_column('repositorytag', 'hidden')
### end Alembic commands ###

36
data/migrations/versions/509d2857566f_track_the_lifetime_start_and_end_for_.py Normal file
View file

@ -0,0 +1,36 @@
"""Track the lifetime start and end for tags to allow the state of a repository to be rewound.
Revision ID: 509d2857566f
Revises: 3e2d38b52a75
Create Date: 2015-02-13 14:35:38.939049
"""
# revision identifiers, used by Alembic.
revision = '509d2857566f'
down_revision = '3e2d38b52a75'
from alembic import op
import sqlalchemy as sa
def upgrade(tables):
### commands auto generated by Alembic - please adjust! ###
op.add_column('repositorytag', sa.Column('lifetime_end_ts', sa.Integer(), nullable=True))
op.add_column('repositorytag', sa.Column('lifetime_start_ts', sa.Integer(), nullable=False, server_default="0"))
op.create_index('repositorytag_lifetime_end_ts', 'repositorytag', ['lifetime_end_ts'], unique=False)
op.drop_index('repositorytag_repository_id_name', table_name='repositorytag')
op.create_index('repositorytag_repository_id_name', 'repositorytag', ['repository_id', 'name'], unique=False)
op.add_column('user', sa.Column('removed_tag_expiration_s', sa.Integer(), nullable=False, server_default="1209600"))
### end Alembic commands ###
def downgrade(tables):
### commands auto generated by Alembic - please adjust! ###
op.drop_column('user', 'removed_tag_expiration_s')
op.drop_index('repositorytag_repository_id_name', table_name='repositorytag')
op.create_index('repositorytag_repository_id_name', 'repositorytag', ['repository_id', 'name'], unique=True)
op.drop_index('repositorytag_lifetime_end_ts', table_name='repositorytag')
op.drop_column('repositorytag', 'lifetime_start_ts')
op.drop_column('repositorytag', 'lifetime_end_ts')
### end Alembic commands ###

4
data/migrations/versions/5a07499ce53f_set_up_initial_database.py
View file

@ -53,7 +53,7 @@ def upgrade(tables):
op.create_index('queueitem_available', 'queueitem', ['available'], unique=False)
op.create_index('queueitem_available_after', 'queueitem', ['available_after'], unique=False)
op.create_index('queueitem_processing_expires', 'queueitem', ['processing_expires'], unique=False)
op.create_index('queueitem_queue_name', 'queueitem', ['queue_name'], unique=False)
op.create_index('queueitem_queue_name', 'queueitem', ['queue_name'], unique=False, mysql_length=767)
op.create_table('role',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
@ -376,7 +376,7 @@ def upgrade(tables):
sa.ForeignKeyConstraint(['storage_id'], ['imagestorage.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('image_ancestors', 'image', ['ancestors'], unique=False)
op.create_index('image_ancestors', 'image', ['ancestors'], unique=False, mysql_length=767)
op.create_index('image_repository_id', 'image', ['repository_id'], unique=False)
op.create_index('image_repository_id_docker_image_id', 'image', ['repository_id', 'docker_image_id'], unique=True)
op.create_index('image_storage_id', 'image', ['storage_id'], unique=False)

55
data/migrations/versions/5ad999136045_add_signature_storage.py Normal file
View file

@ -0,0 +1,55 @@
"""Add signature storage
Revision ID: 5ad999136045
Revises: 228d1af6af1c
Create Date: 2015-02-05 15:01:54.989573
"""
# revision identifiers, used by Alembic.
revision = '5ad999136045'
down_revision = '228d1af6af1c'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
def upgrade(tables):
### commands auto generated by Alembic - please adjust! ###
op.create_table('imagestoragesignaturekind',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_imagestoragesignaturekind'))
)
op.create_index('imagestoragesignaturekind_name', 'imagestoragesignaturekind', ['name'], unique=True)
op.create_table('imagestoragesignature',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('storage_id', sa.Integer(), nullable=False),
sa.Column('kind_id', sa.Integer(), nullable=False),
sa.Column('signature', sa.Text(), nullable=True),
sa.Column('uploading', sa.Boolean(), nullable=True),
sa.ForeignKeyConstraint(['kind_id'], ['imagestoragesignaturekind.id'], name=op.f('fk_imagestoragesignature_kind_id_imagestoragesignaturekind')),
sa.ForeignKeyConstraint(['storage_id'], ['imagestorage.id'], name=op.f('fk_imagestoragesignature_storage_id_imagestorage')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_imagestoragesignature'))
)
op.create_index('imagestoragesignature_kind_id', 'imagestoragesignature', ['kind_id'], unique=False)
op.create_index('imagestoragesignature_kind_id_storage_id', 'imagestoragesignature', ['kind_id', 'storage_id'], unique=True)
op.create_index('imagestoragesignature_storage_id', 'imagestoragesignature', ['storage_id'], unique=False)
### end Alembic commands ###
op.bulk_insert(tables.imagestoragetransformation,
[
{'id': 2, 'name':'aci'},
])
op.bulk_insert(tables.imagestoragesignaturekind,
[
{'id': 1, 'name':'gpg2'},
])
def downgrade(tables):
### commands auto generated by Alembic - please adjust! ###
op.drop_table('imagestoragesignature')
op.drop_table('imagestoragesignaturekind')
### end Alembic commands ###

24
data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py Normal file
View file

@ -0,0 +1,24 @@
"""Convert slack webhook data
Revision ID: 5b84373e5db
Revises: 1c5b738283a5
Create Date: 2014-12-16 12:02:55.167744
"""
# revision identifiers, used by Alembic.
revision = '5b84373e5db'
down_revision = '1c5b738283a5'
from alembic import op
import sqlalchemy as sa
from util.migrateslackwebhook import run_slackwebhook_migration
def upgrade(tables):
run_slackwebhook_migration()
def downgrade(tables):
pass

311
data/model/legacy.py
View file

@ -2,8 +2,10 @@ import bcrypt
import logging
import dateutil.parser
import json
import time
from datetime import datetime, timedelta, date
from uuid import uuid4
from data.database import (User, Repository, Image, AccessToken, Role, RepositoryPermission,
Visibility, RepositoryTag, EmailConfirmation, FederatedLogin,
@ -14,7 +16,9 @@ from data.database import (User, Repository, Image, AccessToken, Role, Repositor
ExternalNotificationEvent, ExternalNotificationMethod,
RepositoryNotification, RepositoryAuthorizedEmail, TeamMemberInvite,
DerivedImageStorage, ImageStorageTransformation, random_string_generator,
db, BUILD_PHASE, QuayUserField, Star)
db, BUILD_PHASE, QuayUserField, ImageStorageSignature, QueueItem,
ImageStorageSignatureKind, validate_database_url, db_for_update,
AccessTokenKind, Star)
from peewee import JOIN_LEFT_OUTER, fn
from util.validation import (validate_username, validate_email, validate_password,
INVALID_PASSWORD_MESSAGE)
@ -105,12 +109,15 @@ class TooManyLoginAttemptsException(Exception):
self.retry_after = retry_after
def _get_repository(namespace_name, repository_name):
return (Repository
.select(Repository, Namespace)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace_name, Repository.name == repository_name)
.get())
def _get_repository(namespace_name, repository_name, for_update=False):
query = (Repository
.select(Repository, Namespace)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace_name, Repository.name == repository_name))
if for_update:
query = db_for_update(query)
return query.get()
def hash_password(password, salt=None):
@ -164,8 +171,7 @@ def _create_user(username, email):
pass
try:
new_user = User.create(username=username, email=email)
return new_user
return User.create(username=username, email=email)
except Exception as ex:
raise DataModelException(ex.message)
@ -295,6 +301,9 @@ def delete_robot(robot_username):
def _list_entity_robots(entity_name):
""" Return the list of robots for the specified entity. This MUST return a query, not a
materialized list so that callers can use db_for_update.
"""
return (User
.select()
.join(FederatedLogin)
@ -901,14 +910,17 @@ def change_password(user, new_password):
delete_notifications_by_kind(user, 'password_required')
def change_username(user, new_username):
def change_username(user_id, new_username):
(username_valid, username_issue) = validate_username(new_username)
if not username_valid:
raise InvalidUsernameException('Invalid username %s: %s' % (new_username, username_issue))
with config.app_config['DB_TRANSACTION_FACTORY'](db):
# Reload the user for update
user = db_for_update(User.select().where(User.id == user_id)).get()
# Rename the robots
for robot in _list_entity_robots(user.username):
for robot in db_for_update(_list_entity_robots(user.username)):
_, robot_shortname = parse_robot_username(robot.username)
new_robot_name = format_robot_username(new_username, robot_shortname)
robot.username = new_robot_name
@ -924,6 +936,11 @@ def change_invoice_email(user, invoice_email):
user.save()
def change_user_tag_expiration(user, tag_expiration_s):
user.removed_tag_expiration_s = tag_expiration_s
user.save()
def update_email(user, new_email, auto_verify=False):
user.email = new_email
user.verified = auto_verify
@ -1087,6 +1104,26 @@ def get_repository(namespace_name, repository_name):
return None
def get_image(repo, dockerfile_id):
try:
return Image.get(Image.docker_image_id == dockerfile_id, Image.repository == repo)
except Image.DoesNotExist:
return None
def find_child_image(repo, parent_image, command):
try:
return (Image.select()
.join(ImageStorage)
.switch(Image)
.where(Image.ancestors % '%/' + parent_image.id + '/%',
ImageStorage.command == command)
.order_by(ImageStorage.created.desc())
.get())
except Image.DoesNotExist:
return None
def get_repo_image(namespace_name, repository_name, docker_image_id):
def limit_to_image_id(query):
return query.where(Image.docker_image_id == docker_image_id).limit(1)
@ -1249,9 +1286,9 @@ def _find_or_link_image(existing_image, repository, username, translations, pref
storage.locations = {placement.location.name
for placement in storage.imagestorageplacement_set}
new_image = Image.create(docker_image_id=existing_image.docker_image_id,
repository=repository, storage=storage,
ancestors=new_image_ancestry)
new_image = Image.create(docker_image_id=existing_image.docker_image_id,
repository=repository, storage=storage,
ancestors=new_image_ancestry)
logger.debug('Storing translation %s -> %s', existing_image.id, new_image.id)
translations[existing_image.id] = new_image.id
@ -1315,7 +1352,28 @@ def find_create_or_link_image(docker_image_id, repository, username, translation
ancestors='/')
def find_or_create_derived_storage(source, transformation_name, preferred_location):
def find_or_create_storage_signature(storage, signature_kind):
found = lookup_storage_signature(storage, signature_kind)
if found is None:
kind = ImageStorageSignatureKind.get(name=signature_kind)
found = ImageStorageSignature.create(storage=storage, kind=kind)
return found
def lookup_storage_signature(storage, signature_kind):
kind = ImageStorageSignatureKind.get(name=signature_kind)
try:
return (ImageStorageSignature
.select()
.where(ImageStorageSignature.storage == storage,
ImageStorageSignature.kind == kind)
.get())
except ImageStorageSignature.DoesNotExist:
return None
def find_derived_storage(source, transformation_name):
try:
found = (ImageStorage
.select(ImageStorage, DerivedImageStorage)
@ -1328,11 +1386,19 @@ def find_or_create_derived_storage(source, transformation_name, preferred_locati
found.locations = {placement.location.name for placement in found.imagestorageplacement_set}
return found
except ImageStorage.DoesNotExist:
logger.debug('Creating storage dervied from source: %s', source.uuid)
trans = ImageStorageTransformation.get(name=transformation_name)
new_storage = _create_storage(preferred_location)
DerivedImageStorage.create(source=source, derivative=new_storage, transformation=trans)
return new_storage
return None
def find_or_create_derived_storage(source, transformation_name, preferred_location):
existing = find_derived_storage(source, transformation_name)
if existing is not None:
return existing
logger.debug('Creating storage dervied from source: %s', source.uuid)
trans = ImageStorageTransformation.get(name=transformation_name)
new_storage = _create_storage(preferred_location)
DerivedImageStorage.create(source=source, derivative=new_storage, transformation=trans)
return new_storage
def delete_derived_storage_by_uuid(storage_uuid):
@ -1401,7 +1467,7 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
Image.docker_image_id == docker_image_id))
try:
fetched = query.get()
fetched = db_for_update(query).get()
except Image.DoesNotExist:
raise DataModelException('No image with specified id and repository')
@ -1489,19 +1555,48 @@ def get_repository_images(namespace_name, repository_name):
return _get_repository_images_base(namespace_name, repository_name, lambda q: q)
def list_repository_tags(namespace_name, repository_name):
return (RepositoryTag
.select(RepositoryTag, Image)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(RepositoryTag)
.join(Image)
.where(Repository.name == repository_name, Namespace.username == namespace_name))
def _tag_alive(query):
return query.where((RepositoryTag.lifetime_end_ts >> None) |
(RepositoryTag.lifetime_end_ts > int(time.time())))
def list_repository_tags(namespace_name, repository_name, include_hidden=False):
query = _tag_alive(RepositoryTag
.select(RepositoryTag, Image)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(RepositoryTag)
.join(Image)
.where(Repository.name == repository_name,
Namespace.username == namespace_name))
if not include_hidden:
query = query.where(RepositoryTag.hidden == False)
return query
def _garbage_collect_tags(namespace_name, repository_name):
to_delete = (RepositoryTag
.select(RepositoryTag.id)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.name == repository_name, Namespace.username == namespace_name,
~(RepositoryTag.lifetime_end_ts >> None),
(RepositoryTag.lifetime_end_ts + Namespace.removed_tag_expiration_s) <=
int(time.time())))
(RepositoryTag
.delete()
.where(RepositoryTag.id << to_delete)
.execute())
def garbage_collect_repository(namespace_name, repository_name):
storage_id_whitelist = {}
_garbage_collect_tags(namespace_name, repository_name)
with config.app_config['DB_TRANSACTION_FACTORY'](db):
# TODO (jake): We could probably select this and all the images in a single query using
# a different kind of join.
@ -1535,12 +1630,10 @@ def garbage_collect_repository(namespace_name, repository_name):
if len(to_remove) > 0:
logger.info('Garbage collecting storage for images: %s', to_remove)
garbage_collect_storage(storage_id_whitelist)
return len(to_remove)
_garbage_collect_storage(storage_id_whitelist)
def garbage_collect_storage(storage_id_whitelist):
def _garbage_collect_storage(storage_id_whitelist):
if len(storage_id_whitelist) == 0:
return
@ -1632,10 +1725,10 @@ def garbage_collect_storage(storage_id_whitelist):
def get_tag_image(namespace_name, repository_name, tag_name):
def limit_to_tag(query):
return (query
.switch(Image)
.join(RepositoryTag)
.where(RepositoryTag.name == tag_name))
return _tag_alive(query
.switch(Image)
.join(RepositoryTag)
.where(RepositoryTag.name == tag_name))
images = _get_repository_images_base(namespace_name, repository_name, limit_to_tag)
if not images:
@ -1643,7 +1736,6 @@ def get_tag_image(namespace_name, repository_name, tag_name):
else:
return images[0]
def get_image_by_id(namespace_name, repository_name, docker_image_id):
image = get_repo_image_extended(namespace_name, repository_name, docker_image_id)
if not image:
@ -1672,45 +1764,69 @@ def get_parent_images(namespace_name, repository_name, image_obj):
def create_or_update_tag(namespace_name, repository_name, tag_name,
tag_docker_image_id):
try:
repo = _get_repository(namespace_name, repository_name)
except Repository.DoesNotExist:
raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name))
try:
image = Image.get(Image.docker_image_id == tag_docker_image_id, Image.repository == repo)
except Image.DoesNotExist:
raise DataModelException('Invalid image with id: %s' % tag_docker_image_id)
with config.app_config['DB_TRANSACTION_FACTORY'](db):
try:
repo = _get_repository(namespace_name, repository_name)
except Repository.DoesNotExist:
raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name))
try:
tag = RepositoryTag.get(RepositoryTag.repository == repo, RepositoryTag.name == tag_name)
tag.image = image
tag.save()
except RepositoryTag.DoesNotExist:
tag = RepositoryTag.create(repository=repo, image=image, name=tag_name)
try:
image = Image.get(Image.docker_image_id == tag_docker_image_id, Image.repository == repo)
except Image.DoesNotExist:
raise DataModelException('Invalid image with id: %s' % tag_docker_image_id)
return tag
now_ts = int(time.time())
try:
# When we move a tag, we really end the timeline of the old one and create a new one
query = _tag_alive(RepositoryTag
.select()
.where(RepositoryTag.repository == repo, RepositoryTag.name == tag_name))
tag = query.get()
tag.lifetime_end_ts = now_ts
tag.save()
except RepositoryTag.DoesNotExist:
# No tag that needs to be ended
pass
return RepositoryTag.create(repository=repo, image=image, name=tag_name,
lifetime_start_ts=now_ts)
def delete_tag(namespace_name, repository_name, tag_name):
try:
found = (RepositoryTag
.select()
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.name == repository_name, Namespace.username == namespace_name,
RepositoryTag.name == tag_name)
.get())
with config.app_config['DB_TRANSACTION_FACTORY'](db):
try:
query = _tag_alive(RepositoryTag
.select(RepositoryTag, Repository)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.name == repository_name,
Namespace.username == namespace_name,
RepositoryTag.name == tag_name))
found = db_for_update(query).get()
except RepositoryTag.DoesNotExist:
msg = ('Invalid repository tag \'%s\' on repository \'%s/%s\'' %
(tag_name, namespace_name, repository_name))
raise DataModelException(msg)
except RepositoryTag.DoesNotExist:
msg = ('Invalid repository tag \'%s\' on repository \'%s/%s\'' %
(tag_name, namespace_name, repository_name))
raise DataModelException(msg)
found.delete_instance()
found.lifetime_end_ts = int(time.time())
found.save()
def delete_all_repository_tags(namespace_name, repository_name):
def create_temporary_hidden_tag(repo, image, expiration_s):
""" Create a tag with a defined timeline, that will not appear in the UI or CLI. Returns the name
of the temporary tag. """
now_ts = int(time.time())
expire_ts = now_ts + expiration_s
tag_name = str(uuid4())
RepositoryTag.create(repository=repo, image=image, name=tag_name, lifetime_start_ts=now_ts,
lifetime_end_ts=expire_ts, hidden=True)
return tag_name
def purge_all_repository_tags(namespace_name, repository_name):
""" Immediately purge all repository tags without respecting the lifeline procedure """
try:
repo = _get_repository(namespace_name, repository_name)
except Repository.DoesNotExist:
@ -1825,7 +1941,7 @@ def set_team_repo_permission(team_name, namespace_name, repository_name,
def purge_repository(namespace_name, repository_name):
# Delete all tags to allow gc to reclaim storage
delete_all_repository_tags(namespace_name, repository_name)
purge_all_repository_tags(namespace_name, repository_name)
# Gc to remove the images and storage
garbage_collect_repository(namespace_name, repository_name)
@ -1845,10 +1961,14 @@ def get_private_repo_count(username):
.count())
def create_access_token(repository, role):
def create_access_token(repository, role, kind=None, friendly_name=None):
role = Role.get(Role.name == role)
kind_ref = None
if kind is not None:
kind_ref = AccessTokenKind.get(AccessTokenKind.name == kind)
new_token = AccessToken.create(repository=repository, temporary=True,
role=role)
role=role, kind=kind_ref, friendly_name=friendly_name)
return new_token
@ -1967,10 +2087,10 @@ def create_repository_build(repo, access_token, job_config_obj, dockerfile_id,
pull_robot = lookup_robot(pull_robot_name)
return RepositoryBuild.create(repository=repo, access_token=access_token,
job_config=json.dumps(job_config_obj),
display_name=display_name, trigger=trigger,
resource_key=dockerfile_id,
pull_robot=pull_robot)
job_config=json.dumps(job_config_obj),
display_name=display_name, trigger=trigger,
resource_key=dockerfile_id,
pull_robot=pull_robot)
def get_pull_robot_name(trigger):
@ -2255,11 +2375,20 @@ def delete_user(user):
# TODO: also delete any repository data associated
def check_health():
def check_health(app_config):
# Attempt to connect to the database first. If the DB is not responding,
# using the validate_database_url will timeout quickly, as opposed to
# making a normal connect which will just hang (thus breaking the health
# check).
try:
validate_database_url(app_config['DB_URI'], connect_timeout=3)
except Exception:
logger.exception('Could not connect to the database')
return False
# We will connect to the db, check that it contains some log entry kinds
try:
found_count = LogEntryKind.select().count()
return found_count > 0
return bool(list(LogEntryKind.select().limit(1)))
except:
return False
@ -2365,6 +2494,32 @@ def confirm_team_invite(code, user):
found.delete_instance()
return (team, inviter)
def cancel_repository_build(build):
with config.app_config['DB_TRANSACTION_FACTORY'](db):
# Reload the build for update.
try:
build = db_for_update(RepositoryBuild.select().where(RepositoryBuild.id == build.id)).get()
except RepositoryBuild.DoesNotExist:
return False
if build.phase != BUILD_PHASE.WAITING or not build.queue_item:
return False
# Load the build queue item for update.
try:
queue_item = db_for_update(QueueItem.select()
.where(QueueItem.id == build.queue_item.id)).get()
except QueueItem.DoesNotExist:
return False
# Check the queue item.
if not queue_item.available or queue_item.retries_remaining == 0:
return False
# Delete the queue item and build.
queue_item.delete_instance(recursive=True)
build.delete_instance()
return True
def get_repository_usage():
one_month_ago = date.today() - timedelta(weeks=4)

101
data/queue.py
View file

@ -1,11 +1,17 @@
from datetime import datetime, timedelta
from data.database import QueueItem, db
from data.database import QueueItem, db, db_for_update
from util.morecollections import AttrDict
MINIMUM_EXTENSION = timedelta(seconds=20)
class NoopWith:
def __enter__(self):
pass
def __exit__(self, type, value, traceback):
pass
class WorkQueue(object):
def __init__(self, queue_name, transaction_factory,
@ -31,31 +37,50 @@ class WorkQueue(object):
QueueItem.processing_expires > now,
QueueItem.queue_name ** name_match_query))
def _available_jobs(self, now, name_match_query, running_query):
def _available_jobs(self, now, name_match_query):
return (QueueItem
.select()
.where(QueueItem.queue_name ** name_match_query, QueueItem.available_after <= now,
((QueueItem.available == True) | (QueueItem.processing_expires <= now)),
QueueItem.retries_remaining > 0, ~(QueueItem.queue_name << running_query)))
QueueItem.retries_remaining > 0))
def _available_jobs_not_running(self, now, name_match_query, running_query):
return (self
._available_jobs(now, name_match_query)
.where(~(QueueItem.queue_name << running_query)))
def _name_match_query(self):
return '%s%%' % self._canonical_name([self._queue_name] + self._canonical_name_match_list)
def update_metrics(self):
if self._reporter is None:
return
def _item_by_id_for_update(self, queue_id):
return db_for_update(QueueItem.select().where(QueueItem.id == queue_id)).get()
with self._transaction_factory(db):
def get_metrics(self, require_transaction=True):
guard = self._transaction_factory(db) if require_transaction else NoopWith()
with guard:
now = datetime.utcnow()
name_match_query = self._name_match_query()
running_query = self._running_jobs(now, name_match_query)
running_count = running_query.distinct().count()
avialable_query = self._available_jobs(now, name_match_query, running_query)
available_count = avialable_query.select(QueueItem.queue_name).distinct().count()
available_query = self._available_jobs(now, name_match_query)
available_count = available_query.select(QueueItem.queue_name).distinct().count()
self._reporter(self._currently_processing, running_count, running_count + available_count)
available_not_running_query = self._available_jobs_not_running(now, name_match_query,
running_query)
available_not_running_count = (available_not_running_query.select(QueueItem.queue_name)
.distinct().count())
return (running_count, available_not_running_count, available_count)
def update_metrics(self):
if self._reporter is None:
return
(running_count, available_not_running_count, available_count) = self.get_metrics()
self._reporter(self._currently_processing, running_count,
running_count + available_not_running_count)
def put(self, canonical_name_list, message, available_after=0, retries_remaining=5):
"""
@ -73,24 +98,31 @@ class WorkQueue(object):
params['available_after'] = available_date
with self._transaction_factory(db):
QueueItem.create(**params)
return QueueItem.create(**params)
def get(self, processing_time=300):
"""
Get an available item and mark it as unavailable for the default of five
minutes.
minutes. The result of this method must always be composed of simple
python objects which are JSON serializable for network portability reasons.
"""
now = datetime.utcnow()
name_match_query = self._name_match_query()
with self._transaction_factory(db):
running = self._running_jobs(now, name_match_query)
avail = self._available_jobs(now, name_match_query, running)
running = self._running_jobs(now, name_match_query)
avail = self._available_jobs_not_running(now, name_match_query, running)
item = None
try:
db_item = avail.order_by(QueueItem.id).get()
item = None
try:
db_item_candidate = avail.order_by(QueueItem.id).get()
with self._transaction_factory(db):
still_available_query = (db_for_update(self
._available_jobs(now, name_match_query)
.where(QueueItem.id == db_item_candidate.id)))
db_item = still_available_query.get()
db_item.available = False
db_item.processing_expires = now + timedelta(seconds=processing_time)
db_item.retries_remaining -= 1
@ -99,25 +131,26 @@ class WorkQueue(object):
item = AttrDict({
'id': db_item.id,
'body': db_item.body,
'retries_remaining': db_item.retries_remaining
})
self._currently_processing = True
except QueueItem.DoesNotExist:
self._currently_processing = False
except QueueItem.DoesNotExist:
self._currently_processing = False
# Return a view of the queue item rather than an active db object
return item
# Return a view of the queue item rather than an active db object
return item
def complete(self, completed_item):
with self._transaction_factory(db):
completed_item_obj = QueueItem.get(QueueItem.id == completed_item.id)
completed_item_obj = self._item_by_id_for_update(completed_item.id)
completed_item_obj.delete_instance()
self._currently_processing = False
def incomplete(self, incomplete_item, retry_after=300, restore_retry=False):
with self._transaction_factory(db):
retry_date = datetime.utcnow() + timedelta(seconds=retry_after)
incomplete_item_obj = QueueItem.get(QueueItem.id == incomplete_item.id)
incomplete_item_obj = self._item_by_id_for_update(incomplete_item.id)
incomplete_item_obj.available_after = retry_date
incomplete_item_obj.available = True
@ -126,16 +159,14 @@ class WorkQueue(object):
incomplete_item_obj.save()
self._currently_processing = False
return incomplete_item_obj.retries_remaining > 0
@staticmethod
def extend_processing(queue_item, seconds_from_now, retry_count=None,
minimum_extension=MINIMUM_EXTENSION):
new_expiration = datetime.utcnow() + timedelta(seconds=seconds_from_now)
def extend_processing(self, item, seconds_from_now, minimum_extension=MINIMUM_EXTENSION):
with self._transaction_factory(db):
queue_item = self._item_by_id_for_update(item.id)
new_expiration = datetime.utcnow() + timedelta(seconds=seconds_from_now)
# Only actually write the new expiration to the db if it moves the expiration some minimum
if new_expiration - queue_item.processing_expires > minimum_extension:
if retry_count is not None:
queue_item.retries_remaining = retry_count
queue_item.processing_expires = new_expiration
queue_item.save()
# Only actually write the new expiration to the db if it moves the expiration some minimum
if new_expiration - queue_item.processing_expires > minimum_extension:
queue_item.processing_expires = new_expiration
queue_item.save()

20
data/runmigration.py Normal file
View file

@ -0,0 +1,20 @@
import logging
from alembic.config import Config
from alembic.script import ScriptDirectory
from alembic.environment import EnvironmentContext
from alembic.migration import __name__ as migration_name
def run_alembic_migration(log_handler=None):
if log_handler:
logging.getLogger(migration_name).addHandler(log_handler)
config = Config()
config.set_main_option("script_location", "data:migrations")
script = ScriptDirectory.from_config(config)
def fn(rev, context):
return script._upgrade_revs('head', rev)
with EnvironmentContext(config, script, fn=fn, destination_rev='head'):
script.run_env()

6
emails/base.html
View file

@ -4,6 +4,12 @@
<meta name="viewport" content="width=device-width" />
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<title>{{ subject }}</title>
{% if action_metadata %}
<script type="application/ld+json">
{{ action_metadata }}
</script>
{% endif %}
</head>
<body bgcolor="#FFFFFF" style="font-family: 'Helvetica Neue', 'Helvetica', Helvetica, Arial, sans-serif; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; margin: 0; padding: 0;"><style type="text/css">
@media only screen and (max-width: 600px) {

25
endpoints/api/__init__.py
View file

@ -280,6 +280,23 @@ require_user_read = require_user_permission(UserReadPermission, scopes.READ_USER
require_user_admin = require_user_permission(UserAdminPermission, None)
require_fresh_user_admin = require_user_permission(UserAdminPermission, None)
def verify_not_prod(func):
@add_method_metadata('enterprise_only', True)
@wraps(func)
def wrapped(*args, **kwargs):
# Verify that we are not running on a production (i.e. hosted) stack. If so, we fail.
# This should never happen (because of the feature-flag on SUPER_USERS), but we want to be
# absolutely sure.
if app.config['SERVER_HOSTNAME'].find('quay.io') >= 0:
logger.error('!!! Super user method called IN PRODUCTION !!!')
raise NotFound()
return func(*args, **kwargs)
return wrapped
def require_fresh_login(func):
@add_method_metadata('requires_fresh_login', True)
@wraps(func)
@ -317,7 +334,11 @@ def validate_json_request(schema_name):
def wrapped(self, *args, **kwargs):
schema = self.schemas[schema_name]
try:
validate(request.get_json(), schema)
json_data = request.get_json()
if json_data is None:
raise InvalidRequest('Missing JSON body')
validate(json_data, schema)
return func(self, *args, **kwargs)
except ValidationError as ex:
raise InvalidRequest(ex.message)
@ -385,8 +406,10 @@ import endpoints.api.repoemail
import endpoints.api.repotoken
import endpoints.api.robot
import endpoints.api.search
import endpoints.api.suconfig
import endpoints.api.superuser
import endpoints.api.tag
import endpoints.api.team
import endpoints.api.trigger
import endpoints.api.user

44
endpoints/api/build.py
View file

@ -9,7 +9,7 @@ from app import app, userfiles as user_files, build_logs, log_archive
from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource,
require_repo_read, require_repo_write, validate_json_request,
ApiResource, internal_only, format_date, api, Unauthorized, NotFound,
path_param)
path_param, InvalidRequest, require_repo_admin)
from endpoints.common import start_build
from endpoints.trigger import BuildTrigger
from data import model, database
@ -70,10 +70,17 @@ def build_status_view(build_obj, can_write=False):
# If the status contains a heartbeat, then check to see if has been written in the last few
# minutes. If not, then the build timed out.
if status is not None and 'heartbeat' in status and status['heartbeat']:
heartbeat = datetime.datetime.fromtimestamp(status['heartbeat'])
if datetime.datetime.now() - heartbeat > datetime.timedelta(minutes=1):
phase = database.BUILD_PHASE.INTERNAL_ERROR
if phase != database.BUILD_PHASE.COMPLETE and phase != database.BUILD_PHASE.ERROR:
if status is not None and 'heartbeat' in status and status['heartbeat']:
heartbeat = datetime.datetime.utcfromtimestamp(status['heartbeat'])
if datetime.datetime.utcnow() - heartbeat > datetime.timedelta(minutes=1):
phase = database.BUILD_PHASE.INTERNAL_ERROR
# If the phase is internal error, return 'error' instead of the number if retries
# on the queue item is 0.
if phase == database.BUILD_PHASE.INTERNAL_ERROR:
if build_obj.queue_item is None or build_obj.queue_item.retries_remaining == 0:
phase = database.BUILD_PHASE.ERROR
logger.debug('Can write: %s job_config: %s', can_write, build_obj.job_config)
resp = {
@ -86,7 +93,7 @@ def build_status_view(build_obj, can_write=False):
'is_writer': can_write,
'trigger': trigger_view(build_obj.trigger),
'resource_key': build_obj.resource_key,
'pull_robot': user_view(build_obj.pull_robot) if build_obj.pull_robot else None,
'pull_robot': user_view(build_obj.pull_robot) if build_obj.pull_robot else None
}
if can_write:
@ -200,6 +207,31 @@ class RepositoryBuildList(RepositoryParamResource):
return resp, 201, headers
@resource('/v1/repository/<repopath:repository>/build/<build_uuid>')
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
@path_param('build_uuid', 'The UUID of the build')
class RepositoryBuildResource(RepositoryParamResource):
""" Resource for dealing with repository builds. """
@require_repo_admin
@nickname('cancelRepoBuild')
def delete(self, namespace, repository, build_uuid):
""" Cancels a repository build if it has not yet been picked up by a build worker. """
try:
build = model.get_repository_build(build_uuid)
except model.InvalidRepositoryBuildException:
raise NotFound()
if build.repository.name != repository or build.repository.namespace_user.username != namespace:
raise NotFound()
if model.cancel_repository_build(build):
return 'Okay', 201
else:
raise InvalidRequest('Build is currently running or has finished')
@resource('/v1/repository/<repopath:repository>/build/<build_uuid>/status')
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
@path_param('build_uuid', 'The UUID of the build')

9
endpoints/api/organization.py
View file

@ -116,6 +116,11 @@ class Organization(ApiResource):
'type': 'boolean',
'description': 'Whether the organization desires to receive emails for invoices',
},
'tag_expiration': {
'type': 'integer',
'maximum': 2592000,
'minimum': 0,
},
},
},
}
@ -161,6 +166,10 @@ class Organization(ApiResource):
logger.debug('Changing email address for organization: %s', org.username)
model.update_email(org, new_email)
if 'tag_expiration' in org_data:
logger.debug('Changing organization tag expiration to: %ss', org_data['tag_expiration'])
model.change_user_tag_expiration(org, org_data['tag_expiration'])
teams = model.get_teams_within_org(org)
return org_view(org, teams)
raise Unauthorized()

362
endpoints/api/suconfig.py Normal file
View file

@ -0,0 +1,362 @@
import logging
import os
import json
import signal
from flask import abort, Response
from endpoints.api import (ApiResource, nickname, resource, internal_only, show_if,
require_fresh_login, request, validate_json_request, verify_not_prod)
from endpoints.common import common_login
from app import app, CONFIG_PROVIDER, superusers
from data import model
from data.database import configure
from auth.permissions import SuperUserPermission
from auth.auth_context import get_authenticated_user
from data.database import User
from util.config.configutil import add_enterprise_config_defaults
from util.config.validator import validate_service_for_config, SSL_FILENAMES
from data.runmigration import run_alembic_migration
import features
logger = logging.getLogger(__name__)
def database_is_valid():
""" Returns whether the database, as configured, is valid. """
if app.config['TESTING']:
return False
try:
list(User.select().limit(1))
return True
except:
return False
def database_has_users():
""" Returns whether the database has any users defined. """
return bool(list(User.select().limit(1)))
@resource('/v1/superuser/registrystatus')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserRegistryStatus(ApiResource):
""" Resource for determining the status of the registry, such as if config exists,
if a database is configured, and if it has any defined users.
"""
@nickname('scRegistryStatus')
@verify_not_prod
def get(self):
""" Returns the status of the registry. """
# If there is no conf/stack volume, then report that status.
if not CONFIG_PROVIDER.volume_exists():
return {
'status': 'missing-config-dir'
}
# If there is no config file, we need to setup the database.
if not CONFIG_PROVIDER.yaml_exists():
return {
'status': 'config-db'
}
# If the database isn't yet valid, then we need to set it up.
if not database_is_valid():
return {
'status': 'setup-db'
}
# If we have SETUP_COMPLETE, then we're ready to go!
if app.config.get('SETUP_COMPLETE', False):
return {
'requires_restart': CONFIG_PROVIDER.requires_restart(app.config),
'status': 'ready'
}
return {
'status': 'create-superuser' if not database_has_users() else 'config'
}
class _AlembicLogHandler(logging.Handler):
def __init__(self):
super(_AlembicLogHandler, self).__init__()
self.records = []
def emit(self, record):
self.records.append({
'level': record.levelname,
'message': record.getMessage()
})
@resource('/v1/superuser/setupdb')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserSetupDatabase(ApiResource):
""" Resource for invoking alembic to setup the database. """
@verify_not_prod
@nickname('scSetupDatabase')
def get(self):
""" Invokes the alembic upgrade process. """
# Note: This method is called after the database configured is saved, but before the
# database has any tables. Therefore, we only allow it to be run in that unique case.
if CONFIG_PROVIDER.yaml_exists() and not database_is_valid():
# Note: We need to reconfigure the database here as the config has changed.
combined = dict(**app.config)
combined.update(CONFIG_PROVIDER.get_yaml())
configure(combined)
app.config['DB_URI'] = combined['DB_URI']
log_handler = _AlembicLogHandler()
try:
run_alembic_migration(log_handler)
except Exception as ex:
return {
'error': str(ex)
}
return {
'logs': log_handler.records
}
abort(403)
@resource('/v1/superuser/shutdown')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserShutdown(ApiResource):
""" Resource for sending a shutdown signal to the container. """
@verify_not_prod
@nickname('scShutdownContainer')
def post(self):
""" Sends a signal to the phusion init system to shut down the container. """
# Note: This method is called to set the database configuration before super users exists,
# so we also allow it to be called if there is no valid registry configuration setup.
if app.config['TESTING'] or not database_has_users() or SuperUserPermission().can():
# Note: We skip if debugging locally.
if app.config.get('DEBUGGING') == True:
return {}
os.kill(1, signal.SIGINT)
return {}
abort(403)
@resource('/v1/superuser/config')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserConfig(ApiResource):
""" Resource for fetching and updating the current configuration, if any. """
schemas = {
'UpdateConfig': {
'id': 'UpdateConfig',
'type': 'object',
'description': 'Updates the YAML config file',
'required': [
'config',
'hostname'
],
'properties': {
'config': {
'type': 'object'
},
'hostname': {
'type': 'string'
}
},
},
}
@require_fresh_login
@verify_not_prod
@nickname('scGetConfig')
def get(self):
""" Returns the currently defined configuration, if any. """
if SuperUserPermission().can():
config_object = CONFIG_PROVIDER.get_yaml()
return {
'config': config_object
}
abort(403)
@nickname('scUpdateConfig')
@verify_not_prod
@validate_json_request('UpdateConfig')
def put(self):
""" Updates the config.yaml file. """
# Note: This method is called to set the database configuration before super users exists,
# so we also allow it to be called if there is no valid registry configuration setup.
if not CONFIG_PROVIDER.yaml_exists() or SuperUserPermission().can():
config_object = request.get_json()['config']
hostname = request.get_json()['hostname']
# Add any enterprise defaults missing from the config.
add_enterprise_config_defaults(config_object, app.config['SECRET_KEY'], hostname)
# Write the configuration changes to the YAML file.
CONFIG_PROVIDER.save_yaml(config_object)
return {
'exists': True,
'config': config_object
}
abort(403)
@resource('/v1/superuser/config/file/<filename>')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserConfigFile(ApiResource):
""" Resource for fetching the status of config files and overriding them. """
@nickname('scConfigFileExists')
@verify_not_prod
def get(self, filename):
""" Returns whether the configuration file with the given name exists. """
if not filename in SSL_FILENAMES:
abort(404)
if SuperUserPermission().can():
return {
'exists': CONFIG_PROVIDER.volume_file_exists(filename)
}
abort(403)
@nickname('scUpdateConfigFile')
@verify_not_prod
def post(self, filename):
""" Updates the configuration file with the given name. """
if not filename in SSL_FILENAMES:
abort(404)
if SuperUserPermission().can():
uploaded_file = request.files['file']
if not uploaded_file:
abort(400)
CONFIG_PROVIDER.save_volume_file(filename, uploaded_file)
return {
'status': True
}
abort(403)
@resource('/v1/superuser/config/createsuperuser')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserCreateInitialSuperUser(ApiResource):
""" Resource for creating the initial super user. """
schemas = {
'CreateSuperUser': {
'id': 'CreateSuperUser',
'type': 'object',
'description': 'Information for creating the initial super user',
'required': [
'username',
'password',
'email'
],
'properties': {
'username': {
'type': 'string',
'description': 'The username for the superuser'
},
'password': {
'type': 'string',
'description': 'The password for the superuser'
},
'email': {
'type': 'string',
'description': 'The e-mail address for the superuser'
},
},
},
}
@nickname('scCreateInitialSuperuser')
@verify_not_prod
@validate_json_request('CreateSuperUser')
def post(self):
""" Creates the initial super user, updates the underlying configuration and
sets the current session to have that super user. """
# Special security check: This method is only accessible when:
# - There is a valid config YAML file.
# - There are currently no users in the database (clean install)
#
# We do this special security check because at the point this method is called, the database
# is clean but does not (yet) have any super users for our permissions code to check against.
if CONFIG_PROVIDER.yaml_exists() and not database_has_users():
data = request.get_json()
username = data['username']
password = data['password']
email = data['email']
# Create the user in the database.
superuser = model.create_user(username, password, email, auto_verify=True)
# Add the user to the config.
config_object = CONFIG_PROVIDER.get_yaml()
config_object['SUPER_USERS'] = [username]
CONFIG_PROVIDER.save_yaml(config_object)
# Update the in-memory config for the new superuser.
superusers.register_superuser(username)
# Conduct login with that user.
common_login(superuser)
return {
'status': True
}
abort(403)
@resource('/v1/superuser/config/validate/<service>')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserConfigValidate(ApiResource):
""" Resource for validating a block of configuration against an external service. """
schemas = {
'ValidateConfig': {
'id': 'ValidateConfig',
'type': 'object',
'description': 'Validates configuration',
'required': [
'config'
],
'properties': {
'config': {
'type': 'object'
}
},
},
}
@nickname('scValidateConfig')
@verify_not_prod
@validate_json_request('ValidateConfig')
def post(self, service):
""" Validates the given config for the given service. """
# Note: This method is called to validate the database configuration before super users exists,
# so we also allow it to be called if there is no valid registry configuration setup. Note that
# this is also safe since this method does not access any information not given in the request.
if not CONFIG_PROVIDER.yaml_exists() or SuperUserPermission().can():
config = request.get_json()['config']
return validate_service_for_config(service, config)
abort(403)

81
endpoints/api/superuser.py
View file

@ -1,15 +1,16 @@
import string
import logging
import json
import os
from random import SystemRandom
from app import app
from app import app, avatar, superusers
from flask import request
from endpoints.api import (ApiResource, nickname, resource, validate_json_request, request_error,
log_action, internal_only, NotFound, require_user_admin, format_date,
InvalidToken, require_scope, format_date, hide_if, show_if, parse_args,
query_param, abort, require_fresh_login, path_param)
query_param, abort, require_fresh_login, path_param, verify_not_prod)
from endpoints.api.logs import get_logs
@ -22,18 +23,76 @@ import features
logger = logging.getLogger(__name__)
def get_immediate_subdirectories(directory):
return [name for name in os.listdir(directory) if os.path.isdir(os.path.join(directory, name))]
def get_services():
services = set(get_immediate_subdirectories(app.config['SYSTEM_SERVICES_PATH']))
services = services - set(app.config['SYSTEM_SERVICE_BLACKLIST'])
return services
@resource('/v1/superuser/systemlogs/<service>')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserGetLogsForService(ApiResource):
""" Resource for fetching the kinds of system logs in the system. """
@require_fresh_login
@verify_not_prod
@nickname('getSystemLogs')
def get(self, service):
""" Returns the logs for the specific service. """
if SuperUserPermission().can():
if not service in get_services():
abort(404)
try:
with open(app.config['SYSTEM_SERVICE_LOGS_PATH'] % service, 'r') as f:
logs = f.read()
except Exception as ex:
logger.exception('Cannot read logs')
abort(400)
return {
'logs': logs
}
abort(403)
@resource('/v1/superuser/systemlogs/')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserSystemLogServices(ApiResource):
""" Resource for fetching the kinds of system logs in the system. """
@require_fresh_login
@verify_not_prod
@nickname('listSystemLogServices')
def get(self):
""" List the system logs for the current system. """
if SuperUserPermission().can():
return {
'services': list(get_services())
}
abort(403)
@resource('/v1/superuser/logs')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserLogs(ApiResource):
""" Resource for fetching all logs in the system. """
@require_fresh_login
@verify_not_prod
@nickname('listAllLogs')
@parse_args
@query_param('starttime', 'Earliest time from which to get logs. (%m/%d/%Y %Z)', type=str)
@query_param('endtime', 'Latest time to which to get logs. (%m/%d/%Y %Z)', type=str)
@query_param('performer', 'Username for which to filter logs.', type=str)
def get(self, args):
""" List the logs for the current system. """
""" List the usage logs for the current system. """
if SuperUserPermission().can():
performer_name = args['performer']
start_time = args['starttime']
@ -49,7 +108,8 @@ def user_view(user):
'username': user.username,
'email': user.email,
'verified': user.verified,
'super_user': user.username in app.config['SUPER_USERS']
'avatar': avatar.compute_hash(user.email, name=user.username),
'super_user': superusers.is_superuser(user.username)
}
@resource('/v1/superuser/usage/')
@ -58,6 +118,7 @@ def user_view(user):
class UsageInformation(ApiResource):
""" Resource for returning the usage information for enterprise customers. """
@require_fresh_login
@verify_not_prod
@nickname('getSystemUsage')
def get(self):
""" Returns the number of repository handles currently held. """
@ -96,6 +157,7 @@ class SuperUserList(ApiResource):
}
@require_fresh_login
@verify_not_prod
@nickname('listAllUsers')
def get(self):
""" Returns a list of all users in the system. """
@ -109,6 +171,7 @@ class SuperUserList(ApiResource):
@require_fresh_login
@verify_not_prod
@nickname('createInstallUser')
@validate_json_request('CreateInstallUser')
def post(self):
@ -146,6 +209,7 @@ class SuperUserList(ApiResource):
class SuperUserSendRecoveryEmail(ApiResource):
""" Resource for sending a recovery user on behalf of a user. """
@require_fresh_login
@verify_not_prod
@nickname('sendInstallUserRecoveryEmail')
def post(self, username):
if SuperUserPermission().can():
@ -153,7 +217,7 @@ class SuperUserSendRecoveryEmail(ApiResource):
if not user or user.organization or user.robot:
abort(404)
if username in app.config['SUPER_USERS']:
if superusers.is_superuser(username):
abort(403)
code = model.create_reset_password_email_code(user.email)
@ -190,6 +254,7 @@ class SuperUserManagement(ApiResource):
}
@require_fresh_login
@verify_not_prod
@nickname('getInstallUser')
def get(self, username):
""" Returns information about the specified user. """
@ -203,6 +268,7 @@ class SuperUserManagement(ApiResource):
abort(403)
@require_fresh_login
@verify_not_prod
@nickname('deleteInstallUser')
def delete(self, username):
""" Deletes the specified user. """
@ -211,7 +277,7 @@ class SuperUserManagement(ApiResource):
if not user or user.organization or user.robot:
abort(404)
if username in app.config['SUPER_USERS']:
if superusers.is_superuser(username):
abort(403)
model.delete_user(user)
@ -220,6 +286,7 @@ class SuperUserManagement(ApiResource):
abort(403)
@require_fresh_login
@verify_not_prod
@nickname('changeInstallUser')
@validate_json_request('UpdateUser')
def put(self, username):
@ -229,7 +296,7 @@ class SuperUserManagement(ApiResource):
if not user or user.organization or user.robot:
abort(404)
if username in app.config['SUPER_USERS']:
if superusers.is_superuser(username):
abort(403)
user_data = request.get_json()

4
endpoints/api/tag.py
View file

@ -54,8 +54,8 @@ class RepositoryTag(RepositoryParamResource):
username = get_authenticated_user().username
log_action('move_tag' if original_image_id else 'create_tag', namespace,
{ 'username': username, 'repo': repository, 'tag': tag,
'image': image_id, 'original_image': original_image_id },
{'username': username, 'repo': repository, 'tag': tag,
'image': image_id, 'original_image': original_image_id},
repo=model.get_repository(namespace, repository))
return 'Updated', 201

4
endpoints/api/trigger.py
View file

@ -415,13 +415,13 @@ class ActivateBuildTrigger(RepositoryParamResource):
try:
run_parameters = request.get_json()
specs = handler.manual_start(trigger.auth_token, config_dict, run_parameters=run_parameters)
dockerfile_id, tags, name, subdir = specs
dockerfile_id, tags, name, subdir, metadata = specs
repo = model.get_repository(namespace, repository)
pull_robot_name = model.get_pull_robot_name(trigger)
build_request = start_build(repo, dockerfile_id, tags, name, subdir, True,
pull_robot_name=pull_robot_name)
pull_robot_name=pull_robot_name, trigger_metadata=metadata)
except TriggerStartException as tse:
raise InvalidRequest(tse.message)

13
endpoints/api/user.py
View file

@ -73,6 +73,7 @@ def user_view(user):
'can_create_repo': True,
'invoice_email': user.invoice_email,
'preferred_namespace': not (user.stripe_id is None),
'tag_expiration': user.removed_tag_expiration_s,
})
if features.SUPER_USERS:
@ -144,6 +145,11 @@ class User(ApiResource):
'type': 'string',
'description': 'The user\'s email address',
},
'tag_expiration': {
'type': 'integer',
'maximum': 2592000,
'minimum': 0,
},
'username': {
'type': 'string',
'description': 'The user\'s username',
@ -227,6 +233,10 @@ class User(ApiResource):
logger.debug('Changing invoice_email for user: %s', user.username)
model.change_invoice_email(user, user_data['invoice_email'])
if 'tag_expiration' in user_data:
logger.debug('Changing user tag expiration to: %ss', user_data['tag_expiration'])
model.change_user_tag_expiration(user, user_data['tag_expiration'])
if 'email' in user_data and user_data['email'] != user.email:
new_email = user_data['email']
if model.find_user_by_email(new_email):
@ -248,7 +258,8 @@ class User(ApiResource):
# Username already used
raise request_error(message='Username is already in use')
model.change_username(user, new_username)
model.change_username(user.id, new_username)
except model.InvalidPasswordException, ex:
raise request_error(exception=ex)

114
endpoints/common.py
View file

@ -3,15 +3,19 @@ import urlparse
import json
import string
import datetime
import os
# Register the various exceptions via decorators.
import endpoints.decorated
from flask import make_response, render_template, request, abort, session
from flask.ext.login import login_user, UserMixin
from flask.ext.login import login_user
from flask.ext.principal import identity_changed
from random import SystemRandom
from data import model
from data.database import db
from app import app, login_manager, dockerfile_build_queue, notification_queue, oauth_apps
from app import app, oauth_apps, dockerfile_build_queue, LoginWrappedDBUser
from auth.permissions import QuayDeferredPermissionUser
from auth import scopes
@ -21,15 +25,30 @@ from functools import wraps
from config import getFrontendVisibleConfig
from external_libraries import get_external_javascript, get_external_css
from endpoints.notificationhelper import spawn_notification
from util.useremails import CannotSendEmailException
import features
logger = logging.getLogger(__name__)
profile = logging.getLogger('application.profiler')
route_data = None
CACHE_BUSTERS_JSON = 'static/dist/cachebusters.json'
CACHE_BUSTERS = None
def get_cache_busters():
""" Retrieves the cache busters hashes. """
global CACHE_BUSTERS
if CACHE_BUSTERS is not None:
return CACHE_BUSTERS
if not os.path.exists(CACHE_BUSTERS_JSON):
return {}
with open(CACHE_BUSTERS_JSON, 'r') as f:
CACHE_BUSTERS = json.loads(f.read())
return CACHE_BUSTERS
class RepoPathConverter(BaseConverter):
regex = '[\.a-zA-Z0-9_\-]+/[\.a-zA-Z0-9_\-]+'
weight = 200
@ -84,34 +103,8 @@ def param_required(param_name):
return wrapper
@login_manager.user_loader
def load_user(user_uuid):
logger.debug('User loader loading deferred user with uuid: %s' % user_uuid)
return _LoginWrappedDBUser(user_uuid)
class _LoginWrappedDBUser(UserMixin):
def __init__(self, user_uuid, db_user=None):
self._uuid = user_uuid
self._db_user = db_user
def db_user(self):
if not self._db_user:
self._db_user = model.get_user_by_uuid(self._uuid)
return self._db_user
def is_authenticated(self):
return self.db_user() is not None
def is_active(self):
return self.db_user().verified
def get_id(self):
return unicode(self._uuid)
def common_login(db_user):
if login_user(_LoginWrappedDBUser(db_user.uuid, db_user)):
if login_user(LoginWrappedDBUser(db_user.uuid, db_user)):
logger.debug('Successfully signed in as: %s (%s)' % (db_user.username, db_user.uuid))
new_identity = QuayDeferredPermissionUser(db_user.uuid, 'user_uuid', {scopes.DIRECT_LOGIN})
identity_changed.send(app, identity=new_identity)
@ -121,17 +114,6 @@ def common_login(db_user):
logger.debug('User could not be logged in, inactive?.')
return False
@app.errorhandler(model.DataModelException)
def handle_dme(ex):
logger.exception(ex)
return make_response(json.dumps({'message': ex.message}), 400)
@app.errorhandler(CannotSendEmailException)
def handle_emailexception(ex):
message = 'Could not send email. Please contact an administrator and report this problem.'
return make_response(json.dumps({'message': message}), 400)
def random_string():
random = SystemRandom()
return ''.join([random.choice(string.ascii_uppercase + string.digits) for _ in range(8)])
@ -148,17 +130,15 @@ def list_files(path, extension):
filepath = 'static/' + path
return [join_path(dp, f) for dp, dn, files in os.walk(filepath) for f in files if matches(f)]
SAVED_CACHE_STRING = random_string()
def render_page_template(name, **kwargs):
if app.config.get('DEBUGGING', False):
debugging = app.config.get('DEBUGGING', False)
if debugging:
# If DEBUGGING is enabled, then we load the full set of individual JS and CSS files
# from the file system.
library_styles = list_files('lib', 'css')
main_styles = list_files('css', 'css')
library_scripts = list_files('lib', 'js')
main_scripts = list_files('js', 'js')
cache_buster = 'debugging'
file_lists = [library_styles, main_styles, library_scripts, main_scripts]
for file_list in file_lists:
@ -168,7 +148,6 @@ def render_page_template(name, **kwargs):
main_styles = ['dist/quay-frontend.css']
library_scripts = []
main_scripts = ['dist/quay-frontend.min.js']
cache_buster = SAVED_CACHE_STRING
use_cdn = app.config.get('USE_CDN', True)
if request.args.get('use_cdn') is not None:
@ -177,6 +156,12 @@ def render_page_template(name, **kwargs):
external_styles = get_external_css(local=not use_cdn)
external_scripts = get_external_javascript(local=not use_cdn)
def add_cachebusters(filenames):
cachebusters = get_cache_busters()
for filename in filenames:
cache_buster = cachebusters.get(filename, random_string()) if not debugging else 'debugging'
yield (filename, cache_buster)
def get_oauth_config():
oauth_config = {}
for oauth_app in oauth_apps:
@ -188,13 +173,14 @@ def render_page_template(name, **kwargs):
if len(app.config.get('CONTACT_INFO', [])) == 1:
contact_href = app.config['CONTACT_INFO'][0]
resp = make_response(render_template(name, route_data=json.dumps(get_route_data()),
resp = make_response(render_template(name,
route_data=json.dumps(get_route_data()),
external_styles=external_styles,
external_scripts=external_scripts,
main_styles=main_styles,
library_styles=library_styles,
main_scripts=main_scripts,
library_scripts=library_scripts,
main_styles=add_cachebusters(main_styles),
library_styles=add_cachebusters(library_styles),
main_scripts=add_cachebusters(main_scripts),
library_scripts=add_cachebusters(library_scripts),
feature_set=json.dumps(features.get_features()),
config_set=json.dumps(getFrontendVisibleConfig(app.config)),
oauth_set=json.dumps(get_oauth_config()),
@ -204,9 +190,10 @@ def render_page_template(name, **kwargs):
sentry_public_dsn=app.config.get('SENTRY_PUBLIC_DSN', ''),
is_debug=str(app.config.get('DEBUGGING', False)).lower(),
show_chat=features.OLARK_CHAT,
cache_buster=cache_buster,
has_billing=features.BILLING,
contact_href=contact_href,
hostname=app.config['SERVER_HOSTNAME'],
preferred_scheme=app.config['PREFERRED_URL_SCHEME'],
**kwargs))
resp.headers['X-FRAME-OPTIONS'] = 'DENY'
@ -224,18 +211,20 @@ def check_repository_usage(user_or_org, plan_found):
def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
trigger=None, pull_robot_name=None):
trigger=None, pull_robot_name=None, trigger_metadata=None):
host = urlparse.urlparse(request.url).netloc
repo_path = '%s/%s/%s' % (host, repository.namespace_user.username, repository.name)
token = model.create_access_token(repository, 'write')
token = model.create_access_token(repository, 'write', kind='build-worker',
friendly_name='Repository Build Token')
logger.debug('Creating build %s with repo %s tags %s and dockerfile_id %s',
build_name, repo_path, tags, dockerfile_id)
job_config = {
'docker_tags': tags,
'registry': host,
'build_subdir': subdir
'build_subdir': subdir,
'trigger_metadata': trigger_metadata or {}
}
with app.config['DB_TRANSACTION_FACTORY'](db):
@ -243,10 +232,17 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
dockerfile_id, build_name,
trigger, pull_robot_name=pull_robot_name)
dockerfile_build_queue.put([repository.namespace_user.username, repository.name], json.dumps({
json_data = json.dumps({
'build_uuid': build_request.uuid,
'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None
}), retries_remaining=1)
})
queue_item = dockerfile_build_queue.put([repository.namespace_user.username, repository.name],
json_data,
retries_remaining=3)
build_request.queue_item = queue_item
build_request.save()
# Add the build to the repo's log.
metadata = {
@ -265,7 +261,7 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
metadata=metadata, repository=repository)
# Add notifications for the build queue.
profile.debug('Adding notifications for repository')
logger.debug('Adding notifications for repository')
event_data = {
'build_id': build_request.uuid,
'build_name': build_name,

16
endpoints/csrf.py
View file

@ -19,19 +19,21 @@ def generate_csrf_token():
return session['_csrf_token']
def verify_csrf():
token = session.get('_csrf_token', None)
found_token = request.values.get('_csrf_token', None)
if not token or token != found_token:
msg = 'CSRF Failure. Session token was %s and request token was %s'
logger.error(msg, token, found_token)
abort(403, message='CSRF token was invalid or missing.')
def csrf_protect(func):
@wraps(func)
def wrapper(*args, **kwargs):
oauth_token = get_validated_oauth_token()
if oauth_token is None and request.method != "GET" and request.method != "HEAD":
token = session.get('_csrf_token', None)
found_token = request.values.get('_csrf_token', None)
if not token or token != found_token:
msg = 'CSRF Failure. Session token was %s and request token was %s'
logger.error(msg, token, found_token)
abort(403, message='CSRF token was invalid or missing.')
verify_csrf()
return func(*args, **kwargs)
return wrapper

19
endpoints/decorated.py Normal file
View file

@ -0,0 +1,19 @@
import logging
import json
from flask import make_response
from app import app
from util.useremails import CannotSendEmailException
from data import model
logger = logging.getLogger(__name__)
@app.errorhandler(model.DataModelException)
def handle_dme(ex):
logger.exception(ex)
return make_response(json.dumps({'message': ex.message}), 400)
@app.errorhandler(CannotSendEmailException)
def handle_emailexception(ex):
message = 'Could not send email. Please contact an administrator and report this problem.'
return make_response(json.dumps({'message': message}), 400)

74
endpoints/index.py
View file

@ -23,7 +23,6 @@ from endpoints.notificationhelper import spawn_notification
import features
logger = logging.getLogger(__name__)
profile = logging.getLogger('application.profiler')
index = Blueprint('index', __name__)
@ -51,7 +50,7 @@ def generate_headers(role='read'):
if has_token_request:
repo = model.get_repository(namespace, repository)
if repo:
token = model.create_access_token(repo, role)
token = model.create_access_token(repo, role, 'pushpull-token')
token_str = 'signature=%s' % token.code
response.headers['WWW-Authenticate'] = token_str
response.headers['X-Docker-Token'] = token_str
@ -120,7 +119,7 @@ def create_user():
else:
# New user case
profile.debug('Creating user')
logger.debug('Creating user')
new_user = None
try:
@ -128,10 +127,10 @@ def create_user():
except model.TooManyUsersException as ex:
abort(402, 'Seat limit has been reached for this license', issue='seat-limit')
profile.debug('Creating email code for user')
logger.debug('Creating email code for user')
code = model.create_confirm_email_code(new_user)
profile.debug('Sending email code to user')
logger.debug('Sending email code to user')
send_confirmation_email(new_user.username, new_user.email, code.code)
return make_response('Created', 201)
@ -168,12 +167,12 @@ def update_user(username):
update_request = request.get_json()
if 'password' in update_request:
profile.debug('Updating user password')
logger.debug('Updating user password')
model.change_password(get_authenticated_user(),
update_request['password'])
if 'email' in update_request:
profile.debug('Updating user email')
logger.debug('Updating user email')
model.update_email(get_authenticated_user(), update_request['email'])
return jsonify({
@ -189,13 +188,13 @@ def update_user(username):
@parse_repository_name
@generate_headers(role='write')
def create_repository(namespace, repository):
profile.debug('Parsing image descriptions')
logger.debug('Parsing image descriptions')
image_descriptions = json.loads(request.data.decode('utf8'))
profile.debug('Looking up repository')
logger.debug('Looking up repository')
repo = model.get_repository(namespace, repository)
profile.debug('Repository looked up')
logger.debug('Repository looked up')
if not repo and get_authenticated_user() is None:
logger.debug('Attempt to create new repository without user auth.')
abort(401,
@ -219,36 +218,10 @@ def create_repository(namespace, repository):
issue='no-create-permission',
namespace=namespace)
profile.debug('Creaing repository with owner: %s', get_authenticated_user().username)
logger.debug('Creaing repository with owner: %s', get_authenticated_user().username)
repo = model.create_repository(namespace, repository,
get_authenticated_user())
profile.debug('Determining already added images')
added_images = OrderedDict([(desc['id'], desc) for desc in image_descriptions])
new_repo_images = dict(added_images)
# Optimization: Lookup any existing images in the repository with matching docker IDs and
# remove them from the added dict, so we don't need to look them up one-by-one.
def chunks(l, n):
for i in xrange(0, len(l), n):
yield l[i:i+n]
# Note: We do this in chunks in an effort to not hit the SQL query size limit.
for chunk in chunks(new_repo_images.keys(), 50):
existing_images = model.lookup_repository_images(namespace, repository, chunk)
for existing in existing_images:
added_images.pop(existing.docker_image_id)
profile.debug('Creating/Linking necessary images')
username = get_authenticated_user() and get_authenticated_user().username
translations = {}
for image_description in added_images.values():
model.find_create_or_link_image(image_description['id'], repo, username,
translations, storage.preferred_locations[0])
profile.debug('Created images')
track_and_log('push_repo', repo)
return make_response('Created', 201)
@ -260,14 +233,14 @@ def update_images(namespace, repository):
permission = ModifyRepositoryPermission(namespace, repository)
if permission.can():
profile.debug('Looking up repository')
logger.debug('Looking up repository')
repo = model.get_repository(namespace, repository)
if not repo:
# Make sure the repo actually exists.
abort(404, message='Unknown repository', issue='unknown-repo')
if get_authenticated_user():
profile.debug('Publishing push event')
logger.debug('Publishing push event')
username = get_authenticated_user().username
# Mark that the user has pushed the repo.
@ -280,17 +253,17 @@ def update_images(namespace, repository):
event = userevents.get_event(username)
event.publish_event_data('docker-cli', user_data)
profile.debug('GCing repository')
num_removed = model.garbage_collect_repository(namespace, repository)
logger.debug('GCing repository')
model.garbage_collect_repository(namespace, repository)
# Generate a job for each notification that has been added to this repo
profile.debug('Adding notifications for repository')
logger.debug('Adding notifications for repository')
updated_tags = session.get('pushed_tags', {})
event_data = {
'updated_tags': updated_tags,
'pruned_image_count': num_removed
}
track_and_log('push_repo', repo)
spawn_notification(repo, 'repo_push', event_data)
return make_response('Updated', 204)
@ -305,17 +278,15 @@ def get_repository_images(namespace, repository):
permission = ReadRepositoryPermission(namespace, repository)
# TODO invalidate token?
profile.debug('Looking up public status of repository')
is_public = model.repository_is_public(namespace, repository)
if permission.can() or is_public:
if permission.can() or model.repository_is_public(namespace, repository):
# We can't rely on permissions to tell us if a repo exists anymore
profile.debug('Looking up repository')
logger.debug('Looking up repository')
repo = model.get_repository(namespace, repository)
if not repo:
abort(404, message='Unknown repository', issue='unknown-repo')
all_images = []
profile.debug('Retrieving repository images')
logger.debug('Retrieving repository images')
for image in model.get_repository_images(namespace, repository):
new_image_view = {
'id': image.docker_image_id,
@ -323,7 +294,7 @@ def get_repository_images(namespace, repository):
}
all_images.append(new_image_view)
profile.debug('Building repository image response')
logger.debug('Building repository image response')
resp = make_response(json.dumps(all_images), 200)
resp.mimetype = 'application/json'
@ -382,6 +353,11 @@ def get_search():
resp.mimetype = 'application/json'
return resp
# Note: This is *not* part of the Docker index spec. This is here for our own health check,
# since we have nginx handle the _ping below.
@index.route('/_internal_ping')
def internal_ping():
return make_response('true', 200)
@index.route('/_ping')
@index.route('/_ping')

23
endpoints/notificationmethod.py
View file

@ -1,14 +1,10 @@
import logging
import io
import os.path
import tarfile
import base64
import json
import requests
import re
from flask.ext.mail import Message
from app import mail, app, get_app_url
from app import mail, app
from data import model
from workers.worker import JobException
@ -363,11 +359,8 @@ class SlackMethod(NotificationMethod):
return 'slack'
def validate(self, repository, config_data):
if not config_data.get('token', ''):
raise CannotValidateNotificationMethodException('Missing Slack Token')
if not config_data.get('subdomain', '').isalnum():
raise CannotValidateNotificationMethodException('Missing Slack Subdomain Name')
if not config_data.get('url', ''):
raise CannotValidateNotificationMethodException('Missing Slack Callback URL')
def format_for_slack(self, message):
message = message.replace('\n', '')
@ -378,10 +371,8 @@ class SlackMethod(NotificationMethod):
def perform(self, notification, event_handler, notification_data):
config_data = json.loads(notification.config_json)
token = config_data.get('token', '')
subdomain = config_data.get('subdomain', '')
if not token or not subdomain:
url = config_data.get('url', '')
if not url:
return
owner = model.get_user_or_org(notification.repository.namespace_user.username)
@ -389,8 +380,6 @@ class SlackMethod(NotificationMethod):
# Something went wrong.
return
url = 'https://%s.slack.com/services/hooks/incoming-webhook?token=%s' % (subdomain, token)
level = event_handler.get_level(notification_data['event_data'], notification_data)
color = {
'info': '#ffffff',
@ -426,5 +415,5 @@ class SlackMethod(NotificationMethod):
raise NotificationMethodPerformException(error_message)
except requests.exceptions.RequestException as ex:
logger.exception('Slack method was unable to be sent: %s' % ex.message)
logger.exception('Slack method was unable to be sent: %s', ex.message)
raise NotificationMethodPerformException(ex.message)

51
endpoints/realtime.py
View file

@ -4,13 +4,62 @@ import json
from flask import request, Blueprint, abort, Response
from flask.ext.login import current_user
from auth.auth import require_session_login
from app import userevents
from endpoints.common import route_show_if
from app import app, userevents
from auth.permissions import SuperUserPermission
import features
import psutil
import time
logger = logging.getLogger(__name__)
realtime = Blueprint('realtime', __name__)
@realtime.route("/ps")
@route_show_if(features.SUPER_USERS)
@require_session_login
def ps():
if not SuperUserPermission().can():
abort(403)
def generator():
while True:
build_status = {}
try:
builder_data = app.config['HTTPCLIENT'].get('http://localhost:8686/status', timeout=1)
if builder_data.status_code == 200:
build_status = json.loads(builder_data.text)
except:
pass
try:
data = {
'count': {
'cpu': psutil.cpu_percent(interval=1, percpu=True),
'virtual_mem': psutil.virtual_memory(),
'swap_mem': psutil.swap_memory(),
'connections': len(psutil.net_connections()),
'processes': len(psutil.pids()),
'network': psutil.net_io_counters()
},
'build': build_status
}
except psutil.AccessDenied:
data = {}
json_string = json.dumps(data)
yield 'data: %s\n\n' % json_string
time.sleep(1)
try:
return Response(generator(), mimetype="text/event-stream")
except:
pass
@realtime.route("/user/")
@require_session_login
def index():

111
endpoints/registry.py
View file

@ -9,6 +9,7 @@ from time import time
from app import storage as store, image_diff_queue, app
from auth.auth import process_auth, extract_namespace_repo_from_session
from auth.auth_context import get_authenticated_user
from util import checksums, changes
from util.http import abort, exact_abort
from auth.permissions import (ReadRepositoryPermission,
@ -20,7 +21,6 @@ from util import gzipstream
registry = Blueprint('registry', __name__)
logger = logging.getLogger(__name__)
profile = logging.getLogger('application.profiler')
class SocketReader(object):
def __init__(self, fp):
@ -100,12 +100,12 @@ def set_cache_headers(f):
def head_image_layer(namespace, repository, image_id, headers):
permission = ReadRepositoryPermission(namespace, repository)
profile.debug('Checking repo permissions')
logger.debug('Checking repo permissions')
if permission.can() or model.repository_is_public(namespace, repository):
profile.debug('Looking up repo image')
logger.debug('Looking up repo image')
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
if not repo_image:
profile.debug('Image not found')
logger.debug('Image not found')
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
image_id=image_id)
@ -114,7 +114,7 @@ def head_image_layer(namespace, repository, image_id, headers):
# Add the Accept-Ranges header if the storage engine supports resumable
# downloads.
if store.get_supports_resumable_downloads(repo_image.storage.locations):
profile.debug('Storage supports resumable downloads')
logger.debug('Storage supports resumable downloads')
extra_headers['Accept-Ranges'] = 'bytes'
resp = make_response('')
@ -133,31 +133,35 @@ def head_image_layer(namespace, repository, image_id, headers):
def get_image_layer(namespace, repository, image_id, headers):
permission = ReadRepositoryPermission(namespace, repository)
profile.debug('Checking repo permissions')
logger.debug('Checking repo permissions')
if permission.can() or model.repository_is_public(namespace, repository):
profile.debug('Looking up repo image')
logger.debug('Looking up repo image')
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
if not repo_image:
logger.debug('Image not found')
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
image_id=image_id)
profile.debug('Looking up the layer path')
logger.debug('Looking up the layer path')
try:
path = store.image_layer_path(repo_image.storage.uuid)
profile.debug('Looking up the direct download URL')
logger.debug('Looking up the direct download URL')
direct_download_url = store.get_direct_download_url(repo_image.storage.locations, path)
if direct_download_url:
profile.debug('Returning direct download URL')
logger.debug('Returning direct download URL')
resp = redirect(direct_download_url)
return resp
profile.debug('Streaming layer data')
logger.debug('Streaming layer data')
# Close the database handle here for this process before we send the long download.
database.close_db_filter(None)
return Response(store.stream_read(repo_image.storage.locations, path), headers=headers)
except (IOError, AttributeError):
profile.debug('Image not found')
logger.exception('Image layer data not found')
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
image_id=image_id)
@ -168,29 +172,30 @@ def get_image_layer(namespace, repository, image_id, headers):
@process_auth
@extract_namespace_repo_from_session
def put_image_layer(namespace, repository, image_id):
profile.debug('Checking repo permissions')
logger.debug('Checking repo permissions')
permission = ModifyRepositoryPermission(namespace, repository)
if not permission.can():
abort(403)
profile.debug('Retrieving image')
logger.debug('Retrieving image')
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
try:
profile.debug('Retrieving image data')
logger.debug('Retrieving image data')
uuid = repo_image.storage.uuid
json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
except (IOError, AttributeError):
logger.exception('Exception when retrieving image data')
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
image_id=image_id)
profile.debug('Retrieving image path info')
logger.debug('Retrieving image path info')
layer_path = store.image_layer_path(uuid)
if (store.exists(repo_image.storage.locations, layer_path) and not
image_is_uploading(repo_image)):
exact_abort(409, 'Image already exists')
profile.debug('Storing layer data')
logger.debug('Storing layer data')
input_stream = request.stream
if request.headers.get('transfer-encoding') == 'chunked':
@ -257,7 +262,7 @@ def put_image_layer(namespace, repository, image_id):
# The layer is ready for download, send a job to the work queue to
# process it.
profile.debug('Adding layer to diff queue')
logger.debug('Adding layer to diff queue')
repo = model.get_repository(namespace, repository)
image_diff_queue.put([repo.namespace_user.username, repository, image_id], json.dumps({
'namespace_user_id': repo.namespace_user.id,
@ -272,7 +277,7 @@ def put_image_layer(namespace, repository, image_id):
@process_auth
@extract_namespace_repo_from_session
def put_image_checksum(namespace, repository, image_id):
profile.debug('Checking repo permissions')
logger.debug('Checking repo permissions')
permission = ModifyRepositoryPermission(namespace, repository)
if not permission.can():
abort(403)
@ -298,23 +303,23 @@ def put_image_checksum(namespace, repository, image_id):
abort(400, 'Checksum not found in Cookie for image %(image_id)s',
issue='missing-checksum-cookie', image_id=image_id)
profile.debug('Looking up repo image')
logger.debug('Looking up repo image')
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
if not repo_image or not repo_image.storage:
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
uuid = repo_image.storage.uuid
profile.debug('Looking up repo layer data')
logger.debug('Looking up repo layer data')
if not store.exists(repo_image.storage.locations, store.image_json_path(uuid)):
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
profile.debug('Marking image path')
logger.debug('Marking image path')
if not image_is_uploading(repo_image):
abort(409, 'Cannot set checksum for image %(image_id)s',
issue='image-write-error', image_id=image_id)
profile.debug('Storing image checksum')
logger.debug('Storing image checksum')
err = store_checksum(repo_image.storage, checksum)
if err:
abort(400, err)
@ -331,7 +336,7 @@ def put_image_checksum(namespace, repository, image_id):
# The layer is ready for download, send a job to the work queue to
# process it.
profile.debug('Adding layer to diff queue')
logger.debug('Adding layer to diff queue')
repo = model.get_repository(namespace, repository)
image_diff_queue.put([repo.namespace_user.username, repository, image_id], json.dumps({
'namespace_user_id': repo.namespace_user.id,
@ -348,23 +353,23 @@ def put_image_checksum(namespace, repository, image_id):
@require_completion
@set_cache_headers
def get_image_json(namespace, repository, image_id, headers):
profile.debug('Checking repo permissions')
logger.debug('Checking repo permissions')
permission = ReadRepositoryPermission(namespace, repository)
if not permission.can() and not model.repository_is_public(namespace,
repository):
abort(403)
profile.debug('Looking up repo image')
logger.debug('Looking up repo image')
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
profile.debug('Looking up repo layer data')
logger.debug('Looking up repo layer data')
try:
uuid = repo_image.storage.uuid
data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
except (IOError, AttributeError):
flask_abort(404)
profile.debug('Looking up repo layer size')
logger.debug('Looking up repo layer size')
size = repo_image.storage.image_size
headers['X-Docker-Size'] = str(size)
@ -379,16 +384,16 @@ def get_image_json(namespace, repository, image_id, headers):
@require_completion
@set_cache_headers
def get_image_ancestry(namespace, repository, image_id, headers):
profile.debug('Checking repo permissions')
logger.debug('Checking repo permissions')
permission = ReadRepositoryPermission(namespace, repository)
if not permission.can() and not model.repository_is_public(namespace,
repository):
abort(403)
profile.debug('Looking up repo image')
logger.debug('Looking up repo image')
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
profile.debug('Looking up image data')
logger.debug('Looking up image data')
try:
uuid = repo_image.storage.uuid
data = store.get_content(repo_image.storage.locations, store.image_ancestry_path(uuid))
@ -396,11 +401,11 @@ def get_image_ancestry(namespace, repository, image_id, headers):
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
image_id=image_id)
profile.debug('Converting to <-> from JSON')
logger.debug('Converting to <-> from JSON')
response = make_response(json.dumps(json.loads(data)), 200)
response.headers.extend(headers)
profile.debug('Done')
logger.debug('Done')
return response
@ -430,12 +435,12 @@ def store_checksum(image_storage, checksum):
@process_auth
@extract_namespace_repo_from_session
def put_image_json(namespace, repository, image_id):
profile.debug('Checking repo permissions')
logger.debug('Checking repo permissions')
permission = ModifyRepositoryPermission(namespace, repository)
if not permission.can():
abort(403)
profile.debug('Parsing image JSON')
logger.debug('Parsing image JSON')
try:
data = json.loads(request.data.decode('utf8'))
except ValueError:
@ -449,12 +454,22 @@ def put_image_json(namespace, repository, image_id):
abort(400, 'Missing key `id` in JSON for image: %(image_id)s',
issue='invalid-request', image_id=image_id)
profile.debug('Looking up repo image')
logger.debug('Looking up repo image')
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
if not repo_image:
profile.debug('Image not found')
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
image_id=image_id)
logger.debug('Image not found, creating image')
repo = model.get_repository(namespace, repository)
if repo is None:
abort(404, 'Repository does not exist: %(namespace)s/%(repository)s', issue='no-repo',
namespace=namespace, repository=repository)
username = get_authenticated_user() and get_authenticated_user().username
repo_image = model.find_create_or_link_image(image_id, repo, username, {},
store.preferred_locations[0])
# Create a temporary tag to prevent this image from getting garbage collected while the push
# is in progress.
model.create_temporary_hidden_tag(repo, repo_image, app.config['PUSH_TEMP_TAG_EXPIRATION_SEC'])
uuid = repo_image.storage.uuid
@ -466,24 +481,24 @@ def put_image_json(namespace, repository, image_id):
parent_image = None
if parent_id:
profile.debug('Looking up parent image')
logger.debug('Looking up parent image')
parent_image = model.get_repo_image_extended(namespace, repository, parent_id)
parent_uuid = parent_image and parent_image.storage.uuid
parent_locations = parent_image and parent_image.storage.locations
if parent_id:
profile.debug('Looking up parent image data')
logger.debug('Looking up parent image data')
if (parent_id and not
store.exists(parent_locations, store.image_json_path(parent_uuid))):
abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s',
issue='invalid-request', image_id=image_id, parent_id=parent_id)
profile.debug('Looking up image storage paths')
logger.debug('Looking up image storage paths')
json_path = store.image_json_path(uuid)
profile.debug('Checking if image already exists')
logger.debug('Checking if image already exists')
if (store.exists(repo_image.storage.locations, json_path) and not
image_is_uploading(repo_image)):
exact_abort(409, 'Image already exists')
@ -496,24 +511,24 @@ def put_image_json(namespace, repository, image_id):
command_list = data.get('container_config', {}).get('Cmd', None)
command = json.dumps(command_list) if command_list else None
profile.debug('Setting image metadata')
logger.debug('Setting image metadata')
model.set_image_metadata(image_id, namespace, repository,
data.get('created'), data.get('comment'), command,
parent_image)
profile.debug('Putting json path')
logger.debug('Putting json path')
store.put_content(repo_image.storage.locations, json_path, request.data)
profile.debug('Generating image ancestry')
logger.debug('Generating image ancestry')
try:
generate_ancestry(image_id, uuid, repo_image.storage.locations, parent_id, parent_uuid,
parent_locations)
except IOError as ioe:
profile.debug('Error when generating ancestry: %s' % ioe.message)
logger.debug('Error when generating ancestry: %s' % ioe.message)
abort(404)
profile.debug('Done')
logger.debug('Done')
return make_response('true', 200)

48
endpoints/trackhelper.py
View file

@ -6,7 +6,6 @@ from flask import request
from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token
logger = logging.getLogger(__name__)
profile = logging.getLogger('application.profiler')
def track_and_log(event_name, repo, **kwargs):
repository = repo.name
@ -19,20 +18,27 @@ def track_and_log(event_name, repo, **kwargs):
analytics_id = 'anonymous'
profile.debug('Logging the %s to Mixpanel and the log system', event_name)
if get_validated_oauth_token():
oauth_token = get_validated_oauth_token()
metadata['oauth_token_id'] = oauth_token.id
metadata['oauth_token_application_id'] = oauth_token.application.client_id
metadata['oauth_token_application'] = oauth_token.application.name
analytics_id = 'oauth:' + oauth_token.id
elif get_authenticated_user():
metadata['username'] = get_authenticated_user().username
analytics_id = get_authenticated_user().username
elif get_validated_token():
metadata['token'] = get_validated_token().friendly_name
metadata['token_code'] = get_validated_token().code
analytics_id = 'token:' + get_validated_token().code
authenticated_oauth_token = get_validated_oauth_token()
authenticated_user = get_authenticated_user()
authenticated_token = get_validated_token() if not authenticated_user else None
logger.debug('Logging the %s to Mixpanel and the log system', event_name)
if authenticated_oauth_token:
metadata['oauth_token_id'] = authenticated_oauth_token.id
metadata['oauth_token_application_id'] = authenticated_oauth_token.application.client_id
metadata['oauth_token_application'] = authenticated_oauth_token.application.name
analytics_id = 'oauth:' + authenticated_oauth_token.id
elif authenticated_user:
metadata['username'] = authenticated_user.username
analytics_id = authenticated_user.username
elif authenticated_token:
metadata['token'] = authenticated_token.friendly_name
metadata['token_code'] = authenticated_token.code
if authenticated_token.kind:
metadata['token_type'] = authenticated_token.kind.name
analytics_id = 'token:' + authenticated_token.code
else:
metadata['public'] = True
analytics_id = 'anonymous'
@ -42,21 +48,27 @@ def track_and_log(event_name, repo, **kwargs):
}
# Publish the user event (if applicable)
if get_authenticated_user():
logger.debug('Checking publishing %s to the user events system', event_name)
if authenticated_user:
logger.debug('Publishing %s to the user events system', event_name)
user_event_data = {
'action': event_name,
'repository': repository,
'namespace': namespace
}
event = userevents.get_event(get_authenticated_user().username)
event = userevents.get_event(authenticated_user.username)
event.publish_event_data('docker-cli', user_event_data)
# Save the action to mixpanel.
logger.debug('Logging the %s to Mixpanel', event_name)
analytics.track(analytics_id, event_name, extra_params)
# Log the action to the database.
logger.debug('Logging the %s to logs system', event_name)
model.log_action(event_name, namespace,
performer=get_authenticated_user(),
performer=authenticated_user,
ip=request.remote_addr, metadata=metadata,
repository=repo)
logger.debug('Track and log of %s complete', event_name)

13
endpoints/trigger.py
View file

@ -226,7 +226,7 @@ class GithubBuildTrigger(BuildTrigger):
'personal': False,
'repos': repo_list,
'info': {
'name': org.name,
'name': org.name or org.login,
'avatar_url': org.avatar_url
}
})
@ -345,8 +345,10 @@ class GithubBuildTrigger(BuildTrigger):
# compute the tag(s)
branch = ref.split('/')[-1]
tags = {branch}
if branch == repo.default_branch:
tags.add('latest')
logger.debug('Pushing to tags: %s' % tags)
# compute the subdir
@ -354,7 +356,14 @@ class GithubBuildTrigger(BuildTrigger):
joined_subdir = os.path.join(tarball_subdir, repo_subdir)
logger.debug('Final subdir: %s' % joined_subdir)
return dockerfile_id, list(tags), build_name, joined_subdir
# compute the metadata
metadata = {
'commit_sha': commit_sha,
'ref': ref,
'default_branch': repo.default_branch
}
return dockerfile_id, list(tags), build_name, joined_subdir, metadata
@staticmethod
def get_display_name(sha):

268
endpoints/verbs.py
View file

@ -2,11 +2,10 @@ import logging
import json
import hashlib
from flask import redirect, Blueprint, abort, send_file, request
from flask import redirect, Blueprint, abort, send_file, make_response
from app import app
from app import app, signer
from auth.auth import process_auth
from auth.auth_context import get_authenticated_user
from auth.permissions import ReadRepositoryPermission
from data import model
from data import database
@ -15,13 +14,16 @@ from storage import Storage
from util.queuefile import QueueFile
from util.queueprocess import QueueProcess
from util.gzipwrap import GzipWrap
from util.dockerloadformat import build_docker_load_stream
from formats.squashed import SquashedDockerImage
from formats.aci import ACIImage
# pylint: disable=invalid-name
verbs = Blueprint('verbs', __name__)
logger = logging.getLogger(__name__)
def _open_stream(namespace, repository, tag, synthetic_image_id, image_json, image_id_list):
def _open_stream(formatter, namespace, repository, tag, synthetic_image_id, image_json,
image_id_list):
store = Storage(app)
# For performance reasons, we load the full image list here, cache it, then disconnect from
@ -42,20 +44,43 @@ def _open_stream(namespace, repository, tag, synthetic_image_id, image_json, ima
current_image_path)
current_image_id = current_image_entry.id
logger.debug('Returning image layer %s: %s' % (current_image_id, current_image_path))
logger.debug('Returning image layer %s: %s', current_image_id, current_image_path)
yield current_image_stream
stream = build_docker_load_stream(namespace, repository, tag, synthetic_image_id, image_json,
stream = formatter.build_stream(namespace, repository, tag, synthetic_image_id, image_json,
get_next_image, get_next_layer)
return stream.read
def _write_synthetic_image_to_storage(linked_storage_uuid, linked_locations, queue_file):
def _sign_sythentic_image(verb, linked_storage_uuid, queue_file):
signature = None
try:
signature = signer.detached_sign(queue_file)
except:
logger.exception('Exception when signing %s image %s', verb, linked_storage_uuid)
return
# Setup the database (since this is a new process) and then disconnect immediately
# once the operation completes.
if not queue_file.raised_exception:
with database.UseThenDisconnect(app.config):
try:
derived = model.get_storage_by_uuid(linked_storage_uuid)
except model.InvalidImageException:
return
signature_entry = model.find_or_create_storage_signature(derived, signer.name)
signature_entry.signature = signature
signature_entry.uploading = False
signature_entry.save()
def _write_synthetic_image_to_storage(verb, linked_storage_uuid, linked_locations, queue_file):
store = Storage(app)
def handle_exception(ex):
logger.debug('Exception when building squashed image %s: %s', linked_storage_uuid, ex)
logger.debug('Exception when building %s image %s: %s', verb, linked_storage_uuid, ex)
with database.UseThenDisconnect(app.config):
model.delete_derived_storage_by_uuid(linked_storage_uuid)
@ -67,86 +92,193 @@ def _write_synthetic_image_to_storage(linked_storage_uuid, linked_locations, que
queue_file.close()
if not queue_file.raised_exception:
# Setup the database (since this is a new process) and then disconnect immediately
# once the operation completes.
with database.UseThenDisconnect(app.config):
done_uploading = model.get_storage_by_uuid(linked_storage_uuid)
done_uploading.uploading = False
done_uploading.save()
@verbs.route('/squash/<namespace>/<repository>/<tag>', methods=['GET'])
@process_auth
def get_squashed_tag(namespace, repository, tag):
# pylint: disable=too-many-locals
def _verify_repo_verb(store, namespace, repository, tag, verb, checker=None):
permission = ReadRepositoryPermission(namespace, repository)
if permission.can() or model.repository_is_public(namespace, repository):
# Lookup the requested tag.
try:
tag_image = model.get_tag_image(namespace, repository, tag)
except model.DataModelException:
abort(404)
# Lookup the tag's image and storage.
repo_image = model.get_repo_image_extended(namespace, repository, tag_image.docker_image_id)
if not repo_image:
abort(404)
# pylint: disable=no-member
if not permission.can() and not model.repository_is_public(namespace, repository):
abort(403)
# Log the action.
track_and_log('repo_verb', repo_image.repository, tag=tag, verb='squash')
# Lookup the requested tag.
try:
tag_image = model.get_tag_image(namespace, repository, tag)
except model.DataModelException:
abort(404)
store = Storage(app)
derived = model.find_or_create_derived_storage(repo_image.storage, 'squash',
store.preferred_locations[0])
if not derived.uploading:
logger.debug('Derived image %s exists in storage', derived.uuid)
derived_layer_path = store.image_layer_path(derived.uuid)
download_url = store.get_direct_download_url(derived.locations, derived_layer_path)
if download_url:
logger.debug('Redirecting to download URL for derived image %s', derived.uuid)
return redirect(download_url)
# Lookup the tag's image and storage.
repo_image = model.get_repo_image_extended(namespace, repository, tag_image.docker_image_id)
if not repo_image:
abort(404)
# Close the database handle here for this process before we send the long download.
database.close_db_filter(None)
# If there is a data checker, call it first.
uuid = repo_image.storage.uuid
image_json = None
logger.debug('Sending cached derived image %s', derived.uuid)
return send_file(store.stream_read_file(derived.locations, derived_layer_path))
# Load the ancestry for the image.
logger.debug('Building and returning derived image %s', derived.uuid)
uuid = repo_image.storage.uuid
ancestry_data = store.get_content(repo_image.storage.locations, store.image_ancestry_path(uuid))
full_image_list = json.loads(ancestry_data)
# Load the image's JSON layer.
if checker is not None:
image_json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
image_json = json.loads(image_json_data)
# Calculate a synthetic image ID.
synthetic_image_id = hashlib.sha256(tag_image.docker_image_id + ':squash').hexdigest()
if not checker(image_json):
logger.debug('Check mismatch on %s/%s:%s, verb %s', namespace, repository, tag, verb)
abort(404)
# Create a queue process to generate the data. The queue files will read from the process
# and send the results to the client and storage.
def _cleanup():
# Close any existing DB connection once the process has exited.
database.close_db_filter(None)
return (repo_image, tag_image, image_json)
args = (namespace, repository, tag, synthetic_image_id, image_json, full_image_list)
queue_process = QueueProcess(_open_stream,
8 * 1024, 10 * 1024 * 1024, # 8K/10M chunk/max
args, finished=_cleanup)
client_queue_file = QueueFile(queue_process.create_queue(), 'client')
storage_queue_file = QueueFile(queue_process.create_queue(), 'storage')
# pylint: disable=too-many-locals
def _repo_verb_signature(namespace, repository, tag, verb, checker=None, **kwargs):
# Verify that the image exists and that we have access to it.
store = Storage(app)
result = _verify_repo_verb(store, namespace, repository, tag, verb, checker)
(repo_image, tag_image, image_json) = result
# Start building.
queue_process.run()
# Lookup the derived image storage for the verb.
derived = model.find_derived_storage(repo_image.storage, verb)
if derived is None or derived.uploading:
abort(404)
# Start the storage saving.
storage_args = (derived.uuid, derived.locations, storage_queue_file)
QueueProcess.run_process(_write_synthetic_image_to_storage, storage_args, finished=_cleanup)
# Check if we have a valid signer configured.
if not signer.name:
abort(404)
# Lookup the signature for the verb.
signature_entry = model.lookup_storage_signature(derived, signer.name)
if signature_entry is None:
abort(404)
# Return the signature.
return make_response(signature_entry.signature)
# pylint: disable=too-many-locals
def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=None, **kwargs):
# Verify that the image exists and that we have access to it.
store = Storage(app)
result = _verify_repo_verb(store, namespace, repository, tag, verb, checker)
(repo_image, tag_image, image_json) = result
# Log the action.
track_and_log('repo_verb', repo_image.repository, tag=tag, verb=verb, **kwargs)
# Lookup/create the derived image storage for the verb.
derived = model.find_or_create_derived_storage(repo_image.storage, verb,
store.preferred_locations[0])
if not derived.uploading:
logger.debug('Derived %s image %s exists in storage', verb, derived.uuid)
derived_layer_path = store.image_layer_path(derived.uuid)
download_url = store.get_direct_download_url(derived.locations, derived_layer_path)
if download_url:
logger.debug('Redirecting to download URL for derived %s image %s', verb, derived.uuid)
return redirect(download_url)
# Close the database handle here for this process before we send the long download.
database.close_db_filter(None)
# Return the client's data.
return send_file(client_queue_file)
logger.debug('Sending cached derived %s image %s', verb, derived.uuid)
return send_file(store.stream_read_file(derived.locations, derived_layer_path))
# Load the ancestry for the image.
uuid = repo_image.storage.uuid
logger.debug('Building and returning derived %s image %s', verb, derived.uuid)
ancestry_data = store.get_content(repo_image.storage.locations, store.image_ancestry_path(uuid))
full_image_list = json.loads(ancestry_data)
# Load the image's JSON layer.
if not image_json:
image_json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
image_json = json.loads(image_json_data)
# Calculate a synthetic image ID.
synthetic_image_id = hashlib.sha256(tag_image.docker_image_id + ':' + verb).hexdigest()
def _cleanup():
# Close any existing DB connection once the process has exited.
database.close_db_filter(None)
# Create a queue process to generate the data. The queue files will read from the process
# and send the results to the client and storage.
args = (formatter, namespace, repository, tag, synthetic_image_id, image_json, full_image_list)
queue_process = QueueProcess(_open_stream,
8 * 1024, 10 * 1024 * 1024, # 8K/10M chunk/max
args, finished=_cleanup)
client_queue_file = QueueFile(queue_process.create_queue(), 'client')
storage_queue_file = QueueFile(queue_process.create_queue(), 'storage')
# If signing is required, add a QueueFile for signing the image as we stream it out.
signing_queue_file = None
if sign and signer.name:
signing_queue_file = QueueFile(queue_process.create_queue(), 'signing')
# Start building.
queue_process.run()
# Start the storage saving.
storage_args = (verb, derived.uuid, derived.locations, storage_queue_file)
QueueProcess.run_process(_write_synthetic_image_to_storage, storage_args, finished=_cleanup)
if sign and signer.name:
signing_args = (verb, derived.uuid, signing_queue_file)
QueueProcess.run_process(_sign_sythentic_image, signing_args, finished=_cleanup)
# Close the database handle here for this process before we send the long download.
database.close_db_filter(None)
# Return the client's data.
return send_file(client_queue_file)
def os_arch_checker(os, arch):
def checker(image_json):
# Verify the architecture and os.
operating_system = image_json.get('os', 'linux')
if operating_system != os:
return False
architecture = image_json.get('architecture', 'amd64')
# Note: Some older Docker images have 'x86_64' rather than 'amd64'.
# We allow the conversion here.
if architecture == 'x86_64' and operating_system == 'linux':
architecture = 'amd64'
if architecture != arch:
return False
return True
return checker
@verbs.route('/aci/<server>/<namespace>/<repository>/<tag>/sig/<os>/<arch>/', methods=['GET'])
@process_auth
# pylint: disable=unused-argument
def get_aci_signature(server, namespace, repository, tag, os, arch):
return _repo_verb_signature(namespace, repository, tag, 'aci', checker=os_arch_checker(os, arch),
os=os, arch=arch)
@verbs.route('/aci/<server>/<namespace>/<repository>/<tag>/aci/<os>/<arch>/', methods=['GET'])
@process_auth
# pylint: disable=unused-argument
def get_aci_image(server, namespace, repository, tag, os, arch):
return _repo_verb(namespace, repository, tag, 'aci', ACIImage(),
sign=True, checker=os_arch_checker(os, arch), os=os, arch=arch)
@verbs.route('/squash/<namespace>/<repository>/<tag>', methods=['GET'])
@process_auth
def get_squashed_tag(namespace, repository, tag):
return _repo_verb(namespace, repository, tag, 'squash', SquashedDockerImage())
abort(403)

92
endpoints/web.py
View file

@ -1,32 +1,38 @@
import logging
from flask import (abort, redirect, request, url_for, make_response, Response,
Blueprint, send_from_directory, jsonify)
Blueprint, send_from_directory, jsonify, send_file)
from avatar_generator import Avatar
from flask.ext.login import current_user
from urlparse import urlparse
from health.healthcheck import HealthCheck
from health.healthcheck import get_healthchecker
from data import model
from data.model.oauth import DatabaseAuthorizationProvider
from app import app, billing as stripe, build_logs, avatar
from app import app, billing as stripe, build_logs, avatar, signer
from auth.auth import require_session_login, process_oauth
from auth.permissions import AdministerOrganizationPermission, ReadRepositoryPermission
from auth.permissions import (AdministerOrganizationPermission, ReadRepositoryPermission,
SuperUserPermission)
from util.invoice import renderInvoiceToPdf
from util.seo import render_snapshot
from util.cache import no_cache
from endpoints.common import common_login, render_page_template, route_show_if, param_required
from endpoints.csrf import csrf_protect, generate_csrf_token
from endpoints.csrf import csrf_protect, generate_csrf_token, verify_csrf
from endpoints.registry import set_cache_headers
from util.names import parse_repository_name
from util.names import parse_repository_name, parse_repository_name_and_tag
from util.useremails import send_email_changed
from util.systemlogs import build_logs_archive
from auth import scopes
import features
logger = logging.getLogger(__name__)
# Capture the unverified SSL errors.
logging.captureWarnings(True)
web = Blueprint('web', __name__)
STATUS_TAGS = app.config['STATUS_TAGS']
@ -57,6 +63,14 @@ def snapshot(path = ''):
abort(404)
@web.route('/aci-signing-key')
@no_cache
def aci_signing_key():
if not signer.name:
abort(404)
return send_file(signer.public_key_path)
@web.route('/plans/')
@no_cache
@route_show_if(features.BILLING)
@ -95,6 +109,7 @@ def organizations():
def user():
return index('')
@web.route('/superuser/')
@no_cache
@route_show_if(features.SUPER_USERS)
@ -102,6 +117,13 @@ def superuser():
return index('')
@web.route('/setup/')
@no_cache
@route_show_if(features.SUPER_USERS)
def setup():
return index('')
@web.route('/signin/')
@no_cache
def signin(redirect=None):
@ -158,33 +180,27 @@ def v1():
return index('')
# TODO(jschorr): Remove this mirrored endpoint once we migrate ELB.
@web.route('/health', methods=['GET'])
@web.route('/health/instance', methods=['GET'])
@no_cache
def health():
db_healthy = model.check_health()
buildlogs_healthy = build_logs.check_health()
check = HealthCheck.get_check(app.config['HEALTH_CHECKER'][0], app.config['HEALTH_CHECKER'][1])
(data, is_healthy) = check.conduct_healthcheck(db_healthy, buildlogs_healthy)
response = jsonify(dict(data = data, is_healthy = is_healthy))
response.status_code = 200 if is_healthy else 503
def instance_health():
checker = get_healthchecker(app)
(data, status_code) = checker.check_instance()
response = jsonify(dict(data=data, status_code=status_code))
response.status_code = status_code
return response
# TODO(jschorr): Remove this mirrored endpoint once we migrate pingdom.
@web.route('/status', methods=['GET'])
@web.route('/health/endtoend', methods=['GET'])
@no_cache
def status():
db_healthy = model.check_health()
buildlogs_healthy = build_logs.check_health()
response = jsonify({
'db_healthy': db_healthy,
'buildlogs_healthy': buildlogs_healthy,
'is_testing': app.config['TESTING'],
})
response.status_code = 200 if db_healthy and buildlogs_healthy else 503
def endtoend_health():
checker = get_healthchecker(app)
(data, status_code) = checker.check_endtoend()
response = jsonify(dict(data=data, status_code=status_code))
response.status_code = status_code
return response
@ -229,14 +245,14 @@ def robots():
@web.route('/<path:repository>')
@no_cache
@process_oauth
@parse_repository_name
def redirect_to_repository(namespace, reponame):
@parse_repository_name_and_tag
def redirect_to_repository(namespace, reponame, tag):
permission = ReadRepositoryPermission(namespace, reponame)
is_public = model.repository_is_public(namespace, reponame)
if permission.can() or is_public:
repository_name = '/'.join([namespace, reponame])
return redirect(url_for('web.repository', path=repository_name))
return redirect(url_for('web.repository', path=repository_name, tag=tag))
abort(404)
@ -471,3 +487,21 @@ def exchange_code_for_token():
provider = FlaskAuthorizationProvider()
return provider.get_token(grant_type, client_id, client_secret, redirect_uri, code, scope=scope)
@web.route('/systemlogsarchive', methods=['GET'])
@process_oauth
@route_show_if(features.SUPER_USERS)
@no_cache
def download_logs_archive():
# Note: We cannot use the decorator here because this is a GET method. That being said, this
# information is sensitive enough that we want the extra protection.
verify_csrf()
if SuperUserPermission().can():
archive_data = build_logs_archive(app)
return Response(archive_data,
mimetype="application/octet-stream",
headers={"Content-Disposition": "attachment;filename=erlogs.tar.gz"})
abort(403)

4
endpoints/webhooks.py
View file

@ -91,7 +91,7 @@ def build_trigger_webhook(trigger_uuid, **kwargs):
try:
specs = handler.handle_trigger_request(request, trigger.auth_token,
config_dict)
dockerfile_id, tags, name, subdir = specs
dockerfile_id, tags, name, subdir, metadata = specs
except ValidationRequestException:
# This was just a validation request, we don't need to build anything
@ -104,7 +104,7 @@ def build_trigger_webhook(trigger_uuid, **kwargs):
pull_robot_name = model.get_pull_robot_name(trigger)
repo = model.get_repository(namespace, repository)
start_build(repo, dockerfile_id, tags, name, subdir, False, trigger,
pull_robot_name=pull_robot_name)
pull_robot_name=pull_robot_name, trigger_metadata=metadata)
return make_response('Okay')

14
external_libraries.py
View file

@ -6,7 +6,7 @@ LOCAL_DIRECTORY = 'static/ldn/'
EXTERNAL_JS = [
'code.jquery.com/jquery.js',
'netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js',
'netdna.bootstrapcdn.com/bootstrap/3.3.2/js/bootstrap.min.js',
'ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular.min.js',
'ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular-route.min.js',
'ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular-sanitize.min.js',
@ -18,15 +18,15 @@ EXTERNAL_JS = [
]
EXTERNAL_CSS = [
'netdna.bootstrapcdn.com/font-awesome/4.1.0/css/font-awesome.css',
'netdna.bootstrapcdn.com/bootstrap/3.0.0/css/bootstrap.no-icons.min.css',
'fonts.googleapis.com/css?family=Droid+Sans:400,700',
'netdna.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.css',
'netdna.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap.min.css',
'fonts.googleapis.com/css?family=Source+Sans+Pro:400,700',
]
EXTERNAL_FONTS = [
'netdna.bootstrapcdn.com/font-awesome/4.0.3/fonts/fontawesome-webfont.woff?v=4.0.3',
'netdna.bootstrapcdn.com/font-awesome/4.0.3/fonts/fontawesome-webfont.ttf?v=4.0.3',
'netdna.bootstrapcdn.com/font-awesome/4.0.3/fonts/fontawesome-webfont.svg?v=4.0.3',
'netdna.bootstrapcdn.com/font-awesome/4.2.0/fonts/fontawesome-webfont.woff?v=4.2.0',
'netdna.bootstrapcdn.com/font-awesome/4.2.0/fonts/fontawesome-webfont.ttf?v=4.2.0',
'netdna.bootstrapcdn.com/font-awesome/4.2.0/fonts/fontawesome-webfont.svg?v=4.2.0',
]

0
formats/__init__.py Normal file
View file

196
formats/aci.py Normal file
View file

@ -0,0 +1,196 @@
from app import app
from util.streamlayerformat import StreamLayerMerger
from formats.tarimageformatter import TarImageFormatter
import json
import re
# pylint: disable=bad-continuation
class ACIImage(TarImageFormatter):
""" Image formatter which produces an ACI-compatible TAR.
"""
# pylint: disable=too-many-arguments
def stream_generator(self, namespace, repository, tag, synthetic_image_id,
layer_json, get_image_iterator, get_layer_iterator):
# ACI Format (.tar):
# manifest - The JSON manifest
# rootfs - The root file system
# Yield the manifest.
yield self.tar_file('manifest', self._build_manifest(namespace, repository, tag, layer_json,
synthetic_image_id))
# Yield the merged layer dtaa.
yield self.tar_folder('rootfs')
layer_merger = StreamLayerMerger(get_layer_iterator, path_prefix='rootfs/')
for entry in layer_merger.get_generator():
yield entry
@staticmethod
def _build_isolators(docker_config):
""" Builds ACI isolator config from the docker config. """
def _isolate_memory(memory):
return {
"name": "memory/limit",
"value": str(memory) + 'B'
}
def _isolate_swap(memory):
return {
"name": "memory/swap",
"value": str(memory) + 'B'
}
def _isolate_cpu(cpu):
return {
"name": "cpu/shares",
"value": str(cpu)
}
def _isolate_capabilities(capabilities_set_value):
capabilities_set = re.split(r'[\s,]', capabilities_set_value)
return {
"name": "capabilities/bounding-set",
"value": ' '.join(capabilities_set)
}
mappers = {
'Memory': _isolate_memory,
'MemorySwap': _isolate_swap,
'CpuShares': _isolate_cpu,
'Cpuset': _isolate_capabilities
}
isolators = []
for config_key in mappers:
value = docker_config.get(config_key)
if value:
isolators.append(mappers[config_key](value))
return isolators
@staticmethod
def _build_ports(docker_config):
""" Builds the ports definitions for the ACI. """
ports = []
for docker_port_definition in docker_config.get('ports', {}):
# Formats:
# port/tcp
# port/udp
# port
protocol = 'tcp'
port_number = -1
if '/' in docker_port_definition:
(port_number, protocol) = docker_port_definition.split('/')
else:
port_number = docker_port_definition
try:
port_number = int(port_number)
ports.append({
"name": "port-%s" % port_number,
"port": port_number,
"protocol": protocol
})
except ValueError:
pass
return ports
@staticmethod
def _build_volumes(docker_config):
""" Builds the volumes definitions for the ACI. """
volumes = []
names = set()
def get_name(docker_volume_path):
parts = docker_volume_path.split('/')
name = ''
while True:
name = name + parts[-1]
parts = parts[0:-1]
if names.add(name):
break
name = '/' + name
return name
for docker_volume_path in docker_config.get('volumes', {}):
volumes.append({
"name": get_name(docker_volume_path),
"path": docker_volume_path,
"readOnly": False
})
return volumes
@staticmethod
def _build_manifest(namespace, repository, tag, docker_layer_data, synthetic_image_id):
""" Builds an ACI manifest from the docker layer data. """
config = docker_layer_data.get('config', {})
source_url = "%s://%s/%s/%s:%s" % (app.config['PREFERRED_URL_SCHEME'],
app.config['SERVER_HOSTNAME'],
namespace, repository, tag)
# ACI requires that the execution command be absolutely referenced. Therefore, if we find
# a relative command, we give it as an argument to /bin/sh to resolve and execute for us.
entrypoint = config.get('Entrypoint', []) or []
exec_path = entrypoint + (config.get('Cmd', []) or [])
if exec_path and not exec_path[0].startswith('/'):
exec_path = ['/bin/sh', '-c', '""%s""' % ' '.join(exec_path)]
# TODO(jschorr): ACI doesn't support : in the name, so remove any ports.
hostname = app.config['SERVER_HOSTNAME']
hostname = hostname.split(':', 1)[0]
manifest = {
"acKind": "ImageManifest",
"acVersion": "0.2.0",
"name": '%s/%s/%s/%s' % (hostname, namespace, repository, tag),
"labels": [
{
"name": "version",
"value": "1.0.0"
},
{
"name": "arch",
"value": docker_layer_data.get('architecture', 'amd64')
},
{
"name": "os",
"value": docker_layer_data.get('os', 'linux')
}
],
"app": {
"exec": exec_path,
# Below, `or 'root'` is required to replace empty string from Dockerfiles.
"user": config.get('User', '') or 'root',
"group": config.get('Group', '') or 'root',
"eventHandlers": [],
"workingDirectory": config.get('WorkingDir', '') or '/',
"environment": [{"name": key, "value": value}
for (key, value) in [e.split('=') for e in config.get('Env')]],
"isolators": ACIImage._build_isolators(config),
"mountPoints": ACIImage._build_volumes(config),
"ports": ACIImage._build_ports(config),
"annotations": [
{"name": "created", "value": docker_layer_data.get('created', '')},
{"name": "homepage", "value": source_url},
{"name": "quay.io/derived-image", "value": synthetic_image_id},
]
},
}
return json.dumps(manifest)

102
formats/squashed.py Normal file
View file

@ -0,0 +1,102 @@
from app import app
from util.gzipwrap import GZIP_BUFFER_SIZE
from util.streamlayerformat import StreamLayerMerger
from formats.tarimageformatter import TarImageFormatter
import copy
import json
class FileEstimationException(Exception):
""" Exception raised by build_docker_load_stream if the estimated size of the layer TAR
was lower than the actual size. This means the sent TAR header is wrong, and we have
to fail.
"""
pass
class SquashedDockerImage(TarImageFormatter):
""" Image formatter which produces a squashed image compatible with the `docker load`
command.
"""
# pylint: disable=too-many-arguments,too-many-locals
def stream_generator(self, namespace, repository, tag, synthetic_image_id,
layer_json, get_image_iterator, get_layer_iterator):
# Docker import V1 Format (.tar):
# repositories - JSON file containing a repo -> tag -> image map
# {image ID folder}:
# json - The layer JSON
# layer.tar - The TARed contents of the layer
# VERSION - The docker import version: '1.0'
layer_merger = StreamLayerMerger(get_layer_iterator)
# Yield the repositories file:
synthetic_layer_info = {}
synthetic_layer_info[tag + '.squash'] = synthetic_image_id
hostname = app.config['SERVER_HOSTNAME']
repositories = {}
repositories[hostname + '/' + namespace + '/' + repository] = synthetic_layer_info
yield self.tar_file('repositories', json.dumps(repositories))
# Yield the image ID folder.
yield self.tar_folder(synthetic_image_id)
# Yield the JSON layer data.
layer_json = SquashedDockerImage._build_layer_json(layer_json, synthetic_image_id)
yield self.tar_file(synthetic_image_id + '/json', json.dumps(layer_json))
# Yield the VERSION file.
yield self.tar_file(synthetic_image_id + '/VERSION', '1.0')
# Yield the merged layer data's header.
estimated_file_size = 0
for image in get_image_iterator():
estimated_file_size += image.storage.uncompressed_size
yield self.tar_file_header(synthetic_image_id + '/layer.tar', estimated_file_size)
# Yield the contents of the merged layer.
yielded_size = 0
for entry in layer_merger.get_generator():
yield entry
yielded_size += len(entry)
# If the yielded size is more than the estimated size (which is unlikely but possible), then
# raise an exception since the tar header will be wrong.
if yielded_size > estimated_file_size:
raise FileEstimationException()
# If the yielded size is less than the estimated size (which is likely), fill the rest with
# zeros.
if yielded_size < estimated_file_size:
to_yield = estimated_file_size - yielded_size
while to_yield > 0:
yielded = min(to_yield, GZIP_BUFFER_SIZE)
yield '\0' * yielded
to_yield -= yielded
# Yield any file padding to 512 bytes that is necessary.
yield self.tar_file_padding(estimated_file_size)
# Last two records are empty in TAR spec.
yield '\0' * 512
yield '\0' * 512
@staticmethod
def _build_layer_json(layer_json, synthetic_image_id):
updated_json = copy.deepcopy(layer_json)
updated_json['id'] = synthetic_image_id
if 'parent' in updated_json:
del updated_json['parent']
if 'config' in updated_json and 'Image' in updated_json['config']:
updated_json['config']['Image'] = synthetic_image_id
if 'container_config' in updated_json and 'Image' in updated_json['container_config']:
updated_json['container_config']['Image'] = synthetic_image_id
return updated_json

46
formats/tarimageformatter.py Normal file
View file

@ -0,0 +1,46 @@
import tarfile
from util.gzipwrap import GzipWrap
class TarImageFormatter(object):
""" Base class for classes which produce a TAR containing image and layer data. """
def build_stream(self, namespace, repository, tag, synthetic_image_id, layer_json,
get_image_iterator, get_layer_iterator):
""" Builds and streams a synthetic .tar.gz that represents the formatted TAR created by this
class's implementation.
"""
return GzipWrap(self.stream_generator(namespace, repository, tag,
synthetic_image_id, layer_json,
get_image_iterator, get_layer_iterator))
def stream_generator(self, namespace, repository, tag, synthetic_image_id,
layer_json, get_image_iterator, get_layer_iterator):
raise NotImplementedError
def tar_file(self, name, contents):
""" Returns the TAR binary representation for a file with the given name and file contents. """
length = len(contents)
tar_data = self.tar_file_header(name, length)
tar_data += contents
tar_data += self.tar_file_padding(length)
return tar_data
def tar_file_padding(self, length):
""" Returns TAR file padding for file data of the given length. """
if length % 512 != 0:
return '\0' * (512 - (length % 512))
return ''
def tar_file_header(self, name, file_size):
""" Returns TAR file header data for a file with the given name and size. """
info = tarfile.TarInfo(name=name)
info.type = tarfile.REGTYPE
info.size = file_size
return info.tobuf()
def tar_folder(self, name):
""" Returns TAR file header data for a folder with the given name. """
info = tarfile.TarInfo(name=name)
info.type = tarfile.DIRTYPE
return info.tobuf()

18
grunt/Gruntfile.js
View file

@ -65,9 +65,22 @@ module.exports = function(grunt) {
}
},
quay: {
src: ['../static/partials/*.html', '../static/directives/*.html'],
src: ['../static/partials/*.html', '../static/directives/*.html', '../static/directives/*.html'
, '../static/directives/config/*.html'],
dest: '../static/dist/template-cache.js'
}
},
cachebuster: {
build: {
options: {
format: 'json',
basedir: '../static/'
},
src: [ '../static/dist/template-cache.js', '../static/dist/<%= pkg.name %>.min.js',
'../static/dist/<%= pkg.name %>.css' ],
dest: '../static/dist/cachebusters.json'
}
}
});
@ -75,7 +88,8 @@ module.exports = function(grunt) {
grunt.loadNpmTasks('grunt-contrib-concat');
grunt.loadNpmTasks('grunt-contrib-cssmin');
grunt.loadNpmTasks('grunt-angular-templates');
grunt.loadNpmTasks('grunt-cachebuster');
// Default task(s).
grunt.registerTask('default', ['ngtemplates', 'concat', 'cssmin', 'uglify']);
grunt.registerTask('default', ['ngtemplates', 'concat', 'cssmin', 'uglify', 'cachebuster']);
};

3
grunt/package.json
View file

@ -6,6 +6,7 @@
"grunt-contrib-concat": "~0.4.0",
"grunt-contrib-cssmin": "~0.9.0",
"grunt-angular-templates": "~0.5.4",
"grunt-contrib-uglify": "~0.4.0"
"grunt-contrib-uglify": "~0.4.0",
"grunt-cachebuster": "~0.1.5"
}
}

137
health/healthcheck.py
View file

@ -1,47 +1,84 @@
import boto.rds2
import logging
from health.services import check_all_services
logger = logging.getLogger(__name__)
class HealthCheck(object):
def __init__(self):
pass
def get_healthchecker(app):
""" Returns a HealthCheck instance for the given app. """
return HealthCheck.get_checker(app)
def conduct_healthcheck(self, db_healthy, buildlogs_healthy):
class HealthCheck(object):
def __init__(self, app):
self.app = app
def check_instance(self):
"""
Conducts any custom healthcheck work, returning a dict representing the HealthCheck
output and a boolean indicating whether the instance is healthy.
Conducts a check on this specific instance, returning a dict representing the HealthCheck
output and a number indicating the health check response code.
"""
raise NotImplementedError
service_statuses = check_all_services(self.app)
return self.get_instance_health(service_statuses)
def check_endtoend(self):
"""
Conducts a check on all services, returning a dict representing the HealthCheck
output and a number indicating the health check response code.
"""
service_statuses = check_all_services(self.app)
return self.calculate_overall_health(service_statuses)
def get_instance_health(self, service_statuses):
"""
For the given service statuses, returns a dict representing the HealthCheck
output and a number indicating the health check response code. By default,
this simply ensures that all services are reporting as healthy.
"""
return self.calculate_overall_health(service_statuses)
def calculate_overall_health(self, service_statuses, skip=None, notes=None):
""" Returns true if and only if all the given service statuses report as healthy. """
is_healthy = True
notes = notes or []
for service_name in service_statuses:
if skip and service_name in skip:
notes.append('%s skipped in compute health' % service_name)
continue
is_healthy = is_healthy and service_statuses[service_name]
data = {
'services': service_statuses,
'notes': notes,
'is_testing': self.app.config['TESTING']
}
return (data, 200 if is_healthy else 503)
@classmethod
def get_check(cls, name, parameters):
def get_checker(cls, app):
name = app.config['HEALTH_CHECKER'][0]
parameters = app.config['HEALTH_CHECKER'][1] or {}
for subc in cls.__subclasses__():
if subc.check_name() == name:
return subc(**parameters)
return subc(app, **parameters)
raise Exception('Unknown health check with name %s' % name)
class LocalHealthCheck(HealthCheck):
def __init__(self):
pass
@classmethod
def check_name(cls):
return 'LocalHealthCheck'
def conduct_healthcheck(self, db_healthy, buildlogs_healthy):
data = {
'db_healthy': db_healthy,
'buildlogs_healthy': buildlogs_healthy
}
return (data, db_healthy and buildlogs_healthy)
class ProductionHealthCheck(HealthCheck):
def __init__(self, access_key, secret_key):
def __init__(self, app, access_key, secret_key):
super(ProductionHealthCheck, self).__init__(app)
self.access_key = access_key
self.secret_key = secret_key
@ -49,36 +86,38 @@ class ProductionHealthCheck(HealthCheck):
def check_name(cls):
return 'ProductionHealthCheck'
def conduct_healthcheck(self, db_healthy, buildlogs_healthy):
data = {
'db_healthy': db_healthy,
'buildlogs_healthy': buildlogs_healthy
}
def get_instance_health(self, service_statuses):
# Note: We skip the redis check because if redis is down, we don't want ELB taking the
# machines out of service. Redis is not considered a high avaliability-required service.
skip = ['redis']
notes = []
# Only report unhealthy if the machine cannot connect to the DB. Redis isn't required for
# mission critical/high avaliability operations.
# If the database is marked as unhealthy, check the status of RDS directly. If RDS is
# reporting as available, then the problem is with this instance. Otherwise, the problem is
# with RDS, and so we skip the DB status so we can keep this machine as 'healthy'.
db_healthy = service_statuses['database']
if not db_healthy:
# If the database is marked as unhealthy, check the status of RDS directly. If RDS is
# reporting as available, then the problem is with this instance. Otherwise, the problem is
# with RDS, and we can keep this machine as 'healthy'.
is_rds_working = False
try:
region = boto.rds2.connect_to_region('us-east-1',
aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key)
response = region.describe_db_instances()['DescribeDBInstancesResponse']
result = response['DescribeDBInstancesResult']
instances = result['DBInstances']
status = instances[0]['DBInstanceStatus']
is_rds_working = status == 'available'
except:
logger.exception("Exception while checking RDS status")
pass
rds_status = self._get_rds_status()
notes.append('DB reports unhealthy; RDS status: %s' % rds_status)
data['db_available_checked'] = True
data['db_available_status'] = is_rds_working
# If the RDS is in any state but available, then we skip the DB check since it will
# fail and bring down the instance.
if rds_status != 'available':
skip.append('database')
# If RDS is down, then we still report the machine as healthy, so that it can handle
# requests once RDS comes back up.
return (data, not is_rds_working)
return self.calculate_overall_health(service_statuses, skip=skip, notes=notes)
return (data, db_healthy)
def _get_rds_status(self):
""" Returns the status of the RDS instance as reported by AWS. """
try:
region = boto.rds2.connect_to_region('us-east-1',
aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key)
response = region.describe_db_instances()['DescribeDBInstancesResponse']
result = response['DescribeDBInstancesResult']
instances = result['DBInstances']
status = instances[0]['DBInstanceStatus']
return status
except:
logger.exception("Exception while checking RDS status")
return 'error'

46
health/services.py Normal file
View file

@ -0,0 +1,46 @@
import logging
from data import model
from app import build_logs
logger = logging.getLogger(__name__)
def _check_registry_gunicorn(app):
""" Returns the status of the registry gunicorn workers. """
# Compute the URL for checking the registry endpoint. We append a port if and only if the
# hostname contains one.
client = app.config['HTTPCLIENT']
hostname_parts = app.config['SERVER_HOSTNAME'].split(':')
port = ''
if len(hostname_parts) == 2:
port = ':' + hostname_parts[1]
registry_url = '%s://localhost%s/v1/_internal_ping' % (app.config['PREFERRED_URL_SCHEME'], port)
try:
return client.get(registry_url, verify=False, timeout=2).status_code == 200
except Exception:
logger.exception('Exception when checking registry health: %s', registry_url)
return False
def _check_database(app):
""" Returns the status of the database, as accessed from this instance. """
return model.check_health(app.config)
def _check_redis(app):
""" Returns the status of Redis, as accessed from this instance. """
return build_logs.check_health()
_SERVICES = {
'registry_gunicorn': _check_registry_gunicorn,
'database': _check_database,
'redis': _check_redis
}
def check_all_services(app):
""" Returns a dictionary containing the status of all the services defined. """
status = {}
for name in _SERVICES:
status[name] = _SERVICES[name](app)
return status

8
initdb.py
View file

@ -192,6 +192,9 @@ def initialize_database():
BuildTriggerService.create(name='github')
AccessTokenKind.create(name='build-worker')
AccessTokenKind.create(name='pushpull-token')
LogEntryKind.create(name='account_change_plan')
LogEntryKind.create(name='account_change_cc')
LogEntryKind.create(name='account_change_password')
@ -255,6 +258,9 @@ def initialize_database():
ImageStorageLocation.create(name='local_us')
ImageStorageTransformation.create(name='squash')
ImageStorageTransformation.create(name='aci')
ImageStorageSignatureKind.create(name='gpg2')
# NOTE: These MUST be copied over to NotificationKind, since every external
# notification can also generate a Quay.io notification.
@ -390,7 +396,7 @@ def populate_database():
'Empty repository which is building.',
False, [], (0, [], None))
token = model.create_access_token(building, 'write')
token = model.create_access_token(building, 'write', 'build-worker')
trigger = model.create_build_trigger(building, 'github', '123authtoken',
new_user_1, pull_robot=dtrobot[0])

19
local-setup-osx.sh Executable file
View file

@ -0,0 +1,19 @@
#!/bin/bash
set -e
# Install Docker and C libraries on which Python libraries are dependent
brew update
brew install boot2docker docker libevent libmagic postgresql
# Some OSX installs don't have /usr/include, which is required for finding SASL headers for our LDAP library
if [ ! -e /usr/include ]; then
sudo ln -s `xcrun --show-sdk-path`/usr/include /usr/include
fi
# Install Python dependencies
sudo pip install -r requirements.txt
# Put the local testing config in place
git clone git@github.com:coreos-inc/quay-config.git ../quay-config
ln -s ../../quay-config/local conf/stack

2
local-test.sh
View file

@ -1 +1 @@
TEST=true python -m unittest discover
TEST=true TROLLIUSDEBUG=1 python -m unittest discover

1
registry.py
View file

@ -7,7 +7,6 @@ from endpoints.index import index
from endpoints.tags import tags
from endpoints.registry import registry
application.register_blueprint(index, url_prefix='/v1')
application.register_blueprint(tags, url_prefix='/v1')
application.register_blueprint(registry, url_prefix='/v1')

11
requirements-nover.txt
View file

@ -1,4 +1,4 @@
autobahn
autobahn==0.9.3-3
aiowsgi
trollius
peewee
@ -22,7 +22,6 @@ xhtml2pdf
redis
hiredis
docker-py
pygithub
flask-restful==0.2.12
jsonschema
git+https://github.com/NateFerrero/oauth2lib.git
@ -40,4 +39,12 @@ pyyaml
git+https://github.com/DevTable/aniso8601-fake.git
git+https://github.com/DevTable/anunidecode.git
git+https://github.com/DevTable/avatar-generator.git
git+https://github.com/DevTable/pygithub.git
git+https://github.com/DevTable/container-cloud-config.git
git+https://github.com/jplana/python-etcd.git
gipc
pyOpenSSL
pygpgme
cachetools
mock
psutil

55
requirements.txt
View file

@ -8,24 +8,22 @@ Jinja2==2.7.3
LogentriesLogger==0.2.1
Mako==1.0.0
MarkupSafe==0.23
Pillow==2.6.1
PyGithub==1.25.2
PyMySQL==0.6.2
PyPDF2==1.23
Pillow==2.7.0
PyMySQL==0.6.3
PyPDF2==1.24
PyYAML==3.11
SQLAlchemy==0.9.8
WebOb==1.4
Werkzeug==0.9.6
alembic==0.7.0
git+https://github.com/DevTable/aniso8601-fake.git
git+https://github.com/DevTable/anunidecode.git
git+https://github.com/DevTable/avatar-generator.git
aiowsgi==0.3
alembic==0.7.4
autobahn==0.9.3-3
backports.ssl-match-hostname==3.4.0.2
beautifulsoup4==4.3.2
blinker==1.3
boto==2.34.0
docker-py==0.6.0
boto==2.35.1
cachetools==1.0.0
docker-py==0.7.1
ecdsa==0.11
futures==2.2.0
gevent==1.0.1
@ -36,26 +34,37 @@ hiredis==0.1.5
html5lib==0.999
itsdangerous==0.24
jsonschema==2.4.0
marisa-trie==0.6
mixpanel-py==3.2.0
git+https://github.com/NateFerrero/oauth2lib.git
paramiko==1.15.1
peewee==2.4.3
marisa-trie==0.7
mixpanel-py==3.2.1
mock==1.0.1
paramiko==1.15.2
peewee==2.4.7
psutil==2.2.1
psycopg2==2.5.4
py-bcrypt==0.4
pycrypto==2.6.1
python-dateutil==2.2
python-ldap==2.4.18
python-dateutil==2.4.0
python-ldap==2.4.19
python-magic==0.4.6
pytz==2014.9
pygpgme==0.3
pytz==2014.10
pyOpenSSL==0.14
raven==5.1.1
redis==2.10.3
reportlab==2.7
requests==2.4.3
six==1.8.0
stripe==1.19.1
trollius==1.0.3
requests==2.5.1
six==1.9.0
stripe==1.20.1
trollius==1.0.4
tzlocal==1.1.2
websocket-client==0.21.0
waitress==0.8.9
websocket-client==0.23.0
wsgiref==0.1.2
xhtml2pdf==0.0.6
git+https://github.com/DevTable/aniso8601-fake.git
git+https://github.com/DevTable/anunidecode.git
git+https://github.com/DevTable/avatar-generator.git
git+https://github.com/DevTable/pygithub.git
git+https://github.com/DevTable/container-cloud-config.git
git+https://github.com/NateFerrero/oauth2lib.git
git+https://github.com/jplana/python-etcd.git

Some files were not shown because too many files have changed in this diff Show more