diff --git a/Bobfile b/Bobfile index 86b721675..80f4c39c7 100644 --- a/Bobfile +++ b/Bobfile @@ -9,14 +9,8 @@ version = 1 [[container]] name = "quay" - Dockerfile = "Dockerfile.web" + Dockerfile = "Dockerfile" project = "quay" tags = ["git:short"] -[[container]] - name = "builder" - Dockerfile = "Dockerfile.buildworker" - project = "builder" - tags = ["git:short"] - # vim:ft=toml diff --git a/Dockerfile.web b/Dockerfile similarity index 82% rename from Dockerfile.web rename to Dockerfile index 46b360b6c..d201270b0 100644 --- a/Dockerfile.web +++ b/Dockerfile @@ -1,27 +1,21 @@ # vim:ft=dockerfile -FROM phusion/baseimage:0.9.15 + +FROM phusion/baseimage:0.9.16 ENV DEBIAN_FRONTEND noninteractive ENV HOME /root # Install the dependencies. -RUN apt-get update # 10SEP2014 +RUN apt-get update # 29JAN2015 # New ubuntu packages should be added as their own apt-get install lines below the existing install commands -RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev libfreetype6-dev +RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev libfreetype6-dev libffi-dev libgpgme11 libgpgme11-dev # Build the python dependencies ADD requirements.txt requirements.txt RUN virtualenv --distribute venv RUN venv/bin/pip install -r requirements.txt -RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev - -### End common section ### - -# Remove SSH. -RUN rm -rf /etc/service/sshd /etc/my_init.d/00_regen_ssh_host_keys.sh - # Install the binary dependencies ADD binary_dependencies binary_dependencies RUN gdebi --n binary_dependencies/*.deb @@ -34,6 +28,10 @@ RUN npm install -g grunt-cli ADD grunt grunt RUN cd grunt && npm install +RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev libgpgme11-dev +RUN apt-get autoremove -y +RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + # Add all of the files! ADD . . @@ -58,14 +56,9 @@ ADD conf/init/buildmanager /etc/service/buildmanager RUN mkdir static/fonts static/ldn RUN venv/bin/python -m external_libraries -RUN apt-get autoremove -y -RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - # Run the tests RUN TEST=true venv/bin/python -m unittest discover -VOLUME ["/conf/stack", "/var/log", "/datastorage", "/tmp"] +VOLUME ["/conf/stack", "/var/log", "/datastorage", "/tmp", "/conf/etcd"] -EXPOSE 443 80 - -CMD ["/sbin/my_init"] +EXPOSE 443 8443 80 diff --git a/Dockerfile.buildworker b/Dockerfile.buildworker deleted file mode 100644 index 4b6f995b9..000000000 --- a/Dockerfile.buildworker +++ /dev/null @@ -1,42 +0,0 @@ -# vim:ft=dockerfile -FROM phusion/baseimage:0.9.15 - -ENV DEBIAN_FRONTEND noninteractive -ENV HOME /root - -# Install the dependencies. -RUN apt-get update # 20NOV2014 - -# New ubuntu packages should be added as their own apt-get install lines below the existing install commands -RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev - -# Build the python dependencies -ADD requirements.txt requirements.txt -RUN virtualenv --distribute venv -RUN venv/bin/pip install -r requirements.txt - -RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev - -### End common section ### - -RUN apt-get install -y lxc aufs-tools - -RUN usermod -v 100000-200000 -w 100000-200000 root - -ADD binary_dependencies/builder binary_dependencies/builder -RUN gdebi --n binary_dependencies/builder/*.deb - -ADD . . - -ADD conf/init/svlogd_config /svlogd_config -ADD conf/init/preplogsdir.sh /etc/my_init.d/ -ADD conf/init/tutumdocker /etc/service/tutumdocker -ADD conf/init/dockerfilebuild /etc/service/dockerfilebuild - -RUN apt-get remove -y --auto-remove nodejs npm git phantomjs -RUN apt-get autoremove -y -RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -VOLUME ["/var/lib/docker", "/var/lib/lxc", "/conf/stack", "/var/log"] - -CMD ["/sbin/my_init"] diff --git a/README.md b/README.md index 5824fea10..a8dd9b7f3 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ to build and upload quay to quay: ``` curl -s https://get.docker.io/ubuntu/ | sudo sh sudo apt-get update && sudo apt-get install -y git -git clone https://bitbucket.org/yackob03/quay.git +git clone https://github.com/coreos-inc/quay.git cd quay rm Dockerfile ln -s Dockerfile.web Dockerfile @@ -33,7 +33,7 @@ start the quay processes: ``` cd ~ -git clone https://bitbucket.org/yackob03/quayconfig.git +git clone https://github.com/coreos-inc/quay.git sudo docker pull staging.quay.io/quay/quay cd ~/gantryd sudo venv/bin/python gantry.py ../quayconfig/production/gantry.json update quay @@ -44,7 +44,7 @@ to build and upload the builder to quay ``` curl -s https://get.docker.io/ubuntu/ | sudo sh sudo apt-get update && sudo apt-get install -y git -git clone git clone https://bitbucket.org/yackob03/quay.git +git clone git clone https://github.com/coreos-inc/quay.git cd quay rm Dockerfile ln -s Dockerfile.buildworker Dockerfile @@ -74,7 +74,7 @@ start the worker ``` cd ~ -git clone https://bitbucket.org/yackob03/quayconfig.git +git clone https://github.com/coreos-inc/quay.git sudo docker pull quay.io/quay/builder cd ~/gantryd sudo venv/bin/python gantry.py ../quayconfig/production/gantry.json update builder diff --git a/app.py b/app.py index cdac98a27..78243de75 100644 --- a/app.py +++ b/app.py @@ -1,71 +1,55 @@ import logging import os import json -import yaml -from flask import Flask as BaseFlask, Config as BaseConfig, request, Request +from flask import Flask, Config, request, Request, _request_ctx_stack from flask.ext.principal import Principal -from flask.ext.login import LoginManager +from flask.ext.login import LoginManager, UserMixin from flask.ext.mail import Mail import features +from avatars.avatars import Avatar from storage import Storage + +from avatars.avatars import Avatar + from data import model from data import database from data.userfiles import Userfiles from data.users import UserAuthentication -from util.analytics import Analytics -from util.exceptionlog import Sentry -from util.queuemetrics import QueueMetrics -from util.names import urn_generator -from util.oauth import GoogleOAuthConfig, GithubOAuthConfig from data.billing import Billing from data.buildlogs import BuildLogs from data.archivedlogs import LogArchive -from data.queue import WorkQueue from data.userevent import UserEventsBuilderModule -from avatars.avatars import Avatar - - -class Config(BaseConfig): - """ Flask config enhanced with a `from_yamlfile` method """ - - def from_yamlfile(self, config_file): - with open(config_file) as f: - c = yaml.load(f) - if not c: - logger.debug('Empty YAML config file') - return - - if isinstance(c, str): - raise Exception('Invalid YAML config file: ' + str(c)) - - for key in c.iterkeys(): - if key.isupper(): - self[key] = c[key] - -class Flask(BaseFlask): - """ Extends the Flask class to implement our custom Config class. """ - - def make_config(self, instance_relative=False): - root_path = self.instance_path if instance_relative else self.root_path - return Config(root_path, self.default_config) - +from data.queue import WorkQueue +from util.analytics import Analytics +from util.exceptionlog import Sentry +from util.names import urn_generator +from util.oauth import GoogleOAuthConfig, GithubOAuthConfig +from util.signing import Signer +from util.queuemetrics import QueueMetrics +from util.config.provider import FileConfigProvider, TestConfigProvider +from util.config.configutil import generate_secret_key +from util.config.superusermanager import SuperUserManager +from buildman.jobutil.buildreporter import BuildMetrics +OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/' OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml' OVERRIDE_CONFIG_PY_FILENAME = 'conf/stack/config.py' OVERRIDE_CONFIG_KEY = 'QUAY_OVERRIDE_CONFIG' LICENSE_FILENAME = 'conf/stack/license.enc' +CONFIG_PROVIDER = FileConfigProvider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py') app = Flask(__name__) logger = logging.getLogger(__name__) -profile = logging.getLogger('profile') - +# Instantiate the default configuration (for test or for normal operation). if 'TEST' in os.environ: + CONFIG_PROVIDER = TestConfigProvider() + from test.testconfig import TestConfig logger.debug('Loading test config.') app.config.from_object(TestConfig()) @@ -73,20 +57,17 @@ else: from config import DefaultConfig logger.debug('Loading default config.') app.config.from_object(DefaultConfig()) - - if os.path.exists(OVERRIDE_CONFIG_PY_FILENAME): - logger.debug('Applying config file: %s', OVERRIDE_CONFIG_PY_FILENAME) - app.config.from_pyfile(OVERRIDE_CONFIG_PY_FILENAME) - - if os.path.exists(OVERRIDE_CONFIG_YAML_FILENAME): - logger.debug('Applying config file: %s', OVERRIDE_CONFIG_YAML_FILENAME) - app.config.from_yamlfile(OVERRIDE_CONFIG_YAML_FILENAME) - - environ_config = json.loads(os.environ.get(OVERRIDE_CONFIG_KEY, '{}')) - app.config.update(environ_config) - app.teardown_request(database.close_db_filter) +# Load the override config via the provider. +CONFIG_PROVIDER.update_app_config(app.config) + +# Update any configuration found in the override environment variable. +OVERRIDE_CONFIG_KEY = 'QUAY_OVERRIDE_CONFIG' + +environ_config = json.loads(os.environ.get(OVERRIDE_CONFIG_KEY, '{}')) +app.config.update(environ_config) + class RequestWithId(Request): request_gen = staticmethod(urn_generator(['request'])) @@ -98,21 +79,24 @@ class RequestWithId(Request): @app.before_request def _request_start(): - profile.debug('Starting request: %s', request.path) + logger.debug('Starting request: %s', request.path) @app.after_request def _request_end(r): - profile.debug('Ending request: %s', request.path) + logger.debug('Ending request: %s', request.path) return r class InjectingFilter(logging.Filter): def filter(self, record): - record.msg = '[%s] %s' % (request.request_id, record.msg) + if _request_ctx_stack.top is not None: + record.msg = '[%s] %s' % (request.request_id, record.msg) return True -profile.addFilter(InjectingFilter()) +# Add the request id filter to all handlers of the root logger +for handler in logging.getLogger().handlers: + handler.addFilter(InjectingFilter()) app.request_class = RequestWithId @@ -130,16 +114,20 @@ analytics = Analytics(app) billing = Billing(app) sentry = Sentry(app) build_logs = BuildLogs(app) -queue_metrics = QueueMetrics(app) authentication = UserAuthentication(app) userevents = UserEventsBuilderModule(app) - -github_login = GithubOAuthConfig(app, 'GITHUB_LOGIN_CONFIG') -github_trigger = GithubOAuthConfig(app, 'GITHUB_TRIGGER_CONFIG') -google_login = GoogleOAuthConfig(app, 'GOOGLE_LOGIN_CONFIG') -oauth_apps = [github_login, github_trigger, google_login] +superusers = SuperUserManager(app) +signer = Signer(app, OVERRIDE_CONFIG_DIRECTORY) +queue_metrics = QueueMetrics(app) +build_metrics = BuildMetrics(app) tf = app.config['DB_TRANSACTION_FACTORY'] + +github_login = GithubOAuthConfig(app.config, 'GITHUB_LOGIN_CONFIG') +github_trigger = GithubOAuthConfig(app.config, 'GITHUB_TRIGGER_CONFIG') +google_login = GoogleOAuthConfig(app.config, 'GOOGLE_LOGIN_CONFIG') +oauth_apps = [github_login, github_trigger, google_login] + image_diff_queue = WorkQueue(app.config['DIFFS_QUEUE_NAME'], tf) dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf, reporter=queue_metrics.report) @@ -149,5 +137,34 @@ database.configure(app.config) model.config.app_config = app.config model.config.store = storage +# Generate a secret key if none was specified. +if app.config['SECRET_KEY'] is None: + logger.debug('Generating in-memory secret key') + app.config['SECRET_KEY'] = generate_secret_key() + +@login_manager.user_loader +def load_user(user_uuid): + logger.debug('User loader loading deferred user with uuid: %s' % user_uuid) + return LoginWrappedDBUser(user_uuid) + +class LoginWrappedDBUser(UserMixin): + def __init__(self, user_uuid, db_user=None): + self._uuid = user_uuid + self._db_user = db_user + + def db_user(self): + if not self._db_user: + self._db_user = model.get_user_by_uuid(self._uuid) + return self._db_user + + def is_authenticated(self): + return self.db_user() is not None + + def is_active(self): + return self.db_user().verified + + def get_id(self): + return unicode(self._uuid) + def get_app_url(): return '%s://%s' % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME']) diff --git a/application.py b/application.py index a9bd0df6e..235a80b16 100644 --- a/application.py +++ b/application.py @@ -11,5 +11,5 @@ import registry if __name__ == '__main__': - logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False) + logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False) application.run(port=5000, debug=True, threaded=True, host='0.0.0.0') diff --git a/auth/permissions.py b/auth/permissions.py index ae398092d..4ee73bdb3 100644 --- a/auth/permissions.py +++ b/auth/permissions.py @@ -7,7 +7,7 @@ from functools import partial import scopes from data import model -from app import app +from app import app, superusers logger = logging.getLogger(__name__) @@ -89,10 +89,14 @@ class QuayDeferredPermissionUser(Identity): if not self._permissions_loaded: logger.debug('Loading user permissions after deferring.') user_object = model.get_user_by_uuid(self.id) + if user_object is None: + return super(QuayDeferredPermissionUser, self).can(permission) + + if user_object is None: + return super(QuayDeferredPermissionUser, self).can(permission) # Add the superuser need, if applicable. - if (user_object.username is not None and - user_object.username in app.config.get('SUPER_USERS', [])): + if superusers.is_superuser(user_object.username): self.provides.add(_SuperUserNeed()) # Add the user specific permissions, only for non-oauth permission diff --git a/binary_dependencies/builder/lxc-docker-1.3.2-userns_1.3.2-userns-20141124223407-72690d2-dirty_amd64.deb b/binary_dependencies/builder/lxc-docker-1.3.2-userns_1.3.2-userns-20141124223407-72690d2-dirty_amd64.deb deleted file mode 100644 index a99bcbd4f..000000000 Binary files a/binary_dependencies/builder/lxc-docker-1.3.2-userns_1.3.2-userns-20141124223407-72690d2-dirty_amd64.deb and /dev/null differ diff --git a/binary_dependencies/nginx_1.4.2-nobuffer-3_amd64.deb b/binary_dependencies/nginx_1.4.2-nobuffer-3_amd64.deb deleted file mode 100644 index 8b3fb3fb6..000000000 Binary files a/binary_dependencies/nginx_1.4.2-nobuffer-3_amd64.deb and /dev/null differ diff --git a/binary_dependencies/tengine_2.1.0-1_amd64.deb b/binary_dependencies/tengine_2.1.0-1_amd64.deb new file mode 100644 index 000000000..0acfbd14a Binary files /dev/null and b/binary_dependencies/tengine_2.1.0-1_amd64.deb differ diff --git a/build.sh b/build.sh new file mode 100755 index 000000000..12ec0faa7 --- /dev/null +++ b/build.sh @@ -0,0 +1,2 @@ +docker build -t quay.io/quay/quay:`git rev-parse --short HEAD` . +echo quay.io/quay/quay:`git rev-parse --short HEAD` \ No newline at end of file diff --git a/buildman/asyncutil.py b/buildman/asyncutil.py new file mode 100644 index 000000000..4f2d4e1a9 --- /dev/null +++ b/buildman/asyncutil.py @@ -0,0 +1,27 @@ +from functools import partial, wraps +from trollius import get_event_loop + + +class AsyncWrapper(object): + """ Wrapper class which will transform a syncronous library to one that can be used with + trollius coroutines. + """ + def __init__(self, delegate, loop=None, executor=None): + self._loop = loop if loop is not None else get_event_loop() + self._delegate = delegate + self._executor = executor + + def __getattr__(self, attrib): + delegate_attr = getattr(self._delegate, attrib) + + if not callable(delegate_attr): + return delegate_attr + + def wrapper(*args, **kwargs): + """ Wraps the delegate_attr with primitives that will transform sync calls to ones shelled + out to a thread pool. + """ + callable_delegate_attr = partial(delegate_attr, *args, **kwargs) + return self._loop.run_in_executor(self._executor, callable_delegate_attr) + + return wrapper diff --git a/buildman/builder.py b/buildman/builder.py index 3e14db3eb..2a0225751 100644 --- a/buildman/builder.py +++ b/buildman/builder.py @@ -6,6 +6,7 @@ import time from app import app, userfiles as user_files, build_logs, dockerfile_build_queue from buildman.manager.enterprise import EnterpriseManager +from buildman.manager.ephemeral import EphemeralBuilderManager from buildman.server import BuilderServer from trollius import SSLContext @@ -13,14 +14,22 @@ from trollius import SSLContext logger = logging.getLogger(__name__) BUILD_MANAGERS = { - 'enterprise': EnterpriseManager + 'enterprise': EnterpriseManager, + 'ephemeral': EphemeralBuilderManager, } EXTERNALLY_MANAGED = 'external' +DEFAULT_WEBSOCKET_PORT = 8787 +DEFAULT_CONTROLLER_PORT = 8686 + +LOG_FORMAT = "%(asctime)s [%(process)d] [%(levelname)s] [%(name)s] %(message)s" + def run_build_manager(): if not features.BUILD_SUPPORT: logger.debug('Building is disabled. Please enable the feature flag') + while True: + time.sleep(1000) return build_manager_config = app.config.get('BUILD_MANAGER') @@ -39,18 +48,32 @@ def run_build_manager(): if manager_klass is None: return + manager_hostname = os.environ.get('BUILDMAN_HOSTNAME', + app.config.get('BUILDMAN_HOSTNAME', + app.config['SERVER_HOSTNAME'])) + websocket_port = int(os.environ.get('BUILDMAN_WEBSOCKET_PORT', + app.config.get('BUILDMAN_WEBSOCKET_PORT', + DEFAULT_WEBSOCKET_PORT))) + controller_port = int(os.environ.get('BUILDMAN_CONTROLLER_PORT', + app.config.get('BUILDMAN_CONTROLLER_PORT', + DEFAULT_CONTROLLER_PORT))) + + logger.debug('Will pass buildman hostname %s to builders for websocket connection', + manager_hostname) + logger.debug('Starting build manager with lifecycle "%s"', build_manager_config[0]) ssl_context = None if os.environ.get('SSL_CONFIG'): logger.debug('Loading SSL cert and key') ssl_context = SSLContext() - ssl_context.load_cert_chain(os.environ.get('SSL_CONFIG') + '/ssl.cert', - os.environ.get('SSL_CONFIG') + '/ssl.key') + ssl_context.load_cert_chain(os.path.join(os.environ.get('SSL_CONFIG'), 'ssl.cert'), + os.path.join(os.environ.get('SSL_CONFIG'), 'ssl.key')) server = BuilderServer(app.config['SERVER_HOSTNAME'], dockerfile_build_queue, build_logs, - user_files, manager_klass) - server.run('0.0.0.0', ssl=ssl_context) + user_files, manager_klass, build_manager_config[1], manager_hostname) + server.run('0.0.0.0', websocket_port, controller_port, ssl=ssl_context) if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT) + logging.getLogger('peewee').setLevel(logging.WARN) run_build_manager() diff --git a/buildman/component/basecomponent.py b/buildman/component/basecomponent.py index 47781dff5..bd4032776 100644 --- a/buildman/component/basecomponent.py +++ b/buildman/component/basecomponent.py @@ -8,3 +8,6 @@ class BaseComponent(ApplicationSession): self.parent_manager = None self.build_logs = None self.user_files = None + + def kind(self): + raise NotImplementedError \ No newline at end of file diff --git a/buildman/component/buildcomponent.py b/buildman/component/buildcomponent.py index d518d3453..647161190 100644 --- a/buildman/component/buildcomponent.py +++ b/buildman/component/buildcomponent.py @@ -6,11 +6,10 @@ import trollius import re from autobahn.wamp.exception import ApplicationError -from trollius.coroutines import From from buildman.server import BuildJobResult from buildman.component.basecomponent import BaseComponent -from buildman.jobutil.buildpack import BuildPackage, BuildPackageException +from buildman.jobutil.buildjob import BuildJobLoadException from buildman.jobutil.buildstatus import StatusHandler from buildman.jobutil.workererror import WorkerError @@ -20,7 +19,7 @@ HEARTBEAT_DELTA = datetime.timedelta(seconds=30) HEARTBEAT_TIMEOUT = 10 INITIAL_TIMEOUT = 25 -SUPPORTED_WORKER_VERSIONS = ['0.1-beta'] +SUPPORTED_WORKER_VERSIONS = ['0.3'] logger = logging.getLogger(__name__) @@ -39,84 +38,71 @@ class BuildComponent(BaseComponent): self.builder_realm = realm self.parent_manager = None - self.server_hostname = None + self.registry_hostname = None self._component_status = ComponentStatus.JOINING self._last_heartbeat = None self._current_job = None self._build_status = None self._image_info = None + self._worker_version = None BaseComponent.__init__(self, config, **kwargs) + def kind(self): + return 'builder' + def onConnect(self): self.join(self.builder_realm) def onJoin(self, details): logger.debug('Registering methods and listeners for component %s', self.builder_realm) - yield From(self.register(self._on_ready, u'io.quay.buildworker.ready')) - yield From(self.register(self._ping, u'io.quay.buildworker.ping')) - yield From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat')) - yield From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage')) + yield trollius.From(self.register(self._on_ready, u'io.quay.buildworker.ready')) + yield trollius.From(self.register(self._determine_cache_tag, + u'io.quay.buildworker.determinecachetag')) + yield trollius.From(self.register(self._ping, u'io.quay.buildworker.ping')) - self._set_status(ComponentStatus.WAITING) + yield trollius.From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat')) + yield trollius.From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage')) + + yield trollius.From(self._set_status(ComponentStatus.WAITING)) def is_ready(self): """ Determines whether a build component is ready to begin a build. """ return self._component_status == ComponentStatus.RUNNING + @trollius.coroutine def start_build(self, build_job): """ Starts a build. """ + logger.debug('Starting build for component %s (worker version: %s)', + self.builder_realm, self._worker_version) + self._current_job = build_job - self._build_status = StatusHandler(self.build_logs, build_job.repo_build()) + self._build_status = StatusHandler(self.build_logs, build_job.repo_build.uuid) self._image_info = {} - self._set_status(ComponentStatus.BUILDING) + yield trollius.From(self._set_status(ComponentStatus.BUILDING)) - # Retrieve the job's buildpack. - buildpack_url = self.user_files.get_file_url(build_job.repo_build().resource_key, + # Send the notification that the build has started. + build_job.send_notification('build_start') + + # Parse the build configuration. + try: + build_config = build_job.build_config + except BuildJobLoadException as irbe: + self._build_failure('Could not load build job information', irbe) + + base_image_information = {} + buildpack_url = self.user_files.get_file_url(build_job.repo_build.resource_key, requires_cors=False) - logger.debug('Retreiving build package: %s', buildpack_url) - buildpack = None - try: - buildpack = BuildPackage.from_url(buildpack_url) - except BuildPackageException as bpe: - self._build_failure('Could not retrieve build package', bpe) - return - - # Extract the base image information from the Dockerfile. - parsed_dockerfile = None - logger.debug('Parsing dockerfile') - - build_config = build_job.build_config() - try: - parsed_dockerfile = buildpack.parse_dockerfile(build_config.get('build_subdir')) - except BuildPackageException as bpe: - self._build_failure('Could not find Dockerfile in build package', bpe) - return - - image_and_tag_tuple = parsed_dockerfile.get_image_and_tag() - if image_and_tag_tuple is None or image_and_tag_tuple[0] is None: - self._build_failure('Missing FROM line in Dockerfile') - return - - base_image_information = { - 'repository': image_and_tag_tuple[0], - 'tag': image_and_tag_tuple[1] - } - - # Extract the number of steps from the Dockerfile. - with self._build_status as status_dict: - status_dict['total_commands'] = len(parsed_dockerfile.commands) - # Add the pull robot information, if any. - if build_config.get('pull_credentials') is not None: - base_image_information['username'] = build_config['pull_credentials'].get('username', '') - base_image_information['password'] = build_config['pull_credentials'].get('password', '') + if build_job.pull_credentials: + base_image_information['username'] = build_job.pull_credentials.get('username', '') + base_image_information['password'] = build_job.pull_credentials.get('password', '') # Retrieve the repository's fully qualified name. - repo = build_job.repo_build().repository + repo = build_job.repo_build.repository repository_name = repo.namespace_user.username + '/' + repo.name # Parse the build queue item into build arguments. @@ -128,29 +114,26 @@ class BuildComponent(BaseComponent): # push_token: The token to use to push the built image. # tag_names: The name(s) of the tag(s) for the newly built image. # base_image: The image name and credentials to use to conduct the base image pull. - # repository: The repository to pull. - # tag: The tag to pull. + # repository: The repository to pull (DEPRECATED 0.2) + # tag: The tag to pull (DEPRECATED in 0.2) # username: The username for pulling the base image (if any). # password: The password for pulling the base image (if any). build_arguments = { 'build_package': buildpack_url, 'sub_directory': build_config.get('build_subdir', ''), 'repository': repository_name, - 'registry': self.server_hostname, - 'pull_token': build_job.repo_build().access_token.code, - 'push_token': build_job.repo_build().access_token.code, + 'registry': self.registry_hostname, + 'pull_token': build_job.repo_build.access_token.code, + 'push_token': build_job.repo_build.access_token.code, 'tag_names': build_config.get('docker_tags', ['latest']), - 'base_image': base_image_information, - 'cached_tag': build_job.determine_cached_tag() or '' + 'base_image': base_image_information } # Invoke the build. logger.debug('Invoking build: %s', self.builder_realm) logger.debug('With Arguments: %s', build_arguments) - return (self - .call("io.quay.builder.build", **build_arguments) - .add_done_callback(self._build_complete)) + self.call("io.quay.builder.build", **build_arguments).add_done_callback(self._build_complete) @staticmethod def _total_completion(statuses, total_images): @@ -237,18 +220,28 @@ class BuildComponent(BaseComponent): elif phase == BUILD_PHASE.BUILDING: self._build_status.append_log(current_status_string) + @trollius.coroutine + def _determine_cache_tag(self, command_comments, base_image_name, base_image_tag, base_image_id): + with self._build_status as status_dict: + status_dict['total_commands'] = len(command_comments) + 1 + + logger.debug('Checking cache on realm %s. Base image: %s:%s (%s)', self.builder_realm, + base_image_name, base_image_tag, base_image_id) + + tag_found = self._current_job.determine_cached_tag(base_image_id, command_comments) + raise trollius.Return(tag_found or '') def _build_failure(self, error_message, exception=None): """ Handles and logs a failed build. """ self._build_status.set_error(error_message, { - 'internal_error': exception.message if exception else None + 'internal_error': str(exception) if exception else None }) - build_id = self._current_job.repo_build().uuid + build_id = self._current_job.repo_build.uuid logger.warning('Build %s failed with message: %s', build_id, error_message) # Mark that the build has finished (in an error state) - self._build_finished(BuildJobResult.ERROR) + trollius.async(self._build_finished(BuildJobResult.ERROR)) def _build_complete(self, result): """ Wraps up a completed build. Handles any errors and calls self._build_finished. """ @@ -256,60 +249,78 @@ class BuildComponent(BaseComponent): # Retrieve the result. This will raise an ApplicationError on any error that occurred. result.result() self._build_status.set_phase(BUILD_PHASE.COMPLETE) - self._build_finished(BuildJobResult.COMPLETE) + trollius.async(self._build_finished(BuildJobResult.COMPLETE)) + + # Send the notification that the build has completed successfully. + self._current_job.send_notification('build_success') except ApplicationError as aex: worker_error = WorkerError(aex.error, aex.kwargs.get('base_error')) # Write the error to the log. self._build_status.set_error(worker_error.public_message(), worker_error.extra_data(), - internal_error=worker_error.is_internal_error()) + internal_error=worker_error.is_internal_error(), + requeued=self._current_job.has_retries_remaining()) + + # Send the notification that the build has failed. + self._current_job.send_notification('build_failure', + error_message=worker_error.public_message()) # Mark the build as completed. if worker_error.is_internal_error(): - self._build_finished(BuildJobResult.INCOMPLETE) + trollius.async(self._build_finished(BuildJobResult.INCOMPLETE)) else: - self._build_finished(BuildJobResult.ERROR) + trollius.async(self._build_finished(BuildJobResult.ERROR)) + @trollius.coroutine def _build_finished(self, job_status): """ Alerts the parent that a build has completed and sets the status back to running. """ - self.parent_manager.job_completed(self._current_job, job_status, self) + yield trollius.From(self.parent_manager.job_completed(self._current_job, job_status, self)) self._current_job = None # Set the component back to a running state. - self._set_status(ComponentStatus.RUNNING) + yield trollius.From(self._set_status(ComponentStatus.RUNNING)) @staticmethod def _ping(): """ Ping pong. """ return 'pong' + @trollius.coroutine def _on_ready(self, token, version): - if not version in SUPPORTED_WORKER_VERSIONS: - logger.warning('Build component (token "%s") is running an out-of-date version: %s', version) - return False + self._worker_version = version - if self._component_status != 'waiting': + if not version in SUPPORTED_WORKER_VERSIONS: + logger.warning('Build component (token "%s") is running an out-of-date version: %s', token, + version) + raise trollius.Return(False) + + if self._component_status != ComponentStatus.WAITING: logger.warning('Build component (token "%s") is already connected', self.expected_token) - return False + raise trollius.Return(False) if token != self.expected_token: - logger.warning('Builder token mismatch. Expected: "%s". Found: "%s"', self.expected_token, token) - return False + logger.warning('Builder token mismatch. Expected: "%s". Found: "%s"', self.expected_token, + token) + raise trollius.Return(False) - self._set_status(ComponentStatus.RUNNING) + yield trollius.From(self._set_status(ComponentStatus.RUNNING)) # Start the heartbeat check and updating loop. loop = trollius.get_event_loop() loop.create_task(self._heartbeat()) logger.debug('Build worker %s is connected and ready', self.builder_realm) - return True + raise trollius.Return(True) + @trollius.coroutine def _set_status(self, phase): + if phase == ComponentStatus.RUNNING: + yield trollius.From(self.parent_manager.build_component_ready(self)) + self._component_status = phase def _on_heartbeat(self): """ Updates the last known heartbeat. """ - self._last_heartbeat = datetime.datetime.now() + self._last_heartbeat = datetime.datetime.utcnow() @trollius.coroutine def _heartbeat(self): @@ -317,13 +328,13 @@ class BuildComponent(BaseComponent): and updating the heartbeat in the build status dictionary (if applicable). This allows the build system to catch crashes from either end. """ - yield From(trollius.sleep(INITIAL_TIMEOUT)) + yield trollius.From(trollius.sleep(INITIAL_TIMEOUT)) while True: # If the component is no longer running or actively building, nothing more to do. if (self._component_status != ComponentStatus.RUNNING and self._component_status != ComponentStatus.BUILDING): - return + raise trollius.Return() # If there is an active build, write the heartbeat to its status. build_status = self._build_status @@ -331,35 +342,37 @@ class BuildComponent(BaseComponent): with build_status as status_dict: status_dict['heartbeat'] = int(time.time()) - # Mark the build item. current_job = self._current_job if current_job is not None: - self.parent_manager.job_heartbeat(current_job) + yield trollius.From(self.parent_manager.job_heartbeat(current_job)) # Check the heartbeat from the worker. logger.debug('Checking heartbeat on realm %s', self.builder_realm) - if self._last_heartbeat and self._last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA: - self._timeout() - return + if (self._last_heartbeat and + self._last_heartbeat < datetime.datetime.utcnow() - HEARTBEAT_DELTA): + yield trollius.From(self._timeout()) + raise trollius.Return() - yield From(trollius.sleep(HEARTBEAT_TIMEOUT)) + yield trollius.From(trollius.sleep(HEARTBEAT_TIMEOUT)) + @trollius.coroutine def _timeout(self): - self._set_status(ComponentStatus.TIMED_OUT) - logger.warning('Build component with realm %s has timed out', self.builder_realm) - self._dispose(timed_out=True) + if self._component_status == ComponentStatus.TIMED_OUT: + raise trollius.Return() + + yield trollius.From(self._set_status(ComponentStatus.TIMED_OUT)) + logger.warning('Build component with realm %s has timed out', self.builder_realm) - def _dispose(self, timed_out=False): # If we still have a running job, then it has not completed and we need to tell the parent # manager. if self._current_job is not None: - if timed_out: - self._build_status.set_error('Build worker timed out', internal_error=True) + self._build_status.set_error('Build worker timed out', internal_error=True, + requeued=self._current_job.has_retries_remaining()) self.parent_manager.job_completed(self._current_job, BuildJobResult.INCOMPLETE, self) self._build_status = None self._current_job = None # Unregister the current component so that it cannot be invoked again. - self.parent_manager.build_component_disposed(self, timed_out) + self.parent_manager.build_component_disposed(self, True) diff --git a/buildman/enums.py b/buildman/enums.py new file mode 100644 index 000000000..3d38217fe --- /dev/null +++ b/buildman/enums.py @@ -0,0 +1,12 @@ +class BuildJobResult(object): + """ Build job result enum """ + INCOMPLETE = 'incomplete' + COMPLETE = 'complete' + ERROR = 'error' + + +class BuildServerStatus(object): + """ Build server status enum """ + STARTING = 'starting' + RUNNING = 'running' + SHUTDOWN = 'shutting_down' diff --git a/buildman/jobutil/buildjob.py b/buildman/jobutil/buildjob.py index 2b9dbf35c..a6361e83a 100644 --- a/buildman/jobutil/buildjob.py +++ b/buildman/jobutil/buildjob.py @@ -1,6 +1,13 @@ -from data import model - import json +import logging + +from cachetools import lru_cache +from endpoints.notificationhelper import spawn_notification +from data import model +from util.imagetree import ImageTree + +logger = logging.getLogger(__name__) + class BuildJobLoadException(Exception): """ Exception raised if a build job could not be instantiated for some reason. """ @@ -9,52 +16,123 @@ class BuildJobLoadException(Exception): class BuildJob(object): """ Represents a single in-progress build job. """ def __init__(self, job_item): - self._job_item = job_item + self.job_item = job_item try: - self._job_details = json.loads(job_item.body) + self.job_details = json.loads(job_item.body) except ValueError: raise BuildJobLoadException( - 'Could not parse build queue item config with ID %s' % self._job_details['build_uuid'] + 'Could not parse build queue item config with ID %s' % self.job_details['build_uuid'] ) + def has_retries_remaining(self): + return self.job_item.retries_remaining > 0 + + def send_notification(self, kind, error_message=None): + tags = self.build_config.get('docker_tags', ['latest']) + event_data = { + 'build_id': self.repo_build.uuid, + 'build_name': self.repo_build.display_name, + 'docker_tags': tags, + 'trigger_id': self.repo_build.trigger.uuid, + 'trigger_kind': self.repo_build.trigger.service.name + } + + if error_message is not None: + event_data['error_message'] = error_message + + spawn_notification(self.repo_build.repository, kind, event_data, + subpage='build?current=%s' % self.repo_build.uuid, + pathargs=['build', self.repo_build.uuid]) + + + @lru_cache(maxsize=1) + def _load_repo_build(self): try: - self._repo_build = model.get_repository_build(self._job_details['namespace'], - self._job_details['repository'], - self._job_details['build_uuid']) + return model.get_repository_build(self.job_details['build_uuid']) except model.InvalidRepositoryBuildException: raise BuildJobLoadException( - 'Could not load repository build with ID %s' % self._job_details['build_uuid']) + 'Could not load repository build with ID %s' % self.job_details['build_uuid']) + @property + def repo_build(self): + return self._load_repo_build() + + @property + def pull_credentials(self): + """ Returns the pull credentials for this job, or None if none. """ + return self.job_details.get('pull_credentials') + + @property + def build_config(self): try: - self._build_config = json.loads(self._repo_build.job_config) + return json.loads(self.repo_build.job_config) except ValueError: raise BuildJobLoadException( - 'Could not parse repository build job config with ID %s' % self._job_details['build_uuid'] + 'Could not parse repository build job config with ID %s' % self.job_details['build_uuid'] ) - def determine_cached_tag(self): + def determine_cached_tag(self, base_image_id=None, cache_comments=None): """ Returns the tag to pull to prime the cache or None if none. """ - # TODO(jschorr): Change this to use the more complicated caching rules, once we have caching - # be a pull of things besides the constructed tags. - tags = self._build_config.get('docker_tags', ['latest']) - existing_tags = model.list_repository_tags(self._job_details['namespace'], - self._job_details['repository']) + cached_tag = None + if base_image_id and cache_comments: + cached_tag = self._determine_cached_tag_by_comments(base_image_id, cache_comments) + if not cached_tag: + cached_tag = self._determine_cached_tag_by_tag() + + logger.debug('Determined cached tag %s for %s: %s', cached_tag, base_image_id, cache_comments) + + return cached_tag + + def _determine_cached_tag_by_comments(self, base_image_id, cache_commands): + """ Determines the tag to use for priming the cache for this build job, by matching commands + starting at the given base_image_id. This mimics the Docker cache checking, so it should, + in theory, provide "perfect" caching. + """ + # Lookup the base image in the repository. If it doesn't exist, nothing more to do. + repo_build = self.repo_build + repo_namespace = repo_build.repository.namespace_user.username + repo_name = repo_build.repository.name + + base_image = model.get_image(repo_build.repository, base_image_id) + if base_image is None: + return None + + # Build an in-memory tree of the full heirarchy of images in the repository. + all_images = model.get_repository_images(repo_namespace, repo_name) + all_tags = model.list_repository_tags(repo_namespace, repo_name) + tree = ImageTree(all_images, all_tags, base_filter=base_image.id) + + # Find a path in the tree, starting at the base image, that matches the cache comments + # or some subset thereof. + def checker(step, image): + if step >= len(cache_commands): + return False + + full_command = '["/bin/sh", "-c", "%s"]' % cache_commands[step] + logger.debug('Checking step #%s: %s, %s == %s', step, image.id, + image.storage.command, full_command) + + return image.storage.command == full_command + + path = tree.find_longest_path(base_image.id, checker) + if not path: + return None + + # Find any tag associated with the last image in the path. + return tree.tag_containing_image(path[-1]) + + + def _determine_cached_tag_by_tag(self): + """ Determines the cached tag by looking for one of the tags being built, and seeing if it + exists in the repository. This is a fallback for when no comment information is available. + """ + tags = self.build_config.get('docker_tags', ['latest']) + repository = self.repo_build.repository + existing_tags = model.list_repository_tags(repository.namespace_user.username, repository.name) cached_tags = set(tags) & set([tag.name for tag in existing_tags]) if cached_tags: return list(cached_tags)[0] return None - - def job_item(self): - """ Returns the job's queue item. """ - return self._job_item - - def repo_build(self): - """ Returns the repository build DB row for the job. """ - return self._repo_build - - def build_config(self): - """ Returns the parsed repository build config for the job. """ - return self._build_config diff --git a/buildman/jobutil/buildpack.py b/buildman/jobutil/buildpack.py deleted file mode 100644 index 9892c65d3..000000000 --- a/buildman/jobutil/buildpack.py +++ /dev/null @@ -1,88 +0,0 @@ -import tarfile -import requests -import os - -from tempfile import TemporaryFile, mkdtemp -from zipfile import ZipFile -from util.dockerfileparse import parse_dockerfile -from util.safetar import safe_extractall - -class BuildPackageException(Exception): - """ Exception raised when retrieving or parsing a build package. """ - pass - - -class BuildPackage(object): - """ Helper class for easy reading and updating of a Dockerfile build pack. """ - - def __init__(self, requests_file): - self._mime_processors = { - 'application/zip': BuildPackage._prepare_zip, - 'application/x-zip-compressed': BuildPackage._prepare_zip, - 'text/plain': BuildPackage._prepare_dockerfile, - 'application/octet-stream': BuildPackage._prepare_dockerfile, - 'application/x-tar': BuildPackage._prepare_tarball, - 'application/gzip': BuildPackage._prepare_tarball, - 'application/x-gzip': BuildPackage._prepare_tarball, - } - - c_type = requests_file.headers['content-type'] - c_type = c_type.split(';')[0] if ';' in c_type else c_type - - if c_type not in self._mime_processors: - raise BuildPackageException('Unknown build package mime type: %s' % c_type) - - self._package_directory = None - try: - self._package_directory = self._mime_processors[c_type](requests_file) - except Exception as ex: - raise BuildPackageException(ex.message) - - def parse_dockerfile(self, subdirectory): - dockerfile_path = os.path.join(self._package_directory, subdirectory, 'Dockerfile') - if not os.path.exists(dockerfile_path): - if subdirectory: - message = 'Build package did not contain a Dockerfile at sub directory %s.' % subdirectory - else: - message = 'Build package did not contain a Dockerfile at the root directory.' - - raise BuildPackageException(message) - - with open(dockerfile_path, 'r') as dockerfileobj: - return parse_dockerfile(dockerfileobj.read()) - - @staticmethod - def from_url(url): - buildpack_resource = requests.get(url, stream=True) - return BuildPackage(buildpack_resource) - - @staticmethod - def _prepare_zip(request_file): - build_dir = mkdtemp(prefix='docker-build-') - - # Save the zip file to temp somewhere - with TemporaryFile() as zip_file: - zip_file.write(request_file.content) - to_extract = ZipFile(zip_file) - to_extract.extractall(build_dir) - - return build_dir - - @staticmethod - def _prepare_dockerfile(request_file): - build_dir = mkdtemp(prefix='docker-build-') - dockerfile_path = os.path.join(build_dir, "Dockerfile") - with open(dockerfile_path, 'w') as dockerfile: - dockerfile.write(request_file.content) - - return build_dir - - @staticmethod - def _prepare_tarball(request_file): - build_dir = mkdtemp(prefix='docker-build-') - - # Save the zip file to temp somewhere - with tarfile.open(mode='r|*', fileobj=request_file.raw) as tar_stream: - safe_extractall(tar_stream, build_dir) - - return build_dir diff --git a/buildman/jobutil/buildreporter.py b/buildman/jobutil/buildreporter.py new file mode 100644 index 000000000..16dd0ca5b --- /dev/null +++ b/buildman/jobutil/buildreporter.py @@ -0,0 +1,72 @@ +from trollius import From + +from buildman.enums import BuildJobResult +from util.cloudwatch import get_queue + + +class BuildReporter(object): + """ + Base class for reporting build statuses to a metrics service. + """ + def report_completion_status(self, status): + """ + Method to invoke the recording of build's completion status to a metric service. + """ + raise NotImplementedError + + +class NullReporter(BuildReporter): + """ + The /dev/null of BuildReporters. + """ + def report_completion_status(self, *args): + pass + + +class CloudWatchBuildReporter(BuildReporter): + """ + Implements a BuildReporter for Amazon's CloudWatch. + """ + def __init__(self, queue, namespace_name, completed_name, failed_name, incompleted_name): + self._queue = queue + self._namespace_name = namespace_name + self._completed_name = completed_name + self._failed_name = failed_name + self._incompleted_name = incompleted_name + + def _send_to_queue(self, *args, **kwargs): + self._queue.put((args, kwargs)) + + def report_completion_status(self, status): + if status == BuildJobResult.COMPLETE: + status_name = self._completed_name + elif status == BuildJobResult.ERROR: + status_name = self._failed_name + elif status == BuildJobResult.INCOMPLETE: + status_name = self._incompleted_name + else: + return + + self._send_to_queue(self._namespace_name, status_name, 1, unit='Count') + + +class BuildMetrics(object): + """ + BuildMetrics initializes a reporter for recording the status of build completions. + """ + def __init__(self, app=None): + self._app = app + self._reporter = NullReporter() + if app is not None: + reporter_type = app.config.get('BUILD_METRICS_TYPE', 'Null') + if reporter_type == 'CloudWatch': + namespace = app.config['BUILD_METRICS_NAMESPACE'] + completed_name = app.config['BUILD_METRICS_COMPLETED_NAME'] + failed_name = app.config['BUILD_METRICS_FAILED_NAME'] + incompleted_name = app.config['BUILD_METRICS_INCOMPLETED_NAME'] + request_queue = get_queue(app) + self._reporter = CloudWatchBuildReporter(request_queue, namespace, completed_name, + failed_name, incompleted_name) + + def __getattr__(self, name): + return getattr(self._reporter, name, None) diff --git a/buildman/jobutil/buildstatus.py b/buildman/jobutil/buildstatus.py index 68b8cd5e3..892f8f6c7 100644 --- a/buildman/jobutil/buildstatus.py +++ b/buildman/jobutil/buildstatus.py @@ -1,16 +1,18 @@ from data.database import BUILD_PHASE +from data import model +import datetime class StatusHandler(object): """ Context wrapper for writing status to build logs. """ - def __init__(self, build_logs, repository_build): + def __init__(self, build_logs, repository_build_uuid): self._current_phase = None - self._repository_build = repository_build - self._uuid = repository_build.uuid + self._current_command = None + self._uuid = repository_build_uuid self._build_logs = build_logs self._status = { - 'total_commands': None, + 'total_commands': 0, 'current_command': None, 'push_completion': 0.0, 'pull_completion': 0.0, @@ -20,16 +22,25 @@ class StatusHandler(object): self.__exit__(None, None, None) def _append_log_message(self, log_message, log_type=None, log_data=None): + log_data = log_data or {} + log_data['datetime'] = str(datetime.datetime.now()) self._build_logs.append_log_message(self._uuid, log_message, log_type, log_data) def append_log(self, log_message, extra_data=None): + if log_message is None: + return + self._append_log_message(log_message, log_data=extra_data) def set_command(self, command, extra_data=None): + if self._current_command == command: + return + + self._current_command = command self._append_log_message(command, self._build_logs.COMMAND, extra_data) - def set_error(self, error_message, extra_data=None, internal_error=False): - self.set_phase(BUILD_PHASE.INTERNAL_ERROR if internal_error else BUILD_PHASE.ERROR) + def set_error(self, error_message, extra_data=None, internal_error=False, requeued=False): + self.set_phase(BUILD_PHASE.INTERNAL_ERROR if internal_error and requeued else BUILD_PHASE.ERROR) extra_data = extra_data or {} extra_data['internal_error'] = internal_error @@ -41,8 +52,12 @@ class StatusHandler(object): self._current_phase = phase self._append_log_message(phase, self._build_logs.PHASE, extra_data) - self._repository_build.phase = phase - self._repository_build.save() + + # Update the repository build with the new phase + repo_build = model.get_repository_build(self._uuid) + repo_build.phase = phase + repo_build.save() + return True def __enter__(self): diff --git a/buildman/jobutil/workererror.py b/buildman/jobutil/workererror.py index 8271976e4..fdf6503b0 100644 --- a/buildman/jobutil/workererror.py +++ b/buildman/jobutil/workererror.py @@ -19,13 +19,19 @@ class WorkerError(object): 'is_internal': True }, + 'io.quay.builder.dockerfileissue': { + 'message': 'Could not find or parse Dockerfile', + 'show_base_error': True + }, + 'io.quay.builder.cannotpullbaseimage': { 'message': 'Could not pull base image', 'show_base_error': True }, 'io.quay.builder.internalerror': { - 'message': 'An internal error occurred while building. Please submit a ticket.' + 'message': 'An internal error occurred while building. Please submit a ticket.', + 'is_internal': True }, 'io.quay.builder.buildrunerror': { @@ -57,6 +63,11 @@ class WorkerError(object): 'io.quay.builder.missingorinvalidargument': { 'message': 'Missing required arguments for builder', 'is_internal': True + }, + + 'io.quay.builder.cachelookupissue': { + 'message': 'Error checking for a cached tag', + 'is_internal': True } } diff --git a/buildman/manager/basemanager.py b/buildman/manager/basemanager.py index f66054c45..2c57ac095 100644 --- a/buildman/manager/basemanager.py +++ b/buildman/manager/basemanager.py @@ -1,12 +1,17 @@ +from trollius import coroutine + class BaseManager(object): """ Base for all worker managers. """ def __init__(self, register_component, unregister_component, job_heartbeat_callback, - job_complete_callback): + job_complete_callback, manager_hostname, heartbeat_period_sec): self.register_component = register_component self.unregister_component = unregister_component self.job_heartbeat_callback = job_heartbeat_callback self.job_complete_callback = job_complete_callback + self.manager_hostname = manager_hostname + self.heartbeat_period_sec = heartbeat_period_sec + @coroutine def job_heartbeat(self, build_job): """ Method invoked to tell the manager that a job is still running. This method will be called every few minutes. """ @@ -25,25 +30,41 @@ class BaseManager(object): """ raise NotImplementedError - def schedule(self, build_job, loop): + @coroutine + def schedule(self, build_job): """ Schedules a queue item to be built. Returns True if the item was properly scheduled and False if all workers are busy. """ raise NotImplementedError - def initialize(self): + def initialize(self, manager_config): """ Runs any initialization code for the manager. Called once the server is in a ready state. """ raise NotImplementedError + @coroutine + def build_component_ready(self, build_component): + """ Method invoked whenever a build component announces itself as ready. + """ + raise NotImplementedError + def build_component_disposed(self, build_component, timed_out): """ Method invoked whenever a build component has been disposed. The timed_out boolean indicates whether the component's heartbeat timed out. """ raise NotImplementedError + @coroutine def job_completed(self, build_job, job_status, build_component): """ Method invoked once a job_item has completed, in some manner. The job_status will be - one of: incomplete, error, complete. If incomplete, the job should be requeued. + one of: incomplete, error, complete. Implementations of this method should call + self.job_complete_callback with a status of Incomplete if they wish for the job to be + automatically requeued. + """ + raise NotImplementedError + + def num_workers(self): + """ Returns the number of active build workers currently registered. This includes those + that are currently busy and awaiting more work. """ raise NotImplementedError diff --git a/buildman/manager/enterprise.py b/buildman/manager/enterprise.py index 824e02d53..0ce69e508 100644 --- a/buildman/manager/enterprise.py +++ b/buildman/manager/enterprise.py @@ -5,7 +5,7 @@ from buildman.component.basecomponent import BaseComponent from buildman.component.buildcomponent import BuildComponent from buildman.manager.basemanager import BaseManager -from trollius.coroutines import From +from trollius import From, Return, coroutine REGISTRATION_REALM = 'registration' logger = logging.getLogger(__name__) @@ -25,13 +25,21 @@ class DynamicRegistrationComponent(BaseComponent): logger.debug('Registering new build component+worker with realm %s', realm) return realm + def kind(self): + return 'registration' + class EnterpriseManager(BaseManager): """ Build manager implementation for the Enterprise Registry. """ - build_components = [] - shutting_down = False - def initialize(self): + def __init__(self, *args, **kwargs): + self.ready_components = set() + self.all_components = set() + self.shutting_down = False + + super(EnterpriseManager, self).__init__(*args, **kwargs) + + def initialize(self, manager_config): # Add a component which is used by build workers for dynamic registration. Unlike # production, build workers in enterprise are long-lived and register dynamically. self.register_component(REGISTRATION_REALM, DynamicRegistrationComponent) @@ -45,28 +53,39 @@ class EnterpriseManager(BaseManager): """ Adds a new build component for an Enterprise Registry. """ # Generate a new unique realm ID for the build worker. realm = str(uuid.uuid4()) - component = self.register_component(realm, BuildComponent, token="") - self.build_components.append(component) + new_component = self.register_component(realm, BuildComponent, token="") + self.all_components.add(new_component) return realm - def schedule(self, build_job, loop): + @coroutine + def schedule(self, build_job): """ Schedules a build for an Enterprise Registry. """ - if self.shutting_down: - return False + if self.shutting_down or not self.ready_components: + raise Return(False) - for component in self.build_components: - if component.is_ready(): - loop.call_soon(component.start_build, build_job) - return True + component = self.ready_components.pop() - return False + yield From(component.start_build(build_job)) + + raise Return(True) + + @coroutine + def build_component_ready(self, build_component): + self.ready_components.add(build_component) def shutdown(self): self.shutting_down = True + @coroutine def job_completed(self, build_job, job_status, build_component): self.job_complete_callback(build_job, job_status) def build_component_disposed(self, build_component, timed_out): - self.build_components.remove(build_component) + self.all_components.remove(build_component) + if build_component in self.ready_components: + self.ready_components.remove(build_component) + self.unregister_component(build_component) + + def num_workers(self): + return len(self.all_components) diff --git a/buildman/manager/ephemeral.py b/buildman/manager/ephemeral.py new file mode 100644 index 000000000..473e75fb3 --- /dev/null +++ b/buildman/manager/ephemeral.py @@ -0,0 +1,326 @@ +import logging +import etcd +import uuid +import calendar +import os.path +import json + +from datetime import datetime, timedelta +from trollius import From, coroutine, Return, async +from concurrent.futures import ThreadPoolExecutor +from urllib3.exceptions import ReadTimeoutError, ProtocolError + +from buildman.manager.basemanager import BaseManager +from buildman.manager.executor import PopenExecutor, EC2Executor +from buildman.component.buildcomponent import BuildComponent +from buildman.jobutil.buildjob import BuildJob +from buildman.asyncutil import AsyncWrapper +from util.morecollections import AttrDict + + +logger = logging.getLogger(__name__) + + +ETCD_DISABLE_TIMEOUT = 0 + + +class EtcdAction(object): + GET = 'get' + SET = 'set' + EXPIRE = 'expire' + UPDATE = 'update' + DELETE = 'delete' + CREATE = 'create' + COMPARE_AND_SWAP = 'compareAndSwap' + COMPARE_AND_DELETE = 'compareAndDelete' + + +class EphemeralBuilderManager(BaseManager): + """ Build manager implementation for the Enterprise Registry. """ + _executors = { + 'popen': PopenExecutor, + 'ec2': EC2Executor, + } + + _etcd_client_klass = etcd.Client + + def __init__(self, *args, **kwargs): + self._shutting_down = False + + self._manager_config = None + self._async_thread_executor = None + self._etcd_client = None + + self._etcd_realm_prefix = None + self._etcd_builder_prefix = None + + self._component_to_job = {} + self._job_uuid_to_component = {} + self._component_to_builder = {} + + self._executor = None + + # Map of etcd keys being watched to the tasks watching them + self._watch_tasks = {} + + super(EphemeralBuilderManager, self).__init__(*args, **kwargs) + + def _watch_etcd(self, etcd_key, change_callback, recursive=True): + watch_task_key = (etcd_key, recursive) + def callback_wrapper(changed_key_future): + if watch_task_key not in self._watch_tasks or self._watch_tasks[watch_task_key].done(): + self._watch_etcd(etcd_key, change_callback) + + if changed_key_future.cancelled(): + # Due to lack of interest, tomorrow has been cancelled + return + + try: + etcd_result = changed_key_future.result() + except (ReadTimeoutError, ProtocolError): + return + + change_callback(etcd_result) + + if not self._shutting_down: + watch_future = self._etcd_client.watch(etcd_key, recursive=recursive, + timeout=ETCD_DISABLE_TIMEOUT) + watch_future.add_done_callback(callback_wrapper) + logger.debug('Scheduling watch of key: %s%s', etcd_key, '/*' if recursive else '') + self._watch_tasks[watch_task_key] = async(watch_future) + + def _handle_builder_expiration(self, etcd_result): + if etcd_result.action == EtcdAction.EXPIRE: + # Handle the expiration + logger.debug('Builder expired, clean up the old build node') + job_metadata = json.loads(etcd_result._prev_node.value) + + if 'builder_id' in job_metadata: + logger.info('Terminating expired build node.') + async(self._executor.stop_builder(job_metadata['builder_id'])) + + def _handle_realm_change(self, etcd_result): + if etcd_result.action == EtcdAction.CREATE: + # We must listen on the realm created by ourselves or another worker + realm_spec = json.loads(etcd_result.value) + self._register_realm(realm_spec) + + elif etcd_result.action == EtcdAction.DELETE or etcd_result.action == EtcdAction.EXPIRE: + # We must stop listening for new connections on the specified realm, if we did not get the + # connection + realm_spec = json.loads(etcd_result._prev_node.value) + build_job = BuildJob(AttrDict(realm_spec['job_queue_item'])) + component = self._job_uuid_to_component.pop(build_job.job_details['build_uuid'], None) + if component is not None: + # We were not the manager which the worker connected to, remove the bookkeeping for it + logger.debug('Unregistering unused component on realm: %s', realm_spec['realm']) + del self._component_to_job[component] + del self._component_to_builder[component] + self.unregister_component(component) + + else: + logger.warning('Unexpected action (%s) on realm key: %s', etcd_result.action, etcd_result.key) + + def _register_realm(self, realm_spec): + logger.debug('Registering realm with manager: %s', realm_spec['realm']) + component = self.register_component(realm_spec['realm'], BuildComponent, + token=realm_spec['token']) + build_job = BuildJob(AttrDict(realm_spec['job_queue_item'])) + self._component_to_job[component] = build_job + self._component_to_builder[component] = realm_spec['builder_id'] + self._job_uuid_to_component[build_job.job_details['build_uuid']] = component + + @coroutine + def _register_existing_realms(self): + try: + all_realms = yield From(self._etcd_client.read(self._etcd_realm_prefix, recursive=True)) + for realm in all_realms.children: + if not realm.dir: + self._register_realm(json.loads(realm.value)) + except KeyError: + # no realms have been registered yet + pass + + def initialize(self, manager_config): + logger.debug('Calling initialize') + self._manager_config = manager_config + + executor_klass = self._executors.get(manager_config.get('EXECUTOR', ''), PopenExecutor) + self._executor = executor_klass(manager_config.get('EXECUTOR_CONFIG', {}), + self.manager_hostname) + + etcd_host = self._manager_config.get('ETCD_HOST', '127.0.0.1') + etcd_port = self._manager_config.get('ETCD_PORT', 2379) + etcd_auth = self._manager_config.get('ETCD_CERT_AND_KEY', None) + etcd_ca_cert = self._manager_config.get('ETCD_CA_CERT', None) + etcd_protocol = 'http' if etcd_auth is None else 'https' + logger.debug('Connecting to etcd on %s:%s', etcd_host, etcd_port) + + worker_threads = self._manager_config.get('ETCD_WORKER_THREADS', 5) + self._async_thread_executor = ThreadPoolExecutor(worker_threads) + self._etcd_client = AsyncWrapper(self._etcd_client_klass(host=etcd_host, port=etcd_port, + cert=etcd_auth, ca_cert=etcd_ca_cert, + protocol=etcd_protocol), + executor=self._async_thread_executor) + + self._etcd_builder_prefix = self._manager_config.get('ETCD_BUILDER_PREFIX', 'building/') + self._watch_etcd(self._etcd_builder_prefix, self._handle_builder_expiration) + + self._etcd_realm_prefix = self._manager_config.get('ETCD_REALM_PREFIX', 'realm/') + self._watch_etcd(self._etcd_realm_prefix, self._handle_realm_change) + + # Load components for all realms currently known to the cluster + async(self._register_existing_realms()) + + def setup_time(self): + setup_time = self._manager_config.get('MACHINE_SETUP_TIME', 300) + return setup_time + + def shutdown(self): + logger.debug('Shutting down worker.') + self._shutting_down = True + + for (etcd_key, _), task in self._watch_tasks.items(): + if not task.done(): + logger.debug('Canceling watch task for %s', etcd_key) + task.cancel() + + if self._async_thread_executor is not None: + logger.debug('Shutting down thread pool executor.') + self._async_thread_executor.shutdown() + + @coroutine + def schedule(self, build_job): + build_uuid = build_job.job_details['build_uuid'] + logger.debug('Calling schedule with job: %s', build_uuid) + + # Check if there are worker slots avialable by checking the number of jobs in etcd + allowed_worker_count = self._manager_config.get('ALLOWED_WORKER_COUNT', 1) + try: + building = yield From(self._etcd_client.read(self._etcd_builder_prefix, recursive=True)) + workers_alive = sum(1 for child in building.children if not child.dir) + except KeyError: + workers_alive = 0 + + logger.debug('Total jobs: %s', workers_alive) + + if workers_alive >= allowed_worker_count: + logger.info('Too many workers alive, unable to start new worker. %s >= %s', workers_alive, + allowed_worker_count) + raise Return(False) + + job_key = self._etcd_job_key(build_job) + + # First try to take a lock for this job, meaning we will be responsible for its lifeline + realm = str(uuid.uuid4()) + token = str(uuid.uuid4()) + ttl = self.setup_time() + expiration = datetime.utcnow() + timedelta(seconds=ttl) + + machine_max_expiration = self._manager_config.get('MACHINE_MAX_TIME', 7200) + max_expiration = datetime.utcnow() + timedelta(seconds=machine_max_expiration) + + payload = { + 'expiration': calendar.timegm(expiration.timetuple()), + 'max_expiration': calendar.timegm(max_expiration.timetuple()), + } + + try: + yield From(self._etcd_client.write(job_key, json.dumps(payload), prevExist=False, ttl=ttl)) + except KeyError: + # The job was already taken by someone else, we are probably a retry + logger.error('Job already exists in etcd, are timeouts misconfigured or is the queue broken?') + raise Return(False) + + logger.debug('Starting builder with executor: %s', self._executor) + builder_id = yield From(self._executor.start_builder(realm, token, build_uuid)) + + # Store the builder in etcd associated with the job id + payload['builder_id'] = builder_id + yield From(self._etcd_client.write(job_key, json.dumps(payload), prevExist=True, ttl=ttl)) + + # Store the realm spec which will allow any manager to accept this builder when it connects + realm_spec = json.dumps({ + 'realm': realm, + 'token': token, + 'builder_id': builder_id, + 'job_queue_item': build_job.job_item, + }) + try: + yield From(self._etcd_client.write(self._etcd_realm_key(realm), realm_spec, prevExist=False, + ttl=ttl)) + except KeyError: + logger.error('Realm already exists in etcd. UUID collision or something is very very wrong.') + raise Return(False) + + raise Return(True) + + @coroutine + def build_component_ready(self, build_component): + try: + # Clean up the bookkeeping for allowing any manager to take the job + job = self._component_to_job.pop(build_component) + del self._job_uuid_to_component[job.job_details['build_uuid']] + yield From(self._etcd_client.delete(self._etcd_realm_key(build_component.builder_realm))) + + logger.debug('Sending build %s to newly ready component on realm %s', + job.job_details['build_uuid'], build_component.builder_realm) + yield From(build_component.start_build(job)) + except KeyError: + logger.debug('Builder is asking for more work, but work already completed') + + def build_component_disposed(self, build_component, timed_out): + logger.debug('Calling build_component_disposed.') + self.unregister_component(build_component) + + @coroutine + def job_completed(self, build_job, job_status, build_component): + logger.debug('Calling job_completed with status: %s', job_status) + + # Kill the ephmeral builder + yield From(self._executor.stop_builder(self._component_to_builder.pop(build_component))) + + # Release the lock in etcd + job_key = self._etcd_job_key(build_job) + yield From(self._etcd_client.delete(job_key)) + + self.job_complete_callback(build_job, job_status) + + @coroutine + def job_heartbeat(self, build_job): + # Extend the deadline in etcd + job_key = self._etcd_job_key(build_job) + build_job_metadata_response = yield From(self._etcd_client.read(job_key)) + build_job_metadata = json.loads(build_job_metadata_response.value) + + max_expiration = datetime.utcfromtimestamp(build_job_metadata['max_expiration']) + max_expiration_remaining = max_expiration - datetime.utcnow() + max_expiration_sec = max(0, int(max_expiration_remaining.total_seconds())) + + ttl = min(self.heartbeat_period_sec * 2, max_expiration_sec) + new_expiration = datetime.utcnow() + timedelta(seconds=ttl) + + payload = { + 'expiration': calendar.timegm(new_expiration.timetuple()), + 'builder_id': build_job_metadata['builder_id'], + 'max_expiration': build_job_metadata['max_expiration'], + } + + yield From(self._etcd_client.write(job_key, json.dumps(payload), ttl=ttl)) + + self.job_heartbeat_callback(build_job) + + def _etcd_job_key(self, build_job): + """ Create a key which is used to track a job in etcd. + """ + return os.path.join(self._etcd_builder_prefix, build_job.job_details['build_uuid']) + + def _etcd_realm_key(self, realm): + """ Create a key which is used to track an incoming connection on a realm. + """ + return os.path.join(self._etcd_realm_prefix, realm) + + def num_workers(self): + """ Return the number of workers we're managing locally. + """ + return len(self._component_to_builder) diff --git a/buildman/manager/executor.py b/buildman/manager/executor.py new file mode 100644 index 000000000..035d5cdf8 --- /dev/null +++ b/buildman/manager/executor.py @@ -0,0 +1,238 @@ +import logging +import os +import uuid +import threading +import boto.ec2 +import requests +import cachetools + +from jinja2 import FileSystemLoader, Environment +from trollius import coroutine, From, Return, get_event_loop +from functools import partial + +from buildman.asyncutil import AsyncWrapper +from container_cloud_config import CloudConfigContext + + +logger = logging.getLogger(__name__) + + +ONE_HOUR = 60*60 + +ENV = Environment(loader=FileSystemLoader('buildman/templates')) +TEMPLATE = ENV.get_template('cloudconfig.yaml') +CloudConfigContext().populate_jinja_environment(ENV) + +class ExecutorException(Exception): + """ Exception raised when there is a problem starting or stopping a builder. + """ + pass + + +class BuilderExecutor(object): + def __init__(self, executor_config, manager_hostname): + self.executor_config = executor_config + self.manager_hostname = manager_hostname + + """ Interface which can be plugged into the EphemeralNodeManager to provide a strategy for + starting and stopping builders. + """ + @coroutine + def start_builder(self, realm, token, build_uuid): + """ Create a builder with the specified config. Returns a unique id which can be used to manage + the builder. + """ + raise NotImplementedError + + @coroutine + def stop_builder(self, builder_id): + """ Stop a builder which is currently running. + """ + raise NotImplementedError + + def get_manager_websocket_url(self): + return 'ws://{0}:' + + def generate_cloud_config(self, realm, token, coreos_channel, manager_hostname, + quay_username=None, quay_password=None): + if quay_username is None: + quay_username = self.executor_config['QUAY_USERNAME'] + + if quay_password is None: + quay_password = self.executor_config['QUAY_PASSWORD'] + + return TEMPLATE.render( + realm=realm, + token=token, + quay_username=quay_username, + quay_password=quay_password, + manager_hostname=manager_hostname, + coreos_channel=coreos_channel, + worker_tag=self.executor_config['WORKER_TAG'], + ) + + +class EC2Executor(BuilderExecutor): + """ Implementation of BuilderExecutor which uses libcloud to start machines on a variety of cloud + providers. + """ + COREOS_STACK_URL = 'http://%s.release.core-os.net/amd64-usr/current/coreos_production_ami_hvm.txt' + + def __init__(self, *args, **kwargs): + self._loop = get_event_loop() + super(EC2Executor, self).__init__(*args, **kwargs) + + def _get_conn(self): + """ Creates an ec2 connection which can be used to manage instances. + """ + return AsyncWrapper(boto.ec2.connect_to_region( + self.executor_config['EC2_REGION'], + aws_access_key_id=self.executor_config['AWS_ACCESS_KEY'], + aws_secret_access_key=self.executor_config['AWS_SECRET_KEY'], + )) + + @classmethod + @cachetools.ttl_cache(ttl=ONE_HOUR) + def _get_coreos_ami(cls, ec2_region, coreos_channel): + """ Retrieve the CoreOS AMI id from the canonical listing. + """ + stack_list_string = requests.get(EC2Executor.COREOS_STACK_URL % coreos_channel).text + stack_amis = dict([stack.split('=') for stack in stack_list_string.split('|')]) + return stack_amis[ec2_region] + + @coroutine + def start_builder(self, realm, token, build_uuid): + region = self.executor_config['EC2_REGION'] + channel = self.executor_config.get('COREOS_CHANNEL', 'stable') + get_ami_callable = partial(self._get_coreos_ami, region, channel) + coreos_ami = yield From(self._loop.run_in_executor(None, get_ami_callable)) + user_data = self.generate_cloud_config(realm, token, channel, self.manager_hostname) + + logger.debug('Generated cloud config: %s', user_data) + + ec2_conn = self._get_conn() + + ssd_root_ebs = boto.ec2.blockdevicemapping.BlockDeviceType( + size=32, + volume_type='gp2', + delete_on_termination=True, + ) + block_devices = boto.ec2.blockdevicemapping.BlockDeviceMapping() + block_devices['/dev/xvda'] = ssd_root_ebs + + interface = boto.ec2.networkinterface.NetworkInterfaceSpecification( + subnet_id=self.executor_config['EC2_VPC_SUBNET_ID'], + groups=self.executor_config['EC2_SECURITY_GROUP_IDS'], + associate_public_ip_address=True, + ) + interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(interface) + + reservation = yield From(ec2_conn.run_instances( + coreos_ami, + instance_type=self.executor_config['EC2_INSTANCE_TYPE'], + key_name=self.executor_config.get('EC2_KEY_NAME', None), + user_data=user_data, + instance_initiated_shutdown_behavior='terminate', + block_device_map=block_devices, + network_interfaces=interfaces, + )) + + if not reservation.instances: + raise ExecutorException('Unable to spawn builder instance.') + elif len(reservation.instances) != 1: + raise ExecutorException('EC2 started wrong number of instances!') + + launched = AsyncWrapper(reservation.instances[0]) + yield From(launched.add_tags({ + 'Name': 'Quay Ephemeral Builder', + 'Realm': realm, + 'Token': token, + 'BuildUUID': build_uuid, + })) + raise Return(launched.id) + + @coroutine + def stop_builder(self, builder_id): + ec2_conn = self._get_conn() + terminated_instances = yield From(ec2_conn.terminate_instances([builder_id])) + if builder_id not in [si.id for si in terminated_instances]: + raise ExecutorException('Unable to terminate instance: %s' % builder_id) + + +class PopenExecutor(BuilderExecutor): + """ Implementation of BuilderExecutor which uses Popen to fork a quay-builder process. + """ + def __init__(self, executor_config, manager_hostname): + self._jobs = {} + + super(PopenExecutor, self).__init__(executor_config, manager_hostname) + + """ Executor which uses Popen to fork a quay-builder process. + """ + @coroutine + def start_builder(self, realm, token, build_uuid): + # Now start a machine for this job, adding the machine id to the etcd information + logger.debug('Forking process for build') + import subprocess + builder_env = { + 'TOKEN': token, + 'REALM': realm, + 'ENDPOINT': 'ws://localhost:8787', + 'DOCKER_TLS_VERIFY': os.environ.get('DOCKER_TLS_VERIFY', ''), + 'DOCKER_CERT_PATH': os.environ.get('DOCKER_CERT_PATH', ''), + 'DOCKER_HOST': os.environ.get('DOCKER_HOST', ''), + } + + logpipe = LogPipe(logging.INFO) + spawned = subprocess.Popen('/Users/jake/bin/quay-builder', stdout=logpipe, stderr=logpipe, + env=builder_env) + + builder_id = str(uuid.uuid4()) + self._jobs[builder_id] = (spawned, logpipe) + logger.debug('Builder spawned with id: %s', builder_id) + raise Return(builder_id) + + @coroutine + def stop_builder(self, builder_id): + if builder_id not in self._jobs: + raise ExecutorException('Builder id not being tracked by executor.') + + logger.debug('Killing builder with id: %s', builder_id) + spawned, logpipe = self._jobs[builder_id] + + if spawned.poll() is None: + spawned.kill() + logpipe.close() + + +class LogPipe(threading.Thread): + """ Adapted from http://codereview.stackexchange.com/a/17959 + """ + def __init__(self, level): + """Setup the object with a logger and a loglevel + and start the thread + """ + threading.Thread.__init__(self) + self.daemon = False + self.level = level + self.fd_read, self.fd_write = os.pipe() + self.pipe_reader = os.fdopen(self.fd_read) + self.start() + + def fileno(self): + """Return the write file descriptor of the pipe + """ + return self.fd_write + + def run(self): + """Run the thread, logging everything. + """ + for line in iter(self.pipe_reader.readline, ''): + logging.log(self.level, line.strip('\n')) + + self.pipe_reader.close() + + def close(self): + """Close the write end of the pipe. + """ + os.close(self.fd_write) diff --git a/buildman/server.py b/buildman/server.py index 3863406f2..855afc212 100644 --- a/buildman/server.py +++ b/buildman/server.py @@ -1,5 +1,6 @@ import logging import trollius +import json from autobahn.asyncio.wamp import RouterFactory, RouterSessionFactory from autobahn.asyncio.websocket import WampWebSocketServerFactory @@ -8,11 +9,15 @@ from autobahn.wamp import types from aiowsgi import create_server as create_wsgi_server from flask import Flask from threading import Event +from trollius.tasks import Task from trollius.coroutines import From -from datetime import datetime, timedelta +from datetime import timedelta +from buildman.enums import BuildJobResult, BuildServerStatus +from buildman.jobutil.buildstatus import StatusHandler from buildman.jobutil.buildjob import BuildJob, BuildJobLoadException -from data.queue import WorkQueue +from data import database +from app import app, build_metrics logger = logging.getLogger(__name__) @@ -21,27 +26,21 @@ TIMEOUT_PERIOD_MINUTES = 20 JOB_TIMEOUT_SECONDS = 300 MINIMUM_JOB_EXTENSION = timedelta(minutes=2) -WEBSOCKET_PORT = 8787 -CONTROLLER_PORT = 8686 - -class BuildJobResult(object): - """ Build job result enum """ - INCOMPLETE = 'incomplete' - COMPLETE = 'complete' - ERROR = 'error' +HEARTBEAT_PERIOD_SEC = 30 class BuilderServer(object): """ Server which handles both HTTP and WAMP requests, managing the full state of the build controller. """ - def __init__(self, server_hostname, queue, build_logs, user_files, lifecycle_manager_klass): + def __init__(self, registry_hostname, queue, build_logs, user_files, lifecycle_manager_klass, + lifecycle_manager_config, manager_hostname): self._loop = None - self._current_status = 'starting' + self._current_status = BuildServerStatus.STARTING self._current_components = [] self._job_count = 0 self._session_factory = RouterSessionFactory(RouterFactory()) - self._server_hostname = server_hostname + self._registry_hostname = registry_hostname self._queue = queue self._build_logs = build_logs self._user_files = user_files @@ -49,11 +48,14 @@ class BuilderServer(object): self._register_component, self._unregister_component, self._job_heartbeat, - self._job_complete + self._job_complete, + manager_hostname, + HEARTBEAT_PERIOD_SEC, ) + self._lifecycle_manager_config = lifecycle_manager_config self._shutdown_event = Event() - self._current_status = 'running' + self._current_status = BuildServerStatus.RUNNING self._register_controller() @@ -63,22 +65,41 @@ class BuilderServer(object): @controller_app.route('/status') def status(): - return server._current_status + metrics = server._queue.get_metrics(require_transaction=False) + (running_count, available_not_running_count, available_count) = metrics + + workers = [component for component in server._current_components + if component.kind() == 'builder'] + + data = { + 'status': server._current_status, + 'running_local': server._job_count, + 'running_total': running_count, + 'workers': len(workers), + 'job_total': available_count + running_count + } + + return json.dumps(data) self._controller_app = controller_app - def run(self, host, ssl=None): + def run(self, host, websocket_port, controller_port, ssl=None): logger.debug('Initializing the lifecycle manager') - self._lifecycle_manager.initialize() + self._lifecycle_manager.initialize(self._lifecycle_manager_config) logger.debug('Initializing all members of the event loop') loop = trollius.get_event_loop() - trollius.Task(self._initialize(loop, host, ssl)) - logger.debug('Starting server on port %s, with controller on port %s', WEBSOCKET_PORT, - CONTROLLER_PORT) + logger.debug('Starting server on port %s, with controller on port %s', websocket_port, + controller_port) + + TASKS = [ + Task(self._initialize(loop, host, websocket_port, controller_port, ssl)), + Task(self._queue_metrics_updater()), + ] + try: - loop.run_forever() + loop.run_until_complete(trollius.wait(TASKS)) except KeyboardInterrupt: pass finally: @@ -86,7 +107,7 @@ class BuilderServer(object): def close(self): logger.debug('Requested server shutdown') - self._current_status = 'shutting_down' + self._current_status = BuildServerStatus.SHUTDOWN self._lifecycle_manager.shutdown() self._shutdown_event.wait() logger.debug('Shutting down server') @@ -102,7 +123,7 @@ class BuilderServer(object): component.parent_manager = self._lifecycle_manager component.build_logs = self._build_logs component.user_files = self._user_files - component.server_hostname = self._server_hostname + component.registry_hostname = self._registry_hostname self._current_components.append(component) self._session_factory.add(component) @@ -116,32 +137,34 @@ class BuilderServer(object): self._session_factory.remove(component) def _job_heartbeat(self, build_job): - WorkQueue.extend_processing(build_job.job_item(), seconds_from_now=JOB_TIMEOUT_SECONDS, - retry_count=1, minimum_extension=MINIMUM_JOB_EXTENSION) + self._queue.extend_processing(build_job.job_item, seconds_from_now=JOB_TIMEOUT_SECONDS, + minimum_extension=MINIMUM_JOB_EXTENSION) def _job_complete(self, build_job, job_status): if job_status == BuildJobResult.INCOMPLETE: - self._queue.incomplete(build_job.job_item(), restore_retry=True, retry_after=30) - elif job_status == BuildJobResult.ERROR: - self._queue.incomplete(build_job.job_item(), restore_retry=False) + self._queue.incomplete(build_job.job_item, restore_retry=False, retry_after=30) else: - self._queue.complete(build_job.job_item()) + self._queue.complete(build_job.job_item) self._job_count = self._job_count - 1 - if self._current_status == 'shutting_down' and not self._job_count: + if self._current_status == BuildServerStatus.SHUTDOWN and not self._job_count: self._shutdown_event.set() - # TODO(jschorr): check for work here? + build_metrics.report_completion_status(job_status) @trollius.coroutine def _work_checker(self): - while self._current_status == 'running': - logger.debug('Checking for more work') + while self._current_status == BuildServerStatus.RUNNING: + with database.CloseForLongOperation(app.config): + yield From(trollius.sleep(WORK_CHECK_TIMEOUT)) + + logger.debug('Checking for more work for %d active workers', + self._lifecycle_manager.num_workers()) + job_item = self._queue.get(processing_time=self._lifecycle_manager.setup_time()) if job_item is None: logger.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT) - yield From(trollius.sleep(WORK_CHECK_TIMEOUT)) continue try: @@ -149,20 +172,28 @@ class BuilderServer(object): except BuildJobLoadException as irbe: logger.exception(irbe) self._queue.incomplete(job_item, restore_retry=False) + continue logger.debug('Build job found. Checking for an avaliable worker.') - if self._lifecycle_manager.schedule(build_job, self._loop): + scheduled = yield From(self._lifecycle_manager.schedule(build_job)) + if scheduled: + status_handler = StatusHandler(self._build_logs, build_job.repo_build.uuid) + status_handler.set_phase('build-scheduled') + self._job_count = self._job_count + 1 logger.debug('Build job scheduled. Running: %s', self._job_count) else: logger.debug('All workers are busy. Requeuing.') self._queue.incomplete(job_item, restore_retry=True, retry_after=0) - yield From(trollius.sleep(WORK_CHECK_TIMEOUT)) - + @trollius.coroutine + def _queue_metrics_updater(self): + while self._current_status == BuildServerStatus.RUNNING: + yield From(trollius.sleep(30)) + self._queue.update_metrics() @trollius.coroutine - def _initialize(self, loop, host, ssl=None): + def _initialize(self, loop, host, websocket_port, controller_port, ssl=None): self._loop = loop # Create the WAMP server. @@ -170,8 +201,8 @@ class BuilderServer(object): transport_factory.setProtocolOptions(failByDrop=True) # Initialize the controller server and the WAMP server - create_wsgi_server(self._controller_app, loop=loop, host=host, port=CONTROLLER_PORT, ssl=ssl) - yield From(loop.create_server(transport_factory, host, WEBSOCKET_PORT, ssl=ssl)) + create_wsgi_server(self._controller_app, loop=loop, host=host, port=controller_port, ssl=ssl) + yield From(loop.create_server(transport_factory, host, websocket_port, ssl=ssl)) # Initialize the work queue checker. yield From(self._work_checker()) diff --git a/buildman/templates/cloudconfig.yaml b/buildman/templates/cloudconfig.yaml new file mode 100644 index 000000000..51bb2f090 --- /dev/null +++ b/buildman/templates/cloudconfig.yaml @@ -0,0 +1,31 @@ +#cloud-config + +ssh_authorized_keys: +- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCC0m+hVmyR3vn/xoxJe9+atRWBxSK+YXgyufNVDMcb7H00Jfnc341QH3kDVYZamUbhVh/nyc2RP7YbnZR5zORFtgOaNSdkMYrPozzBvxjnvSUokkCCWbLqXDHvIKiR12r+UTSijPJE/Yk702Mb2ejAFuae1C3Ec+qKAoOCagDjpQ3THyb5oaKE7VPHdwCWjWIQLRhC+plu77ObhoXIFJLD13gCi01L/rp4mYVCxIc2lX5A8rkK+bZHnIZwWUQ4t8SIjWxIaUo0FE7oZ83nKuNkYj5ngmLHQLY23Nx2WhE9H6NBthUpik9SmqQPtVYbhIG+bISPoH9Xs8CLrFb0VRjz Joey's Mac +- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCo6FhAP7mFFOAzM91gtaKW7saahtaN4lur42FMMztz6aqUycIltCmvxo+3FmrXgCG30maMNU36Vm1+9QRtVQEd+eRuoIWP28t+8MT01Fh4zPuE2Wca3pOHSNo3X81FfWJLzmwEHiQKs9HPQqUhezR9PcVWVkbMyAzw85c0UycGmHGFNb0UiRd9HFY6XbgbxhZv/mvKLZ99xE3xkOzS1PNsdSNvjUKwZR7pSUPqNS5S/1NXyR4GhFTU24VPH/bTATOv2ATH+PSzsZ7Qyz9UHj38tKC+ALJHEDJ4HXGzobyOUP78cHGZOfCB5FYubq0zmOudAjKIAhwI8XTFvJ2DX1P3 jimmyzelinskie +- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNvw8qo9m8np7yQ/Smv/oklM8bo8VyNRZriGYBDuolWDL/mZpYCQnZJXphQo7RFdNABYistikjJlBuuwUohLf2uSq0iKoFa2TgwI43wViWzvuzU4nA02/ITD5BZdmWAFNyIoqeB50Ol4qUgDwLAZ+7Kv7uCi6chcgr9gTi99jY3GHyZjrMiXMHGVGi+FExFuzhVC2drKjbz5q6oRfQeLtNfG4psl5GU3MQU6FkX4fgoCx0r9R48/b7l4+TT7pWblJQiRfeldixu6308vyoTUEHasdkU3/X0OTaGz/h5XqTKnGQc6stvvoED3w+L3QFp0H5Z8sZ9stSsitmCBrmbcKZ jakemoshenko + +write_files: +- path: /root/overrides.list + permission: '0644' + content: | + REALM={{ realm }} + TOKEN={{ token }} + SERVER=wss://{{ manager_hostname }} + +coreos: + update: + reboot-strategy: off + group: {{ coreos_channel }} + + units: + {{ dockersystemd('quay-builder', + 'quay.io/coreos/registry-build-worker', + quay_username, + quay_password, + worker_tag, + extra_args='--net=host --privileged --env-file /root/overrides.list -v /var/run/docker.sock:/var/run/docker.sock -v /usr/share/ca-certificates:/etc/ssl/certs', + exec_stop_post=['/bin/sh -xc "/bin/sleep 120; /usr/bin/systemctl --no-block poweroff"'], + flattened=True, + restart_policy='no' + ) | indent(4) }} diff --git a/conf/gunicorn_local.py b/conf/gunicorn_local.py index aa16e63ec..49a30682d 100644 --- a/conf/gunicorn_local.py +++ b/conf/gunicorn_local.py @@ -1,7 +1,7 @@ bind = '0.0.0.0:5000' workers = 2 worker_class = 'gevent' -timeout = 2000 daemon = False -logconfig = 'conf/logging.conf' +logconfig = 'conf/logging_debug.conf' pythonpath = '.' +preload_app = True diff --git a/conf/gunicorn_registry.py b/conf/gunicorn_registry.py index 4f7bb37f2..944608868 100644 --- a/conf/gunicorn_registry.py +++ b/conf/gunicorn_registry.py @@ -1,7 +1,6 @@ bind = 'unix:/tmp/gunicorn_registry.sock' workers = 8 worker_class = 'gevent' -timeout = 2000 logconfig = 'conf/logging.conf' pythonpath = '.' preload_app = True diff --git a/conf/gunicorn_verbs.py b/conf/gunicorn_verbs.py index eaf8041df..f329a8cbe 100644 --- a/conf/gunicorn_verbs.py +++ b/conf/gunicorn_verbs.py @@ -1,6 +1,5 @@ bind = 'unix:/tmp/gunicorn_verbs.sock' workers = 4 -timeout = 2000 logconfig = 'conf/logging.conf' pythonpath = '.' preload_app = True diff --git a/conf/gunicorn_web.py b/conf/gunicorn_web.py index bdfa8001a..cb9f78d24 100644 --- a/conf/gunicorn_web.py +++ b/conf/gunicorn_web.py @@ -1,7 +1,6 @@ bind = 'unix:/tmp/gunicorn_web.sock' workers = 2 worker_class = 'gevent' -timeout = 30 logconfig = 'conf/logging.conf' pythonpath = '.' preload_app = True diff --git a/conf/hosted-http-base.conf b/conf/hosted-http-base.conf index c3e910e8f..fa5994e6f 100644 --- a/conf/hosted-http-base.conf +++ b/conf/hosted-http-base.conf @@ -1,3 +1,5 @@ +# vim: ft=nginx + server { listen 80 default_server; server_name _; diff --git a/conf/http-base.conf b/conf/http-base.conf index 1eb0b6170..d525b3dd3 100644 --- a/conf/http-base.conf +++ b/conf/http-base.conf @@ -1,3 +1,5 @@ +# vim: ft=nginx + types_hash_max_size 2048; include /usr/local/nginx/conf/mime.types.default; @@ -30,4 +32,4 @@ upstream build_manager_controller_server { upstream build_manager_websocket_server { server localhost:8787; -} \ No newline at end of file +} diff --git a/conf/init/dockerfilebuild/log/run b/conf/init/dockerfilebuild/log/run deleted file mode 100755 index c971f6159..000000000 --- a/conf/init/dockerfilebuild/log/run +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec svlogd /var/log/dockerfilebuild/ \ No newline at end of file diff --git a/conf/init/dockerfilebuild/run b/conf/init/dockerfilebuild/run deleted file mode 100755 index b557a2823..000000000 --- a/conf/init/dockerfilebuild/run +++ /dev/null @@ -1,6 +0,0 @@ -#! /bin/bash - -sv start tutumdocker || exit 1 - -cd / -venv/bin/python -m workers.dockerfilebuild \ No newline at end of file diff --git a/conf/init/gunicorn_registry/run b/conf/init/gunicorn_registry/run index a0a09f5a2..3c88fd0e3 100755 --- a/conf/init/gunicorn_registry/run +++ b/conf/init/gunicorn_registry/run @@ -3,6 +3,6 @@ echo 'Starting gunicon' cd / -venv/bin/gunicorn -c conf/gunicorn_registry.py registry:application +nice -n 10 venv/bin/gunicorn -c conf/gunicorn_registry.py registry:application echo 'Gunicorn exited' \ No newline at end of file diff --git a/conf/init/gunicorn_verbs/run b/conf/init/gunicorn_verbs/run index 1cf2ee51c..d76a7adcf 100755 --- a/conf/init/gunicorn_verbs/run +++ b/conf/init/gunicorn_verbs/run @@ -3,6 +3,6 @@ echo 'Starting gunicon' cd / -nice -10 venv/bin/gunicorn -c conf/gunicorn_verbs.py verbs:application +nice -n 10 venv/bin/gunicorn -c conf/gunicorn_verbs.py verbs:application echo 'Gunicorn exited' \ No newline at end of file diff --git a/conf/init/tutumdocker/log/run b/conf/init/tutumdocker/log/run deleted file mode 100755 index dbabad38b..000000000 --- a/conf/init/tutumdocker/log/run +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec svlogd /var/log/tutumdocker/ \ No newline at end of file diff --git a/conf/init/tutumdocker/run b/conf/init/tutumdocker/run deleted file mode 100755 index 9221134b9..000000000 --- a/conf/init/tutumdocker/run +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash - -# First, make sure that cgroups are mounted correctly. -CGROUP=/sys/fs/cgroup - -[ -d $CGROUP ] || - mkdir $CGROUP - -mountpoint -q $CGROUP || - mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || { - echo "Could not make a tmpfs mount. Did you use -privileged?" - exit 1 - } - -if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security -then - mount -t securityfs none /sys/kernel/security || { - echo "Could not mount /sys/kernel/security." - echo "AppArmor detection and -privileged mode might break." - } -fi - -# Mount the cgroup hierarchies exactly as they are in the parent system. -for SUBSYS in $(cut -d: -f2 /proc/1/cgroup) -do - [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS - mountpoint -q $CGROUP/$SUBSYS || - mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS - - # The two following sections address a bug which manifests itself - # by a cryptic "lxc-start: no ns_cgroup option specified" when - # trying to start containers withina container. - # The bug seems to appear when the cgroup hierarchies are not - # mounted on the exact same directories in the host, and in the - # container. - - # Named, control-less cgroups are mounted with "-o name=foo" - # (and appear as such under /proc//cgroup) but are usually - # mounted on a directory named "foo" (without the "name=" prefix). - # Systemd and OpenRC (and possibly others) both create such a - # cgroup. To avoid the aforementioned bug, we symlink "foo" to - # "name=foo". This shouldn't have any adverse effect. - echo $SUBSYS | grep -q ^name= && { - NAME=$(echo $SUBSYS | sed s/^name=//) - ln -s $SUBSYS $CGROUP/$NAME - } - - # Likewise, on at least one system, it has been reported that - # systemd would mount the CPU and CPU accounting controllers - # (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu" - # but on a directory called "cpu,cpuacct" (note the inversion - # in the order of the groups). This tries to work around it. - [ $SUBSYS = cpuacct,cpu ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct -done - -# Note: as I write those lines, the LXC userland tools cannot setup -# a "sub-container" properly if the "devices" cgroup is not in its -# own hierarchy. Let's detect this and issue a warning. -grep -q :devices: /proc/1/cgroup || - echo "WARNING: the 'devices' cgroup should be in its own hierarchy." -grep -qw devices /proc/1/cgroup || - echo "WARNING: it looks like the 'devices' cgroup is not mounted." - -# Now, close extraneous file descriptors. -pushd /proc/self/fd >/dev/null -for FD in * -do - case "$FD" in - # Keep stdin/stdout/stderr - [012]) - ;; - # Nuke everything else - *) - eval exec "$FD>&-" - ;; - esac -done -popd >/dev/null - - -# If a pidfile is still around (for example after a container restart), -# delete it so that docker can start. -rm -rf /var/run/docker.pid - -chmod 777 /var/lib/lxc -chmod 777 /var/lib/docker - - -# If we were given a PORT environment variable, start as a simple daemon; -# otherwise, spawn a shell as well -if [ "$PORT" ] -then - exec docker -d -H 0.0.0.0:$PORT -else - docker -d -D -e lxc 2>&1 -fi \ No newline at end of file diff --git a/conf/logging.conf b/conf/logging.conf index d009f08ee..317803a24 100644 --- a/conf/logging.conf +++ b/conf/logging.conf @@ -1,5 +1,5 @@ [loggers] -keys=root, gunicorn.error, gunicorn.access, application.profiler, boto, werkzeug +keys=root [handlers] keys=console @@ -7,39 +7,9 @@ keys=console [formatters] keys=generic -[logger_application.profiler] -level=DEBUG -handlers=console -propagate=0 -qualname=application.profiler - [logger_root] -level=DEBUG -handlers=console - -[logger_boto] level=INFO handlers=console -propagate=0 -qualname=boto - -[logger_werkzeug] -level=DEBUG -handlers=console -propagate=0 -qualname=werkzeug - -[logger_gunicorn.error] -level=INFO -handlers=console -propagate=1 -qualname=gunicorn.error - -[logger_gunicorn.access] -level=INFO -handlers=console -propagate=0 -qualname=gunicorn.access [handler_console] class=StreamHandler diff --git a/conf/logging_debug.conf b/conf/logging_debug.conf new file mode 100644 index 000000000..01a3c8fbb --- /dev/null +++ b/conf/logging_debug.conf @@ -0,0 +1,21 @@ +[loggers] +keys=root + +[handlers] +keys=console + +[formatters] +keys=generic + +[logger_root] +level=DEBUG +handlers=console + +[handler_console] +class=StreamHandler +formatter=generic +args=(sys.stdout, ) + +[formatter_generic] +format=%(asctime)s [%(process)d] [%(levelname)s] [%(name)s] %(message)s +class=logging.Formatter diff --git a/conf/nginx-nossl.conf b/conf/nginx-nossl.conf index fbcce63c0..13c5d73b2 100644 --- a/conf/nginx-nossl.conf +++ b/conf/nginx-nossl.conf @@ -1,14 +1,12 @@ +# vim: ft=nginx + include root-base.conf; -worker_processes 2; - -user root nogroup; - -daemon off; - http { include http-base.conf; + include rate-limiting.conf; + server { include server-base.conf; diff --git a/conf/nginx.conf b/conf/nginx.conf index e208d30e0..792b58faf 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -1,16 +1,14 @@ +# vim: ft=nginx + include root-base.conf; -worker_processes 2; - -user root nogroup; - -daemon off; - http { include http-base.conf; include hosted-http-base.conf; + include rate-limiting.conf; + server { include server-base.conf; @@ -24,4 +22,20 @@ http { ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; ssl_prefer_server_ciphers on; } + + server { + include proxy-protocol.conf; + + include proxy-server-base.conf; + + listen 8443 default proxy_protocol; + + ssl on; + ssl_certificate ./stack/ssl.cert; + ssl_certificate_key ./stack/ssl.key; + ssl_session_timeout 5m; + ssl_protocols SSLv3 TLSv1; + ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; + ssl_prefer_server_ciphers on; + } } diff --git a/conf/proxy-protocol.conf b/conf/proxy-protocol.conf new file mode 100644 index 000000000..5897f1839 --- /dev/null +++ b/conf/proxy-protocol.conf @@ -0,0 +1,8 @@ +# vim: ft=nginx + +set_real_ip_from 0.0.0.0/0; +real_ip_header proxy_protocol; +log_format elb_pp '$proxy_protocol_addr - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent"'; +access_log /var/log/nginx/nginx.access.log elb_pp; diff --git a/conf/proxy-server-base.conf b/conf/proxy-server-base.conf new file mode 100644 index 000000000..6230dbfd8 --- /dev/null +++ b/conf/proxy-server-base.conf @@ -0,0 +1,87 @@ +# vim: ft=nginx + +client_body_temp_path /var/log/nginx/client_body 1 2; +server_name _; + +keepalive_timeout 5; + +if ($args ~ "_escaped_fragment_") { + rewrite ^ /snapshot$uri; +} + +proxy_set_header X-Forwarded-For $proxy_protocol_addr; +proxy_set_header X-Forwarded-Proto $scheme; +proxy_set_header Host $http_host; +proxy_redirect off; + +proxy_set_header Transfer-Encoding $http_transfer_encoding; + +location / { + proxy_pass http://web_app_server; + + limit_req zone=webapp burst=25 nodelay; +} + +location /realtime { + proxy_pass http://web_app_server; + proxy_buffering off; + proxy_request_buffering off; +} + +location /v1/repositories/ { + proxy_buffering off; + + proxy_request_buffering off; + + proxy_pass http://registry_app_server; + proxy_temp_path /var/log/nginx/proxy_temp 1 2; + + client_max_body_size 20G; + + limit_req zone=repositories burst=5 nodelay; +} + +location /v1/ { + proxy_buffering off; + + proxy_request_buffering off; + + proxy_pass http://registry_app_server; + proxy_temp_path /var/log/nginx/proxy_temp 1 2; + + client_max_body_size 20G; +} + +location /c1/ { + proxy_buffering off; + + proxy_request_buffering off; + + proxy_pass http://verbs_app_server; + proxy_temp_path /var/log/nginx/proxy_temp 1 2; + + limit_req zone=api burst=5 nodelay; +} + +location /static/ { + # checks for static file, if not found proxy to app + alias /static/; +} + +location /v1/_ping { + add_header Content-Type text/plain; + add_header X-Docker-Registry-Version 0.6.0; + add_header X-Docker-Registry-Standalone 0; + return 200 'true'; +} + +location ~ ^/b1/controller(/?)(.*) { + proxy_pass http://build_manager_controller_server/$2; +} + +location ~ ^/b1/socket(/?)(.*) { + proxy_pass http://build_manager_websocket_server/$2; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; +} diff --git a/conf/rate-limiting.conf b/conf/rate-limiting.conf new file mode 100644 index 000000000..3e2538510 --- /dev/null +++ b/conf/rate-limiting.conf @@ -0,0 +1,7 @@ +# vim: ft=nginx + +limit_req_zone $proxy_protocol_addr zone=webapp:10m rate=25r/s; +limit_req_zone $proxy_protocol_addr zone=repositories:10m rate=1r/s; +limit_req_zone $proxy_protocol_addr zone=api:10m rate=1r/s; +limit_req_status 429; +limit_req_log_level warn; diff --git a/conf/root-base.conf b/conf/root-base.conf index be8072945..02c004564 100644 --- a/conf/root-base.conf +++ b/conf/root-base.conf @@ -1,7 +1,17 @@ +# vim: ft=nginx + pid /tmp/nginx.pid; error_log /var/log/nginx/nginx.error.log; +worker_processes 2; +worker_priority -10; +worker_rlimit_nofile 10240; + +user root nogroup; + +daemon off; + events { - worker_connections 1024; + worker_connections 10240; accept_mutex off; } diff --git a/conf/server-base.conf b/conf/server-base.conf index 5b06b76c5..4122a99eb 100644 --- a/conf/server-base.conf +++ b/conf/server-base.conf @@ -1,3 +1,5 @@ +# vim: ft=nginx + client_body_temp_path /var/log/nginx/client_body 1 2; server_name _; @@ -33,7 +35,6 @@ location /v1/ { proxy_request_buffering off; proxy_pass http://registry_app_server; - proxy_read_timeout 2000; proxy_temp_path /var/log/nginx/proxy_temp 1 2; client_max_body_size 20G; @@ -45,7 +46,6 @@ location /c1/ { proxy_request_buffering off; proxy_pass http://verbs_app_server; - proxy_read_timeout 2000; proxy_temp_path /var/log/nginx/proxy_temp 1 2; } @@ -63,7 +63,6 @@ location /v1/_ping { location ~ ^/b1/controller(/?)(.*) { proxy_pass http://build_manager_controller_server/$2; - proxy_read_timeout 2000; } location ~ ^/b1/socket(/?)(.*) { diff --git a/config.py b/config.py index 39a257b15..4bb44fca2 100644 --- a/config.py +++ b/config.py @@ -36,7 +36,6 @@ def getFrontendVisibleConfig(config_dict): class DefaultConfig(object): # Flask config - SECRET_KEY = 'a36c9d7d-25a9-4d3f-a586-3d2f8dc40a83' JSONIFY_PRETTYPRINT_REGULAR = False SESSION_COOKIE_SECURE = False @@ -48,8 +47,9 @@ class DefaultConfig(object): AVATAR_KIND = 'local' - REGISTRY_TITLE = 'Quay.io' - REGISTRY_TITLE_SHORT = 'Quay.io' + REGISTRY_TITLE = 'CoreOS Enterprise Registry' + REGISTRY_TITLE_SHORT = 'Enterprise Registry' + CONTACT_INFO = [ 'mailto:support@quay.io', 'irc://chat.freenode.net:6665/quayio', @@ -132,6 +132,9 @@ class DefaultConfig(object): # Super user config. Note: This MUST BE an empty list for the default config. SUPER_USERS = [] + # Feature Flag: Whether super users are supported. + FEATURE_SUPER_USERS = True + # Feature Flag: Whether billing is required. FEATURE_BILLING = False @@ -147,9 +150,6 @@ class DefaultConfig(object): # Feature flag, whether to enable olark chat FEATURE_OLARK_CHAT = False - # Feature Flag: Whether super users are supported. - FEATURE_SUPER_USERS = False - # Feature Flag: Whether to support GitHub build triggers. FEATURE_GITHUB_BUILD = False @@ -187,3 +187,14 @@ class DefaultConfig(object): # For enterprise: MAXIMUM_REPOSITORY_USAGE = 20 + + # System logs. + SYSTEM_LOGS_PATH = "/var/log/" + SYSTEM_SERVICE_LOGS_PATH = "/var/log/%s/current" + SYSTEM_SERVICES_PATH = "conf/init/" + + # Services that should not be shown in the logs view. + SYSTEM_SERVICE_BLACKLIST = [] + + # Temporary tag expiration in seconds, this may actually be longer based on GC policy + PUSH_TEMP_TAG_EXPIRATION_SEC = 60 * 60 diff --git a/data/database.py b/data/database.py index e687a67f2..81377d980 100644 --- a/data/database.py +++ b/data/database.py @@ -1,14 +1,15 @@ import string import logging import uuid +import time from random import SystemRandom from datetime import datetime -from peewee import (Proxy, MySQLDatabase, SqliteDatabase, PostgresqlDatabase, fn, CharField, - BooleanField, IntegerField, DateTimeField, ForeignKeyField, TextField, - BigIntegerField) +from peewee import * from data.read_slave import ReadSlaveModel from sqlalchemy.engine.url import make_url + +from data.read_slave import ReadSlaveModel from util.names import urn_generator @@ -31,6 +32,16 @@ SCHEME_RANDOM_FUNCTION = { 'postgresql+psycopg2': fn.Random, } +def real_for_update(query): + return query.for_update() + +def null_for_update(query): + return query + +SCHEME_SPECIALIZED_FOR_UPDATE = { + 'sqlite': null_for_update, +} + class CallableProxy(Proxy): def __call__(self, *args, **kwargs): if self.obj is None: @@ -70,6 +81,15 @@ class UseThenDisconnect(object): db = Proxy() read_slave = Proxy() db_random_func = CallableProxy() +db_for_update = CallableProxy() + + +def validate_database_url(url, connect_timeout=5): + driver = _db_from_url(url, { + 'connect_timeout': connect_timeout + }) + driver.connect() + driver.close() def _db_from_url(url, db_kwargs): @@ -84,6 +104,10 @@ def _db_from_url(url, db_kwargs): if parsed_url.password: db_kwargs['password'] = parsed_url.password + # Note: sqlite does not support connect_timeout. + if parsed_url.drivername == 'sqlite' and 'connect_timeout' in db_kwargs: + del db_kwargs['connect_timeout'] + return SCHEME_DRIVERS[parsed_url.drivername](parsed_url.database, **db_kwargs) @@ -95,6 +119,8 @@ def configure(config_object): parsed_write_uri = make_url(write_db_uri) db_random_func.initialize(SCHEME_RANDOM_FUNCTION[parsed_write_uri.drivername]) + db_for_update.initialize(SCHEME_SPECIALIZED_FOR_UPDATE.get(parsed_write_uri.drivername, + real_for_update)) read_slave_uri = config_object.get('DB_READ_SLAVE_URI', None) if read_slave_uri is not None: @@ -113,6 +139,9 @@ def uuid_generator(): return str(uuid.uuid4()) +_get_epoch_timestamp = lambda: int(time.time()) + + def close_db_filter(_): if not db.is_closed(): logger.debug('Disconnecting from database.') @@ -124,8 +153,9 @@ def close_db_filter(_): class QuayUserField(ForeignKeyField): - def __init__(self, allows_robots=False, *args, **kwargs): + def __init__(self, allows_robots=False, robot_null_delete=False, *args, **kwargs): self.allows_robots = allows_robots + self.robot_null_delete = robot_null_delete if not 'rel_model' in kwargs: kwargs['rel_model'] = User @@ -151,6 +181,7 @@ class User(BaseModel): invoice_email = BooleanField(default=False) invalid_login_attempts = IntegerField(default=0) last_invalid_login = DateTimeField(default=datetime.utcnow) + removed_tag_expiration_s = IntegerField(default=1209600) # Two weeks def delete_instance(self, recursive=False, delete_nullable=False): # If we are deleting a robot account, only execute the subset of queries necessary. @@ -159,7 +190,11 @@ class User(BaseModel): for query, fk in self.dependencies(search_nullable=True): if isinstance(fk, QuayUserField) and fk.allows_robots: model = fk.model_class - model.delete().where(query).execute() + + if fk.robot_null_delete: + model.update(**{fk.name: None}).where(query).execute() + else: + model.delete().where(query).execute() # Delete the instance itself. super(User, self).delete_instance(recursive=False, delete_nullable=False) @@ -319,6 +354,10 @@ class PermissionPrototype(BaseModel): ) +class AccessTokenKind(BaseModel): + name = CharField(unique=True, index=True) + + class AccessToken(BaseModel): friendly_name = CharField(null=True) code = CharField(default=random_string_generator(length=64), unique=True, @@ -327,6 +366,7 @@ class AccessToken(BaseModel): created = DateTimeField(default=datetime.now) role = ForeignKeyField(Role) temporary = BooleanField(default=True) + kind = ForeignKeyField(AccessTokenKind, null=True) class BuildTriggerService(BaseModel): @@ -368,6 +408,24 @@ class ImageStorageTransformation(BaseModel): name = CharField(index=True, unique=True) +class ImageStorageSignatureKind(BaseModel): + name = CharField(index=True, unique=True) + + +class ImageStorageSignature(BaseModel): + storage = ForeignKeyField(ImageStorage, index=True) + kind = ForeignKeyField(ImageStorageSignatureKind) + signature = TextField(null=True) + uploading = BooleanField(default=True, null=True) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('kind', 'storage'), True), + ) + + class DerivedImageStorage(BaseModel): source = ForeignKeyField(ImageStorage, null=True, related_name='source') derivative = ForeignKeyField(ImageStorage, related_name='derivative') @@ -424,12 +482,15 @@ class RepositoryTag(BaseModel): name = CharField() image = ForeignKeyField(Image) repository = ForeignKeyField(Repository) + lifetime_start_ts = IntegerField(default=_get_epoch_timestamp) + lifetime_end_ts = IntegerField(null=True, index=True) + hidden = BooleanField(default=False) class Meta: database = db read_slaves = (read_slave,) indexes = ( - (('repository', 'name'), True), + (('repository', 'name'), False), ) @@ -441,23 +502,10 @@ class BUILD_PHASE(object): PULLING = 'pulling' BUILDING = 'building' PUSHING = 'pushing' + WAITING = 'waiting' COMPLETE = 'complete' -class RepositoryBuild(BaseModel): - uuid = CharField(default=uuid_generator, index=True) - repository = ForeignKeyField(Repository, index=True) - access_token = ForeignKeyField(AccessToken) - resource_key = CharField(index=True) - job_config = TextField() - phase = CharField(default='waiting') - started = DateTimeField(default=datetime.now) - display_name = CharField() - trigger = ForeignKeyField(RepositoryBuildTrigger, null=True, index=True) - pull_robot = QuayUserField(null=True, related_name='buildpullrobot') - logs_archived = BooleanField(default=False) - - class QueueItem(BaseModel): queue_name = CharField(index=True, max_length=1024) body = TextField() @@ -467,6 +515,21 @@ class QueueItem(BaseModel): retries_remaining = IntegerField(default=5) +class RepositoryBuild(BaseModel): + uuid = CharField(default=uuid_generator, index=True) + repository = ForeignKeyField(Repository, index=True) + access_token = ForeignKeyField(AccessToken) + resource_key = CharField(index=True) + job_config = TextField() + phase = CharField(default=BUILD_PHASE.WAITING) + started = DateTimeField(default=datetime.now) + display_name = CharField() + trigger = ForeignKeyField(RepositoryBuildTrigger, null=True, index=True) + pull_robot = QuayUserField(null=True, related_name='buildpullrobot') + logs_archived = BooleanField(default=False) + queue_item = ForeignKeyField(QueueItem, null=True, index=True) + + class LogEntryKind(BaseModel): name = CharField(index=True, unique=True) @@ -475,7 +538,7 @@ class LogEntry(BaseModel): kind = ForeignKeyField(LogEntryKind, index=True) account = QuayUserField(index=True, related_name='account') performer = QuayUserField(allows_robots=True, index=True, null=True, - related_name='performer') + related_name='performer', robot_null_delete=True) repository = ForeignKeyField(Repository, index=True, null=True) datetime = DateTimeField(default=datetime.now, index=True) ip = CharField(null=True) @@ -566,4 +629,5 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, Notification, ImageStorageLocation, ImageStoragePlacement, ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification, RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage, - TeamMemberInvite, Star] + TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind, + AccessTokenKind, Star] diff --git a/data/migrations/env.py b/data/migrations/env.py index 3b2df5186..108c4c496 100644 --- a/data/migrations/env.py +++ b/data/migrations/env.py @@ -18,7 +18,8 @@ config.set_main_option('sqlalchemy.url', unquote(app.config['DB_URI'])) # Interpret the config file for Python logging. # This line sets up loggers basically. -fileConfig(config.config_file_name) +if config.config_file_name: + fileConfig(config.config_file_name) # add your model's MetaData object here # for 'autogenerate' support diff --git a/data/migrations/migration.sh b/data/migrations/migration.sh index dedaf2445..17901e130 100755 --- a/data/migrations/migration.sh +++ b/data/migrations/migration.sh @@ -2,13 +2,14 @@ set -e DOCKER_IP=`echo $DOCKER_HOST | sed 's/tcp:\/\///' | sed 's/:.*//'` MYSQL_CONFIG_OVERRIDE="{\"DB_URI\":\"mysql+pymysql://root:password@$DOCKER_IP/genschema\"}" +PERCONA_CONFIG_OVERRIDE="{\"DB_URI\":\"mysql+pymysql://root@$DOCKER_IP/genschema\"}" PGSQL_CONFIG_OVERRIDE="{\"DB_URI\":\"postgresql://postgres@$DOCKER_IP/genschema\"}" up_mysql() { # Run a SQL database on port 3306 inside of Docker. docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mysql - # Sleep for 5s to get MySQL get started. + # Sleep for 10s to get MySQL get started. echo 'Sleeping for 10...' sleep 10 @@ -21,6 +22,40 @@ down_mysql() { docker rm mysql } +up_mariadb() { + # Run a SQL database on port 3306 inside of Docker. + docker run --name mariadb -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mariadb + + # Sleep for 10s to get MySQL get started. + echo 'Sleeping for 10...' + sleep 10 + + # Add the database to mysql. + docker run --rm --link mariadb:mariadb mariadb sh -c 'echo "create database genschema" | mysql -h"$MARIADB_PORT_3306_TCP_ADDR" -P"$MARIADB_PORT_3306_TCP_PORT" -uroot -ppassword' +} + +down_mariadb() { + docker kill mariadb + docker rm mariadb +} + +up_percona() { + # Run a SQL database on port 3306 inside of Docker. + docker run --name percona -p 3306:3306 -d dockerfile/percona + + # Sleep for 10s + echo 'Sleeping for 10...' + sleep 10 + + # Add the daabase to mysql. + docker run --rm --link percona:percona dockerfile/percona sh -c 'echo "create database genschema" | mysql -h $PERCONA_PORT_3306_TCP_ADDR' +} + +down_percona() { + docker kill percona + docker rm percona +} + up_postgres() { # Run a SQL database on port 5432 inside of Docker. docker run --name postgres -p 5432:5432 -d postgres @@ -73,6 +108,26 @@ test_migrate $MYSQL_CONFIG_OVERRIDE set -e down_mysql +# Test via MariaDB. +echo '> Starting MariaDB' +up_mariadb + +echo '> Testing Migration (mariadb)' +set +e +test_migrate $MYSQL_CONFIG_OVERRIDE +set -e +down_mariadb + +# Test via Percona. +echo '> Starting Percona' +up_percona + +echo '> Testing Migration (percona)' +set +e +test_migrate $PERCONA_CONFIG_OVERRIDE +set -e +down_percona + # Test via Postgres. echo '> Starting Postgres' up_postgres diff --git a/data/migrations/versions/14fe12ade3df_add_build_queue_item_reference_to_the_.py b/data/migrations/versions/14fe12ade3df_add_build_queue_item_reference_to_the_.py new file mode 100644 index 000000000..5e8d21211 --- /dev/null +++ b/data/migrations/versions/14fe12ade3df_add_build_queue_item_reference_to_the_.py @@ -0,0 +1,30 @@ +"""Add build queue item reference to the repositorybuild table + +Revision ID: 14fe12ade3df +Revises: 5ad999136045 +Create Date: 2015-02-12 16:11:57.814645 + +""" + +# revision identifiers, used by Alembic. +revision = '14fe12ade3df' +down_revision = '5ad999136045' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('repositorybuild', sa.Column('queue_item_id', sa.Integer(), nullable=True)) + op.create_index('repositorybuild_queue_item_id', 'repositorybuild', ['queue_item_id'], unique=False) + op.create_foreign_key(op.f('fk_repositorybuild_queue_item_id_queueitem'), 'repositorybuild', 'queueitem', ['queue_item_id'], ['id']) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(op.f('fk_repositorybuild_queue_item_id_queueitem'), 'repositorybuild', type_='foreignkey') + op.drop_index('repositorybuild_queue_item_id', table_name='repositorybuild') + op.drop_column('repositorybuild', 'queue_item_id') + ### end Alembic commands ### diff --git a/data/migrations/versions/1d2d86d09fcd_actually_remove_the_column.py b/data/migrations/versions/1d2d86d09fcd_actually_remove_the_column.py new file mode 100644 index 000000000..a7942b7d4 --- /dev/null +++ b/data/migrations/versions/1d2d86d09fcd_actually_remove_the_column.py @@ -0,0 +1,37 @@ +"""Actually remove the column access_token_id + +Revision ID: 1d2d86d09fcd +Revises: 14fe12ade3df +Create Date: 2015-02-12 16:27:30.260797 + +""" + +# revision identifiers, used by Alembic. +revision = '1d2d86d09fcd' +down_revision = '14fe12ade3df' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql +from sqlalchemy.exc import InternalError + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + try: + op.drop_constraint(u'fk_logentry_access_token_id_accesstoken', 'logentry', type_='foreignkey') + op.drop_index('logentry_access_token_id', table_name='logentry') + op.drop_column('logentry', 'access_token_id') + except InternalError: + pass + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + try: + op.add_column('logentry', sa.Column('access_token_id', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True)) + op.create_foreign_key(u'fk_logentry_access_token_id_accesstoken', 'logentry', 'accesstoken', ['access_token_id'], ['id']) + op.create_index('logentry_access_token_id', 'logentry', ['access_token_id'], unique=False) + except InternalError: + pass + ### end Alembic commands ### diff --git a/data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py b/data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py new file mode 100644 index 000000000..2f6ff722b --- /dev/null +++ b/data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py @@ -0,0 +1,25 @@ +"""mysql max index lengths + +Revision ID: 228d1af6af1c +Revises: 5b84373e5db +Create Date: 2015-01-06 14:35:24.651424 + +""" + +# revision identifiers, used by Alembic. +revision = '228d1af6af1c' +down_revision = '5b84373e5db' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(tables): + op.drop_index('queueitem_queue_name', table_name='queueitem') + op.create_index('queueitem_queue_name', 'queueitem', ['queue_name'], unique=False, mysql_length=767) + + op.drop_index('image_ancestors', table_name='image') + op.create_index('image_ancestors', 'image', ['ancestors'], unique=False, mysql_length=767) + +def downgrade(tables): + pass diff --git a/data/migrations/versions/3e2d38b52a75_add_access_token_kinds_type.py b/data/migrations/versions/3e2d38b52a75_add_access_token_kinds_type.py new file mode 100644 index 000000000..53d0ae9df --- /dev/null +++ b/data/migrations/versions/3e2d38b52a75_add_access_token_kinds_type.py @@ -0,0 +1,44 @@ +"""Add access token kinds type + +Revision ID: 3e2d38b52a75 +Revises: 1d2d86d09fcd +Create Date: 2015-02-17 12:03:26.422485 + +""" + +# revision identifiers, used by Alembic. +revision = '3e2d38b52a75' +down_revision = '1d2d86d09fcd' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('accesstokenkind', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_accesstokenkind')) + ) + op.create_index('accesstokenkind_name', 'accesstokenkind', ['name'], unique=True) + op.add_column(u'accesstoken', sa.Column('kind_id', sa.Integer(), nullable=True)) + op.create_index('accesstoken_kind_id', 'accesstoken', ['kind_id'], unique=False) + op.create_foreign_key(op.f('fk_accesstoken_kind_id_accesstokenkind'), 'accesstoken', 'accesstokenkind', ['kind_id'], ['id']) + ### end Alembic commands ### + + op.bulk_insert(tables.accesstokenkind, + [ + {'id': 1, 'name':'build-worker'}, + {'id': 2, 'name':'pushpull-token'}, + ]) + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(op.f('fk_accesstoken_kind_id_accesstokenkind'), 'accesstoken', type_='foreignkey') + op.drop_index('accesstoken_kind_id', table_name='accesstoken') + op.drop_column(u'accesstoken', 'kind_id') + op.drop_index('accesstokenkind_name', table_name='accesstokenkind') + op.drop_table('accesstokenkind') + ### end Alembic commands ### diff --git a/data/migrations/versions/4ef04c61fcf9_allow_tags_to_be_marked_as_hidden.py b/data/migrations/versions/4ef04c61fcf9_allow_tags_to_be_marked_as_hidden.py new file mode 100644 index 000000000..e4fc1ea5e --- /dev/null +++ b/data/migrations/versions/4ef04c61fcf9_allow_tags_to_be_marked_as_hidden.py @@ -0,0 +1,26 @@ +"""Allow tags to be marked as hidden. + +Revision ID: 4ef04c61fcf9 +Revises: 509d2857566f +Create Date: 2015-02-18 16:34:16.586129 + +""" + +# revision identifiers, used by Alembic. +revision = '4ef04c61fcf9' +down_revision = '509d2857566f' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('repositorytag', sa.Column('hidden', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false())) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('repositorytag', 'hidden') + ### end Alembic commands ### diff --git a/data/migrations/versions/509d2857566f_track_the_lifetime_start_and_end_for_.py b/data/migrations/versions/509d2857566f_track_the_lifetime_start_and_end_for_.py new file mode 100644 index 000000000..a13ec00d1 --- /dev/null +++ b/data/migrations/versions/509d2857566f_track_the_lifetime_start_and_end_for_.py @@ -0,0 +1,36 @@ +"""Track the lifetime start and end for tags to allow the state of a repository to be rewound. + +Revision ID: 509d2857566f +Revises: 3e2d38b52a75 +Create Date: 2015-02-13 14:35:38.939049 + +""" + +# revision identifiers, used by Alembic. +revision = '509d2857566f' +down_revision = '3e2d38b52a75' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('repositorytag', sa.Column('lifetime_end_ts', sa.Integer(), nullable=True)) + op.add_column('repositorytag', sa.Column('lifetime_start_ts', sa.Integer(), nullable=False, server_default="0")) + op.create_index('repositorytag_lifetime_end_ts', 'repositorytag', ['lifetime_end_ts'], unique=False) + op.drop_index('repositorytag_repository_id_name', table_name='repositorytag') + op.create_index('repositorytag_repository_id_name', 'repositorytag', ['repository_id', 'name'], unique=False) + op.add_column('user', sa.Column('removed_tag_expiration_s', sa.Integer(), nullable=False, server_default="1209600")) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('user', 'removed_tag_expiration_s') + op.drop_index('repositorytag_repository_id_name', table_name='repositorytag') + op.create_index('repositorytag_repository_id_name', 'repositorytag', ['repository_id', 'name'], unique=True) + op.drop_index('repositorytag_lifetime_end_ts', table_name='repositorytag') + op.drop_column('repositorytag', 'lifetime_start_ts') + op.drop_column('repositorytag', 'lifetime_end_ts') + ### end Alembic commands ### diff --git a/data/migrations/versions/5a07499ce53f_set_up_initial_database.py b/data/migrations/versions/5a07499ce53f_set_up_initial_database.py index 0e786de14..c1d082066 100644 --- a/data/migrations/versions/5a07499ce53f_set_up_initial_database.py +++ b/data/migrations/versions/5a07499ce53f_set_up_initial_database.py @@ -53,7 +53,7 @@ def upgrade(tables): op.create_index('queueitem_available', 'queueitem', ['available'], unique=False) op.create_index('queueitem_available_after', 'queueitem', ['available_after'], unique=False) op.create_index('queueitem_processing_expires', 'queueitem', ['processing_expires'], unique=False) - op.create_index('queueitem_queue_name', 'queueitem', ['queue_name'], unique=False) + op.create_index('queueitem_queue_name', 'queueitem', ['queue_name'], unique=False, mysql_length=767) op.create_table('role', sa.Column('id', sa.Integer(), nullable=False), sa.Column('name', sa.String(length=255), nullable=False), @@ -376,7 +376,7 @@ def upgrade(tables): sa.ForeignKeyConstraint(['storage_id'], ['imagestorage.id'], ), sa.PrimaryKeyConstraint('id') ) - op.create_index('image_ancestors', 'image', ['ancestors'], unique=False) + op.create_index('image_ancestors', 'image', ['ancestors'], unique=False, mysql_length=767) op.create_index('image_repository_id', 'image', ['repository_id'], unique=False) op.create_index('image_repository_id_docker_image_id', 'image', ['repository_id', 'docker_image_id'], unique=True) op.create_index('image_storage_id', 'image', ['storage_id'], unique=False) diff --git a/data/migrations/versions/5ad999136045_add_signature_storage.py b/data/migrations/versions/5ad999136045_add_signature_storage.py new file mode 100644 index 000000000..f306c58b8 --- /dev/null +++ b/data/migrations/versions/5ad999136045_add_signature_storage.py @@ -0,0 +1,55 @@ +"""Add signature storage + +Revision ID: 5ad999136045 +Revises: 228d1af6af1c +Create Date: 2015-02-05 15:01:54.989573 + +""" + +# revision identifiers, used by Alembic. +revision = '5ad999136045' +down_revision = '228d1af6af1c' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('imagestoragesignaturekind', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_imagestoragesignaturekind')) + ) + op.create_index('imagestoragesignaturekind_name', 'imagestoragesignaturekind', ['name'], unique=True) + op.create_table('imagestoragesignature', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('storage_id', sa.Integer(), nullable=False), + sa.Column('kind_id', sa.Integer(), nullable=False), + sa.Column('signature', sa.Text(), nullable=True), + sa.Column('uploading', sa.Boolean(), nullable=True), + sa.ForeignKeyConstraint(['kind_id'], ['imagestoragesignaturekind.id'], name=op.f('fk_imagestoragesignature_kind_id_imagestoragesignaturekind')), + sa.ForeignKeyConstraint(['storage_id'], ['imagestorage.id'], name=op.f('fk_imagestoragesignature_storage_id_imagestorage')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_imagestoragesignature')) + ) + op.create_index('imagestoragesignature_kind_id', 'imagestoragesignature', ['kind_id'], unique=False) + op.create_index('imagestoragesignature_kind_id_storage_id', 'imagestoragesignature', ['kind_id', 'storage_id'], unique=True) + op.create_index('imagestoragesignature_storage_id', 'imagestoragesignature', ['storage_id'], unique=False) + ### end Alembic commands ### + + op.bulk_insert(tables.imagestoragetransformation, + [ + {'id': 2, 'name':'aci'}, + ]) + + op.bulk_insert(tables.imagestoragesignaturekind, + [ + {'id': 1, 'name':'gpg2'}, + ]) + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('imagestoragesignature') + op.drop_table('imagestoragesignaturekind') + ### end Alembic commands ### diff --git a/data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py b/data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py new file mode 100644 index 000000000..a117fd2b1 --- /dev/null +++ b/data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py @@ -0,0 +1,24 @@ +"""Convert slack webhook data + +Revision ID: 5b84373e5db +Revises: 1c5b738283a5 +Create Date: 2014-12-16 12:02:55.167744 + +""" + +# revision identifiers, used by Alembic. +revision = '5b84373e5db' +down_revision = '1c5b738283a5' + +from alembic import op +import sqlalchemy as sa + +from util.migrateslackwebhook import run_slackwebhook_migration + + +def upgrade(tables): + run_slackwebhook_migration() + + +def downgrade(tables): + pass diff --git a/data/model/legacy.py b/data/model/legacy.py index a399fb465..f7f5c51f9 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -2,8 +2,10 @@ import bcrypt import logging import dateutil.parser import json +import time from datetime import datetime, timedelta, date +from uuid import uuid4 from data.database import (User, Repository, Image, AccessToken, Role, RepositoryPermission, Visibility, RepositoryTag, EmailConfirmation, FederatedLogin, @@ -14,7 +16,9 @@ from data.database import (User, Repository, Image, AccessToken, Role, Repositor ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification, RepositoryAuthorizedEmail, TeamMemberInvite, DerivedImageStorage, ImageStorageTransformation, random_string_generator, - db, BUILD_PHASE, QuayUserField, Star) + db, BUILD_PHASE, QuayUserField, ImageStorageSignature, QueueItem, + ImageStorageSignatureKind, validate_database_url, db_for_update, + AccessTokenKind, Star) from peewee import JOIN_LEFT_OUTER, fn from util.validation import (validate_username, validate_email, validate_password, INVALID_PASSWORD_MESSAGE) @@ -105,12 +109,15 @@ class TooManyLoginAttemptsException(Exception): self.retry_after = retry_after -def _get_repository(namespace_name, repository_name): - return (Repository - .select(Repository, Namespace) - .join(Namespace, on=(Repository.namespace_user == Namespace.id)) - .where(Namespace.username == namespace_name, Repository.name == repository_name) - .get()) +def _get_repository(namespace_name, repository_name, for_update=False): + query = (Repository + .select(Repository, Namespace) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name)) + if for_update: + query = db_for_update(query) + + return query.get() def hash_password(password, salt=None): @@ -164,8 +171,7 @@ def _create_user(username, email): pass try: - new_user = User.create(username=username, email=email) - return new_user + return User.create(username=username, email=email) except Exception as ex: raise DataModelException(ex.message) @@ -295,6 +301,9 @@ def delete_robot(robot_username): def _list_entity_robots(entity_name): + """ Return the list of robots for the specified entity. This MUST return a query, not a + materialized list so that callers can use db_for_update. + """ return (User .select() .join(FederatedLogin) @@ -901,14 +910,17 @@ def change_password(user, new_password): delete_notifications_by_kind(user, 'password_required') -def change_username(user, new_username): +def change_username(user_id, new_username): (username_valid, username_issue) = validate_username(new_username) if not username_valid: raise InvalidUsernameException('Invalid username %s: %s' % (new_username, username_issue)) with config.app_config['DB_TRANSACTION_FACTORY'](db): + # Reload the user for update + user = db_for_update(User.select().where(User.id == user_id)).get() + # Rename the robots - for robot in _list_entity_robots(user.username): + for robot in db_for_update(_list_entity_robots(user.username)): _, robot_shortname = parse_robot_username(robot.username) new_robot_name = format_robot_username(new_username, robot_shortname) robot.username = new_robot_name @@ -924,6 +936,11 @@ def change_invoice_email(user, invoice_email): user.save() +def change_user_tag_expiration(user, tag_expiration_s): + user.removed_tag_expiration_s = tag_expiration_s + user.save() + + def update_email(user, new_email, auto_verify=False): user.email = new_email user.verified = auto_verify @@ -1087,6 +1104,26 @@ def get_repository(namespace_name, repository_name): return None +def get_image(repo, dockerfile_id): + try: + return Image.get(Image.docker_image_id == dockerfile_id, Image.repository == repo) + except Image.DoesNotExist: + return None + + +def find_child_image(repo, parent_image, command): + try: + return (Image.select() + .join(ImageStorage) + .switch(Image) + .where(Image.ancestors % '%/' + parent_image.id + '/%', + ImageStorage.command == command) + .order_by(ImageStorage.created.desc()) + .get()) + except Image.DoesNotExist: + return None + + def get_repo_image(namespace_name, repository_name, docker_image_id): def limit_to_image_id(query): return query.where(Image.docker_image_id == docker_image_id).limit(1) @@ -1249,9 +1286,9 @@ def _find_or_link_image(existing_image, repository, username, translations, pref storage.locations = {placement.location.name for placement in storage.imagestorageplacement_set} - new_image = Image.create(docker_image_id=existing_image.docker_image_id, - repository=repository, storage=storage, - ancestors=new_image_ancestry) + new_image = Image.create(docker_image_id=existing_image.docker_image_id, + repository=repository, storage=storage, + ancestors=new_image_ancestry) logger.debug('Storing translation %s -> %s', existing_image.id, new_image.id) translations[existing_image.id] = new_image.id @@ -1315,7 +1352,28 @@ def find_create_or_link_image(docker_image_id, repository, username, translation ancestors='/') -def find_or_create_derived_storage(source, transformation_name, preferred_location): +def find_or_create_storage_signature(storage, signature_kind): + found = lookup_storage_signature(storage, signature_kind) + if found is None: + kind = ImageStorageSignatureKind.get(name=signature_kind) + found = ImageStorageSignature.create(storage=storage, kind=kind) + + return found + + +def lookup_storage_signature(storage, signature_kind): + kind = ImageStorageSignatureKind.get(name=signature_kind) + try: + return (ImageStorageSignature + .select() + .where(ImageStorageSignature.storage == storage, + ImageStorageSignature.kind == kind) + .get()) + except ImageStorageSignature.DoesNotExist: + return None + + +def find_derived_storage(source, transformation_name): try: found = (ImageStorage .select(ImageStorage, DerivedImageStorage) @@ -1328,11 +1386,19 @@ def find_or_create_derived_storage(source, transformation_name, preferred_locati found.locations = {placement.location.name for placement in found.imagestorageplacement_set} return found except ImageStorage.DoesNotExist: - logger.debug('Creating storage dervied from source: %s', source.uuid) - trans = ImageStorageTransformation.get(name=transformation_name) - new_storage = _create_storage(preferred_location) - DerivedImageStorage.create(source=source, derivative=new_storage, transformation=trans) - return new_storage + return None + + +def find_or_create_derived_storage(source, transformation_name, preferred_location): + existing = find_derived_storage(source, transformation_name) + if existing is not None: + return existing + + logger.debug('Creating storage dervied from source: %s', source.uuid) + trans = ImageStorageTransformation.get(name=transformation_name) + new_storage = _create_storage(preferred_location) + DerivedImageStorage.create(source=source, derivative=new_storage, transformation=trans) + return new_storage def delete_derived_storage_by_uuid(storage_uuid): @@ -1401,7 +1467,7 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created Image.docker_image_id == docker_image_id)) try: - fetched = query.get() + fetched = db_for_update(query).get() except Image.DoesNotExist: raise DataModelException('No image with specified id and repository') @@ -1489,19 +1555,48 @@ def get_repository_images(namespace_name, repository_name): return _get_repository_images_base(namespace_name, repository_name, lambda q: q) -def list_repository_tags(namespace_name, repository_name): - return (RepositoryTag - .select(RepositoryTag, Image) - .join(Repository) - .join(Namespace, on=(Repository.namespace_user == Namespace.id)) - .switch(RepositoryTag) - .join(Image) - .where(Repository.name == repository_name, Namespace.username == namespace_name)) +def _tag_alive(query): + return query.where((RepositoryTag.lifetime_end_ts >> None) | + (RepositoryTag.lifetime_end_ts > int(time.time()))) + + +def list_repository_tags(namespace_name, repository_name, include_hidden=False): + query = _tag_alive(RepositoryTag + .select(RepositoryTag, Image) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(RepositoryTag) + .join(Image) + .where(Repository.name == repository_name, + Namespace.username == namespace_name)) + + if not include_hidden: + query = query.where(RepositoryTag.hidden == False) + + return query + + +def _garbage_collect_tags(namespace_name, repository_name): + to_delete = (RepositoryTag + .select(RepositoryTag.id) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + ~(RepositoryTag.lifetime_end_ts >> None), + (RepositoryTag.lifetime_end_ts + Namespace.removed_tag_expiration_s) <= + int(time.time()))) + + (RepositoryTag + .delete() + .where(RepositoryTag.id << to_delete) + .execute()) def garbage_collect_repository(namespace_name, repository_name): storage_id_whitelist = {} + _garbage_collect_tags(namespace_name, repository_name) + with config.app_config['DB_TRANSACTION_FACTORY'](db): # TODO (jake): We could probably select this and all the images in a single query using # a different kind of join. @@ -1535,12 +1630,10 @@ def garbage_collect_repository(namespace_name, repository_name): if len(to_remove) > 0: logger.info('Garbage collecting storage for images: %s', to_remove) - garbage_collect_storage(storage_id_whitelist) - - return len(to_remove) + _garbage_collect_storage(storage_id_whitelist) -def garbage_collect_storage(storage_id_whitelist): +def _garbage_collect_storage(storage_id_whitelist): if len(storage_id_whitelist) == 0: return @@ -1632,10 +1725,10 @@ def garbage_collect_storage(storage_id_whitelist): def get_tag_image(namespace_name, repository_name, tag_name): def limit_to_tag(query): - return (query - .switch(Image) - .join(RepositoryTag) - .where(RepositoryTag.name == tag_name)) + return _tag_alive(query + .switch(Image) + .join(RepositoryTag) + .where(RepositoryTag.name == tag_name)) images = _get_repository_images_base(namespace_name, repository_name, limit_to_tag) if not images: @@ -1643,7 +1736,6 @@ def get_tag_image(namespace_name, repository_name, tag_name): else: return images[0] - def get_image_by_id(namespace_name, repository_name, docker_image_id): image = get_repo_image_extended(namespace_name, repository_name, docker_image_id) if not image: @@ -1672,45 +1764,69 @@ def get_parent_images(namespace_name, repository_name, image_obj): def create_or_update_tag(namespace_name, repository_name, tag_name, tag_docker_image_id): - try: - repo = _get_repository(namespace_name, repository_name) - except Repository.DoesNotExist: - raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name)) - try: - image = Image.get(Image.docker_image_id == tag_docker_image_id, Image.repository == repo) - except Image.DoesNotExist: - raise DataModelException('Invalid image with id: %s' % tag_docker_image_id) + with config.app_config['DB_TRANSACTION_FACTORY'](db): + try: + repo = _get_repository(namespace_name, repository_name) + except Repository.DoesNotExist: + raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name)) - try: - tag = RepositoryTag.get(RepositoryTag.repository == repo, RepositoryTag.name == tag_name) - tag.image = image - tag.save() - except RepositoryTag.DoesNotExist: - tag = RepositoryTag.create(repository=repo, image=image, name=tag_name) + try: + image = Image.get(Image.docker_image_id == tag_docker_image_id, Image.repository == repo) + except Image.DoesNotExist: + raise DataModelException('Invalid image with id: %s' % tag_docker_image_id) - return tag + now_ts = int(time.time()) + + try: + # When we move a tag, we really end the timeline of the old one and create a new one + query = _tag_alive(RepositoryTag + .select() + .where(RepositoryTag.repository == repo, RepositoryTag.name == tag_name)) + tag = query.get() + tag.lifetime_end_ts = now_ts + tag.save() + except RepositoryTag.DoesNotExist: + # No tag that needs to be ended + pass + + return RepositoryTag.create(repository=repo, image=image, name=tag_name, + lifetime_start_ts=now_ts) def delete_tag(namespace_name, repository_name, tag_name): - try: - found = (RepositoryTag - .select() - .join(Repository) - .join(Namespace, on=(Repository.namespace_user == Namespace.id)) - .where(Repository.name == repository_name, Namespace.username == namespace_name, - RepositoryTag.name == tag_name) - .get()) + with config.app_config['DB_TRANSACTION_FACTORY'](db): + try: + query = _tag_alive(RepositoryTag + .select(RepositoryTag, Repository) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.name == repository_name, + Namespace.username == namespace_name, + RepositoryTag.name == tag_name)) + found = db_for_update(query).get() + except RepositoryTag.DoesNotExist: + msg = ('Invalid repository tag \'%s\' on repository \'%s/%s\'' % + (tag_name, namespace_name, repository_name)) + raise DataModelException(msg) - except RepositoryTag.DoesNotExist: - msg = ('Invalid repository tag \'%s\' on repository \'%s/%s\'' % - (tag_name, namespace_name, repository_name)) - raise DataModelException(msg) - - found.delete_instance() + found.lifetime_end_ts = int(time.time()) + found.save() -def delete_all_repository_tags(namespace_name, repository_name): +def create_temporary_hidden_tag(repo, image, expiration_s): + """ Create a tag with a defined timeline, that will not appear in the UI or CLI. Returns the name + of the temporary tag. """ + now_ts = int(time.time()) + expire_ts = now_ts + expiration_s + tag_name = str(uuid4()) + RepositoryTag.create(repository=repo, image=image, name=tag_name, lifetime_start_ts=now_ts, + lifetime_end_ts=expire_ts, hidden=True) + return tag_name + + +def purge_all_repository_tags(namespace_name, repository_name): + """ Immediately purge all repository tags without respecting the lifeline procedure """ try: repo = _get_repository(namespace_name, repository_name) except Repository.DoesNotExist: @@ -1825,7 +1941,7 @@ def set_team_repo_permission(team_name, namespace_name, repository_name, def purge_repository(namespace_name, repository_name): # Delete all tags to allow gc to reclaim storage - delete_all_repository_tags(namespace_name, repository_name) + purge_all_repository_tags(namespace_name, repository_name) # Gc to remove the images and storage garbage_collect_repository(namespace_name, repository_name) @@ -1845,10 +1961,14 @@ def get_private_repo_count(username): .count()) -def create_access_token(repository, role): +def create_access_token(repository, role, kind=None, friendly_name=None): role = Role.get(Role.name == role) + kind_ref = None + if kind is not None: + kind_ref = AccessTokenKind.get(AccessTokenKind.name == kind) + new_token = AccessToken.create(repository=repository, temporary=True, - role=role) + role=role, kind=kind_ref, friendly_name=friendly_name) return new_token @@ -1967,10 +2087,10 @@ def create_repository_build(repo, access_token, job_config_obj, dockerfile_id, pull_robot = lookup_robot(pull_robot_name) return RepositoryBuild.create(repository=repo, access_token=access_token, - job_config=json.dumps(job_config_obj), - display_name=display_name, trigger=trigger, - resource_key=dockerfile_id, - pull_robot=pull_robot) + job_config=json.dumps(job_config_obj), + display_name=display_name, trigger=trigger, + resource_key=dockerfile_id, + pull_robot=pull_robot) def get_pull_robot_name(trigger): @@ -2255,11 +2375,20 @@ def delete_user(user): # TODO: also delete any repository data associated -def check_health(): +def check_health(app_config): + # Attempt to connect to the database first. If the DB is not responding, + # using the validate_database_url will timeout quickly, as opposed to + # making a normal connect which will just hang (thus breaking the health + # check). + try: + validate_database_url(app_config['DB_URI'], connect_timeout=3) + except Exception: + logger.exception('Could not connect to the database') + return False + # We will connect to the db, check that it contains some log entry kinds try: - found_count = LogEntryKind.select().count() - return found_count > 0 + return bool(list(LogEntryKind.select().limit(1))) except: return False @@ -2365,6 +2494,32 @@ def confirm_team_invite(code, user): found.delete_instance() return (team, inviter) +def cancel_repository_build(build): + with config.app_config['DB_TRANSACTION_FACTORY'](db): + # Reload the build for update. + try: + build = db_for_update(RepositoryBuild.select().where(RepositoryBuild.id == build.id)).get() + except RepositoryBuild.DoesNotExist: + return False + + if build.phase != BUILD_PHASE.WAITING or not build.queue_item: + return False + + # Load the build queue item for update. + try: + queue_item = db_for_update(QueueItem.select() + .where(QueueItem.id == build.queue_item.id)).get() + except QueueItem.DoesNotExist: + return False + + # Check the queue item. + if not queue_item.available or queue_item.retries_remaining == 0: + return False + + # Delete the queue item and build. + queue_item.delete_instance(recursive=True) + build.delete_instance() + return True def get_repository_usage(): one_month_ago = date.today() - timedelta(weeks=4) diff --git a/data/queue.py b/data/queue.py index aaebcc86b..c1fb871ad 100644 --- a/data/queue.py +++ b/data/queue.py @@ -1,11 +1,17 @@ from datetime import datetime, timedelta -from data.database import QueueItem, db +from data.database import QueueItem, db, db_for_update from util.morecollections import AttrDict MINIMUM_EXTENSION = timedelta(seconds=20) +class NoopWith: + def __enter__(self): + pass + + def __exit__(self, type, value, traceback): + pass class WorkQueue(object): def __init__(self, queue_name, transaction_factory, @@ -31,31 +37,50 @@ class WorkQueue(object): QueueItem.processing_expires > now, QueueItem.queue_name ** name_match_query)) - def _available_jobs(self, now, name_match_query, running_query): + def _available_jobs(self, now, name_match_query): return (QueueItem .select() .where(QueueItem.queue_name ** name_match_query, QueueItem.available_after <= now, ((QueueItem.available == True) | (QueueItem.processing_expires <= now)), - QueueItem.retries_remaining > 0, ~(QueueItem.queue_name << running_query))) + QueueItem.retries_remaining > 0)) + + def _available_jobs_not_running(self, now, name_match_query, running_query): + return (self + ._available_jobs(now, name_match_query) + .where(~(QueueItem.queue_name << running_query))) def _name_match_query(self): return '%s%%' % self._canonical_name([self._queue_name] + self._canonical_name_match_list) - def update_metrics(self): - if self._reporter is None: - return + def _item_by_id_for_update(self, queue_id): + return db_for_update(QueueItem.select().where(QueueItem.id == queue_id)).get() - with self._transaction_factory(db): + def get_metrics(self, require_transaction=True): + guard = self._transaction_factory(db) if require_transaction else NoopWith() + with guard: now = datetime.utcnow() name_match_query = self._name_match_query() running_query = self._running_jobs(now, name_match_query) running_count = running_query.distinct().count() - avialable_query = self._available_jobs(now, name_match_query, running_query) - available_count = avialable_query.select(QueueItem.queue_name).distinct().count() + available_query = self._available_jobs(now, name_match_query) + available_count = available_query.select(QueueItem.queue_name).distinct().count() - self._reporter(self._currently_processing, running_count, running_count + available_count) + available_not_running_query = self._available_jobs_not_running(now, name_match_query, + running_query) + available_not_running_count = (available_not_running_query.select(QueueItem.queue_name) + .distinct().count()) + + return (running_count, available_not_running_count, available_count) + + def update_metrics(self): + if self._reporter is None: + return + + (running_count, available_not_running_count, available_count) = self.get_metrics() + self._reporter(self._currently_processing, running_count, + running_count + available_not_running_count) def put(self, canonical_name_list, message, available_after=0, retries_remaining=5): """ @@ -73,24 +98,31 @@ class WorkQueue(object): params['available_after'] = available_date with self._transaction_factory(db): - QueueItem.create(**params) + return QueueItem.create(**params) def get(self, processing_time=300): """ Get an available item and mark it as unavailable for the default of five - minutes. + minutes. The result of this method must always be composed of simple + python objects which are JSON serializable for network portability reasons. """ now = datetime.utcnow() name_match_query = self._name_match_query() - with self._transaction_factory(db): - running = self._running_jobs(now, name_match_query) - avail = self._available_jobs(now, name_match_query, running) + running = self._running_jobs(now, name_match_query) + avail = self._available_jobs_not_running(now, name_match_query, running) - item = None - try: - db_item = avail.order_by(QueueItem.id).get() + item = None + try: + db_item_candidate = avail.order_by(QueueItem.id).get() + + with self._transaction_factory(db): + still_available_query = (db_for_update(self + ._available_jobs(now, name_match_query) + .where(QueueItem.id == db_item_candidate.id))) + + db_item = still_available_query.get() db_item.available = False db_item.processing_expires = now + timedelta(seconds=processing_time) db_item.retries_remaining -= 1 @@ -99,25 +131,26 @@ class WorkQueue(object): item = AttrDict({ 'id': db_item.id, 'body': db_item.body, + 'retries_remaining': db_item.retries_remaining }) self._currently_processing = True - except QueueItem.DoesNotExist: - self._currently_processing = False + except QueueItem.DoesNotExist: + self._currently_processing = False - # Return a view of the queue item rather than an active db object - return item + # Return a view of the queue item rather than an active db object + return item def complete(self, completed_item): with self._transaction_factory(db): - completed_item_obj = QueueItem.get(QueueItem.id == completed_item.id) + completed_item_obj = self._item_by_id_for_update(completed_item.id) completed_item_obj.delete_instance() self._currently_processing = False def incomplete(self, incomplete_item, retry_after=300, restore_retry=False): with self._transaction_factory(db): retry_date = datetime.utcnow() + timedelta(seconds=retry_after) - incomplete_item_obj = QueueItem.get(QueueItem.id == incomplete_item.id) + incomplete_item_obj = self._item_by_id_for_update(incomplete_item.id) incomplete_item_obj.available_after = retry_date incomplete_item_obj.available = True @@ -126,16 +159,14 @@ class WorkQueue(object): incomplete_item_obj.save() self._currently_processing = False + return incomplete_item_obj.retries_remaining > 0 - @staticmethod - def extend_processing(queue_item, seconds_from_now, retry_count=None, - minimum_extension=MINIMUM_EXTENSION): - new_expiration = datetime.utcnow() + timedelta(seconds=seconds_from_now) + def extend_processing(self, item, seconds_from_now, minimum_extension=MINIMUM_EXTENSION): + with self._transaction_factory(db): + queue_item = self._item_by_id_for_update(item.id) + new_expiration = datetime.utcnow() + timedelta(seconds=seconds_from_now) - # Only actually write the new expiration to the db if it moves the expiration some minimum - if new_expiration - queue_item.processing_expires > minimum_extension: - if retry_count is not None: - queue_item.retries_remaining = retry_count - - queue_item.processing_expires = new_expiration - queue_item.save() \ No newline at end of file + # Only actually write the new expiration to the db if it moves the expiration some minimum + if new_expiration - queue_item.processing_expires > minimum_extension: + queue_item.processing_expires = new_expiration + queue_item.save() diff --git a/data/runmigration.py b/data/runmigration.py new file mode 100644 index 000000000..b06cf861d --- /dev/null +++ b/data/runmigration.py @@ -0,0 +1,20 @@ +import logging + +from alembic.config import Config +from alembic.script import ScriptDirectory +from alembic.environment import EnvironmentContext +from alembic.migration import __name__ as migration_name + +def run_alembic_migration(log_handler=None): + if log_handler: + logging.getLogger(migration_name).addHandler(log_handler) + + config = Config() + config.set_main_option("script_location", "data:migrations") + script = ScriptDirectory.from_config(config) + + def fn(rev, context): + return script._upgrade_revs('head', rev) + + with EnvironmentContext(config, script, fn=fn, destination_rev='head'): + script.run_env() \ No newline at end of file diff --git a/emails/base.html b/emails/base.html index e01343d53..00a89e51d 100644 --- a/emails/base.html +++ b/emails/base.html @@ -4,6 +4,12 @@ {{ subject }} + + {% if action_metadata %} + + {% endif %} +{% endblock %} + {% block body_content %}
-

Privacy Policy

- +

CoreOS Privacy Policy

+

Last Revised: February 2, 2015

+

Welcome to Quay.io from CoreOS, Inc. (“CoreOS”, “we”, “us” or “our”).

+

This privacy policy explains how we collect, use and disclose information about you when you use any of the websites owned or operated by CoreOS (the “Sites”) and any of the online products and services that link to this privacy policy (collectively, the “Services”) or when you otherwise interact with us. By using any of our Services, you consent to our collection, use and disclosure of your information as described in this privacy policy.

+

The Services allow users to store, manage, and retrieve container repositories.

+

We may change this privacy policy from time-to-time. If we make changes, we will notify you by revising the date at the top of the policy and, in some cases, we will provide you with additional notice (such as adding a statement to our homepage or sending you an email notification). We encourage you to review the privacy policy periodically to stay informed about our practices and the ways you can help protect your privacy.

-
What information do we collect?
+
Collection of Information
+ +
Information You Provide to Us
- We collect information from you when you register on our site or subscribe to the service.. - When ordering or registering on our site, as appropriate, you may be asked to enter your: e-mail address, mailing address or credit card information. You may, however, visit the public portion of our site anonymously. + We collect information you directly give us. For example, we collect information about you when you sign up for one of our Services, participate in any interactive features of the Services, fill out a form, give feedback, ideas or submissions about any of the Services, communicate with us via third party social media sites, request customer support or otherwise communicate with us. The types of information we may collect include your email address, username, your credit/debit card information and any other information you choose to provide. For information as to how to restrict the collection of contact information, please see the “Your Choices” section below. If you choose not to provide certain information, we may not be able to provide certain of our Services to you or certain features of our Services may be unavailable or work differently.
-
What do we use your information for?
-
Any of the information we collect from you may be used in one of the following ways: -
    -
  • To personalize your experience(your information helps us to better respond to your individual needs)
    • -
  • - To improve our website
    - (we continually strive to improve our website offerings based on the information and feedback we receive from you)
    • -
  • - To improve customer service
    - (your information helps us to more effectively respond to your customer service requests and support needs) -
    • -
  • - To process transactions
    - Your information, whether public or private, will not be sold, exchanged, transferred, or given to any other company for any reason whatsoever, without your consent, other than for the express purpose of delivering the purchased product or service requested. -
    • -
  • - To send periodic emails
    - The email address you provide for order processing, may be used to send you information and updates pertaining to your order, in addition to receiving occasional company news, updates, related product or service information, etc.
    - Note: If at any time you would like to unsubscribe from receiving future emails, we include detailed unsubscribe instructions at the bottom of each email. -
    • -
-
+
Information We Collect Automatically When You Use the Services
+
+ When you access or use our Services (or certain portions of the Services), we automatically collect certain information about you. This information includes: +
    +
  • Log Information: We log information about your use of the Services, including the type of device you use, access times, IP address, pages viewed, and the page you visited before navigating to one of our Services. We use this information for analytic and product improvement purposes.
    • +
  • Device Information: We collect information about the computer you use to access our Services, including the hardware model, operating system and version and unique device identifiers.
    • +
  • Information Collected by Cookies and Other Tracking Technologies: We use various technologies to collect information, and this may include cookies and web beacons. Cookies are small data files stored on your hard drive or in device memory. Web beacons (also known as “tracking pixels”) are non-visible electronic images. These technologies are used for analytic and product improvement purposes, such as seeing which areas and features of our Services are popular and determining whether an email has been opened and acted upon. For more information about cookies, and how to disable them, please see “Your Choices” below.
    • +
+
-
How do we protect your information?
-
- We implement a variety of security measures to maintain the safety of your personal information when you place an order or enter, submit, or access your personal information. - We offer the use of a secure server. All supplied sensitive/credit information is transmitted via Secure Socket Layer (SSL) technology and then encrypted into our Payment gateway providers database only to be accessible by those authorized with special access rights to such systems, and are required to keep the information confidential. - After a transaction, your private information (credit cards, social security numbers, financials, etc.) will be kept on file for more than 60 days in order to continue subscription billing.. -
-
Do we use cookies?
-
- Yes (Cookies are small files that a site or its service provider transfers to your computers hard drive through your Web browser (if you allow) that enables the sites or service providers systems to recognize your browser and capture and remember certain information - We use cookies to understand and save your preferences for future visits and compile aggregate data about site traffic and site interaction so that we can offer better site experiences and tools in the future. We may contract with third-party service providers to assist us in better understanding our site visitors. These service providers are not permitted to use the information collected on our behalf except to help us conduct and improve our business. -
-
Do we disclose any information to outside parties?
-
- We do not sell, trade, or otherwise transfer to outside parties your personally identifiable information. This does not include trusted third parties who assist us in operating our website, conducting our business, or servicing you, so long as those parties agree to keep this information confidential. We may also release your information when we believe release is appropriate to comply with the law, enforce our site policies, or protect ours or others rights, property, or safety. However, non-personally identifiable visitor information may be provided to other parties for marketing, advertising, or other uses. -
-
Third party links
-
- Occasionally, at our discretion, we may include or offer third party products or services on our website. These third party sites have separate and independent privacy policies. We therefore have no responsibility or liability for the content and activities of these linked sites. Nonetheless, we seek to protect the integrity of our site and welcome any feedback about these sites. -
-
California Online Privacy Protection Act Compliance
-
- Because we value your privacy we have taken the necessary precautions to be in compliance with the California Online Privacy Protection Act. We therefore will not distribute your personal information to outside parties without your consent. - As part of the California Online Privacy Protection Act, all users of our site may make any changes to their information at anytime by logging into the service and modifying their Account Settings and Payment Information. -
-
Childrens Online Privacy Protection Act Compliance
-
- We are in compliance with the requirements of COPPA (Childrens Online Privacy Protection Act), we do not collect any information from anyone under 13 years of age. Our website, products and services are all directed to people who are at least 13 years old or older. -
-
Terms and Conditions
-
- Please also visit our Terms and Conditions section establishing the use, disclaimers, and limitations of liability governing the use of our website at https://quay.io/tos -
-
Your Consent
-
- By using our site, you consent to our privacy policy. -
-
Changes to our Privacy Policy
-
- If we decide to change our privacy policy, we will post those changes on this page. - If you have any questions or concerns about our privacy policy, please direct them to the following email address: - support@quay.io -
+
Information We Collect From Other Sources
+
+ We may also obtain information from other sources and combine that with information we collect through our Services. For example, if you create or log into your account through a site like Google.com or GitHub.com, we will have access to certain information from that site, such as your name, account information and friends lists, in accordance with the authorization procedures determined by these sites. +
+
Use of Information
+
We may use information about you for various purposes, including to: +
    +
  • Provide, deliver, maintain, test and improve our Services;
    • +
  • Send you technical notices, updates, confirmations, security alerts and support and administrative messages;
    • +
  • Respond to your comments, questions and requests and provide customer service;
    • +
  • Communicate with you about products, services, offers, promotions, rewards and events offered by CoreOS and others, and provide news and information we think will be of interest to you;
    • +
  • Monitor and analyze trends, usage and activities in connection with our Services and improve our Services;
    • +
  • Detect, investigate and prevent any suspected breaches of the terms applicable to the use of our Services (including, our Sites); and
    • +
  • Link or combine with information we get from others to help understand your needs and provide you with better service.
    • +
+ CoreOS is based in the United States, and the information we collect is governed by U.S. law. By accessing or using any of our Services or otherwise providing information to us, you consent to the processing and transfer of information in and to the U.S. and other countries. +
+
Sharing of Information
+
+ We may share information about you as follows or as otherwise described in this Privacy Policy: +
    +
  • With vendors, consultants and other service providers who need access to such information to carry out work on our behalf;
    • +
  • In response to a request for information if we believe disclosure is in accordance with any applicable law, regulation or legal process, or as otherwise required by any applicable law, rule or regulation;
    • +
  • If we believe your actions are inconsistent with the spirit or language of our user agreements or policies, or to protect the rights, property and safety of CoreOS or others;
    • +
  • In connection with, or during negotiations of, any financing with respect to CoreOS;
    • +
  • In connection with, or during negotiations of, any merger, sale of CoreOS’ assets or acquisition of all or a portion of our business to another company; and
    • +
  • With your consent or at your direction, including if we notify you through any of the Services that the information you provide will be shared in a particular manner and you provide such information.
    • +
+ We may also share aggregated or anonymized information that does not directly identify you. +
+
Security
+
+ We take reasonable measures to help protect information about you from loss, theft, misuse and unauthorized access, disclosure, alteration and destruction. +
+
Analytics Services
+
+ We may allow others to provide analytics services in connection with the Services (or portions the Services). These entities may use cookies, web beacons and other technologies to collect information about your use of the Services and other websites, including your IP address, web browser, pages viewed, time spent on pages, links clicked and conversion information. We and others may use this information to, among other things, analyze and track data, determine the popularity of certain content, personalize the user experience, and better understand your activity. +
+
Your Choices
+
Account Information
+
+ If you wish to delete your account, please contact support at support@quay.io. Note that we may retain certain information as required by law or for legitimate business purposes as may be necessary to fulfill the purposes identified in the privacy policy. We may also retain cached or archived copies of information (including, location information) about you for a certain period of time. +
+
Cookies
+
+ Most web browsers are set to accept cookies by default. If you prefer, you can usually choose to set your browser to remove or reject browser cookies. Please note that if you choose to remove or reject cookies, this could affect the availability and functionality of certain of the Services. +
+
Promotional Communications
+
+ You may opt out of receiving promotional communications from CoreOS by following the instructions in those communications. If you opt out, we may still send you non-promotional communications, such as those about your account or our ongoing business relations. +
+
Contact Us
+
+ If you have any questions or concerns about this privacy policy or any privacy issues, please email us at partners@coreos.com. +
+
{% endblock %} diff --git a/templates/tos.html b/templates/tos.html index ccb2e11a7..b86128256 100644 --- a/templates/tos.html +++ b/templates/tos.html @@ -8,94 +8,164 @@ {% endblock %} +{% block added_stylesheets %} + +{% endblock %} + {% block body_content %}
-

Terms of Service

-

The following terms and conditions govern all use of the Quay.io website and all content, services and products available at or through the website. The Website is owned and operated by DevTable, LLC. (“DevTable”). The Website is offered subject to your acceptance without modification of all of the terms and conditions contained herein and all other operating rules, policies (including, without limitation, Quay.io’s Privacy Policy) and procedures that may be published from time to time on this Site by DevTable (collectively, the “Agreement”).

-

Please read this Agreement carefully before accessing or using the Website. By accessing or using any part of the web site, you agree to become bound by the terms and conditions of this agreement. If you do not agree to all the terms and conditions of this agreement, then you may not access the Website or use any services. If these terms and conditions are considered an offer by DevTable, acceptance is expressly limited to these terms. The Website is available only to individuals who are at least 13 years old.

+

CoreOS Terms of Service

+

Last Revised: February 5, 2015

+ +

These Quay.io Terms of Service (these “Terms”) apply to the features and functions provided by CoreOS, Inc. (“CoreOS,” “our,” or “we”) via quay.io (the “Site”) (collectively, the “Services”). By accessing or using the Services, you agree to be bound by these Terms. If you do not agree to these Terms, do not use any of the Services. The “Effective Date” of these Terms is the date you first access any of the Services.

+

If you are accessing the Services in your capacity as an employee, consultant or agent of a company (or other entity), you represent that you are an employee, consultant or agent of such company (or other entity) and you have the authority to agree (and be legally bound) on behalf of such company (or other entity) to all of the terms and conditions of these Terms.

+

For the purpose of these Terms, you and, if applicable, such company (or other entity) constitutes “Customer” or “you”.

+

CoreOS reserves the right to change or modify any of the terms and conditions contained in these Terms (or any policy or guideline of CoreOS) at any time and in its sole discretion by providing notice that these Terms have been modified. Such notice may be provided by sending an email, posting a notice on the Site, posting the revised Terms on the Site and revising the date at the top of these Terms or such other form of notice as determined by CoreOS. Any changes or modifications will be effective 30 days after providing notice that these Terms have been modified (the “Notice Period”). Your continued use of any of the Services following the Notice Period will constitute your acceptance of such changes or modifications. Therefore, you should review these Terms whenever you access the Services and at least every 30 days to make sure that you understand the terms and conditions that will apply to your use of the Services.

+

These terms form a binding agreement between you and CoreOS.

+
  1. - Your Quay.io Account. If you create an account on the Website, you are responsible for maintaining the security of your account, and you are fully responsible for all activities that occur under the account and any other actions taken in connection with the account. You must immediately notify DevTable of any unauthorized uses of your account or any other breaches of security. DevTable will not be liable for any acts or omissions by You, including any damages of any kind incurred as a result of such acts or omissions. + Privacy +

    Please see CoreOS’ privacy policy at https://quay.io/privacy for information about how CoreOS collects, uses and discloses information about users of the Site and the Services.

  2. - Responsibility of Contributors. If you share your repository, publish images, code or content, or otherwise make (or allow any third party to make) material available by means of the Website (any such material, “Content”), You are entirely responsible for the content of, and any harm resulting from, that Content. That is the case regardless of whether the Content in question constitutes text, graphics, an audio file, or computer software. By making Content available, you represent and warrant that: + Registration +

    In order to access the Services, you must complete the CoreOS registration form provided via the Site. During the registration process, you must select a CoreOS package which includes: (a) the monthly or annual period during which you can access the Services (the “Subscription Period”); and (b) the monthly or annual fee you must pay to CoreOS in exchange for your rights to the Services (the “Subscription Fees”). All such information is incorporated into these Terms by reference.

    +

    You agree to: (a) provide accurate, current and complete information about you as may be prompted by the registration forms via the Site (“Registration Data”); (b) maintain the security of your password; (c) maintain and promptly update the Registration Data, and any other information you provide to CoreOS, to keep it accurate, current and complete; and (d) accept all risks of unauthorized access to the Registration Data and any other information you provide to CoreOS.

    +

    You are responsible for safeguarding the password that you use to access the Services, and you agree to be fully responsible for activities or transactions that relate to your account and password

    +
    3. +
  3. + Services +

    Subject to the terms and conditions of these Terms, CoreOS grants you a limited, non-transferable, non-exclusive and revocable right and license to access and use the Services.

    +
    4. +
  4. + Restrictions +

    Except as expressly authorized by these Terms, you may not (a) modify, disclose, alter, translate or create derivative works of the Services, (b) license, sublicense, resell, distribute, lease, rent, lend, transfer, assign or otherwise dispose of the Services (or any components thereof), (c) use the Services to store or transmit any viruses, software routines or other code designed to permit unauthorized access, to disable, erase or otherwise harm software, hardware or data, or to perform any other harmful actions, (d) build a competitive product or service, or copy any features or functions of the Services, (e) interfere with or disrupt the integrity or performance of the Services, (f) disclose to any third party any performance information or analysis relating to the Services, (g) remove, alter or obscure any proprietary notices in or on the Services, including copyright notices, or (h) cause or permit any third party to do any of the foregoing.

    +
    5. +
  5. + Your Responsibilities +

    If you share your repository, publish images, code or content, or otherwise make (or allow any third party to make) material available by means of the Site (“Content”), you are entirely responsible for such Content of, and any harm resulting from, that Content. That is the case regardless of whether the Content in question constitutes text, graphics, an audio file, or computer software. By making Content available, you represent and warrant that:

      -
    • - the downloading, copying and use of the Content will not infringe the proprietary rights, including but not limited to the copyright, patent, trademark or trade secret rights, of any third party; -
      • -
    • - if your employer has rights to intellectual property you create, you have either (i) received permission from your employer to post or make available the Content, including but not limited to any software, or (ii) secured from your employer a waiver as to all rights in or to the Content; -
      • -
    • - you have fully complied with any third-party licenses relating to the Content, and have done all things necessary to successfully pass through to end users any required terms; -
      • -
    • - the Content does not contain or install any viruses, worms, malware, Trojan horses or other harmful or destructive content; -
      • -
    • - the Content is not spam, is not randomly-generated, and does not contain unethical or unwanted commercial content designed to drive traffic to third party sites or boost the search engine rankings of third party sites, or to further unlawful acts (such as phishing) or mislead recipients as to the source of the material (such as spoofing); -
      • -
    • - the Content does not contain threats or incite violence, and does not violate the privacy or publicity rights of any third party; -
      • -
    • - your Content is not getting advertised via unwanted electronic messages such as spam links on newsgroups, email lists, other blogs and web sites, and similar unsolicited promotional methods; -
      • -
    • - your Content is not named in a manner that misleads your readers into thinking that you are another person or company. For example, your Content’s URL or name is not the name of a person other than yourself or company other than your own; and -
      • -
    • - you have, in the case of Content that includes computer code, accurately categorized and/or described the type, nature, uses and effects of the materials, whether requested to do so by DevTable or otherwise. -
      • -
    +
  6. the downloading, copying and use of the Content will not infringe, violate or misappropriate any Intellectual Property Rights of any third party;
    7. +
  7. if your employer has rights to intellectual property you create, you have either (a) received permission from your employer to post or make available the Content, including but not limited to any software, or (b) secured from your employer a waiver as to all rights in or to the Content;
    8. +
  8. you have fully complied with any third-party licenses relating to the Content, and have done all things necessary to successfully pass through to end users any required terms;
    9. +
  9. the Content does not contain or install any viruses, worms, malware, Trojan horses or other harmful or destructive content;
    10. +
  10. the Content is not spam, is not randomly-generated, and does not contain unethical or unwanted commercial content designed to drive traffic to third party sites or boost the search engine rankings of third party sites, or to further unlawful acts (such as phishing) or mislead recipients as to the source of the material (such as spoofing);
    11. +
  11. the Content does not contain threats or incite violence, and does not violate the privacy or publicity rights of any third party;
    12. +
  12. your Content is not getting advertised via unwanted electronic messages such as spam links on newsgroups, email lists, other blogs and web sites, and similar unsolicited promotional methods;
    13. +
  13. your Content is not named in a manner that misleads your readers into thinking that you are another person or company. For example, your Content’s URL or name is not the name of a person other than yourself or company other than your own; and
    14. +
  14. you have, in the case of Content that includes computer code, accurately categorized and/or described the type, nature, uses and effects of the materials, whether requested to do so by CoreOS or otherwise.
    15. + +

    By submitting Content or computer code to CoreOS for inclusion in your repositories, you grant CoreOS a world-wide, royalty-free, and non-exclusive license to reproduce, modify, adapt and publish the Content solely for the purpose of providing the services you request. If you delete Content, CoreOS will use reasonable efforts to remove it from the Services, but you acknowledge that caching or references to the Content may not be made immediately unavailable.

    +

    Without limiting any of those representations or warranties, CoreOS has the right (though not the obligation) to, in CoreOS’ sole discretion (a) refuse or remove any content that, in CoreOS’ reasonable opinion, violates any CoreOS policy or is in any way harmful or objectionable, or (b) terminate or deny access to and use of the Site to any individual or entity for any reason, in CoreOS’ sole discretion. CoreOS will have no obligation to provide a refund of any amounts previously paid.

  15. - By submitting Content or computer code to DevTable for inclusion in your Repositories, you grant DevTable a world-wide, royalty-free, and non-exclusive license to reproduce, modify, adapt and publish the Content solely for the purpose of providing the services you request. If you delete Content, DevTable will use reasonable efforts to remove it from the Service, but you acknowledge that caching or references to the Content may not be made immediately unavailable. + Fees and Payment Terms +

    In exchange for your rights to the Services, you will pay to CoreOS the Subscription Fees. The Subscription Fees do not include taxes, and the Subscription Fees are payable in advance in accordance with your Quay.io Plan.

    +

    Unless CoreOS states otherwise, all payments must be made (a) in U.S. Dollars; and (b) by payment card via an authorized CoreOS payment processor. If you pay via a payment card, you hereby (i) authorize CoreOS (or its authorized payment processor) to make automatic recurring charges to your designated payment card number in the applicable amount of the Subscription Fees on an annual or monthly basis (as applicable) for the duration of the Subscription Period, (ii) represent and warrant that you are authorized to use and have fees charged to the payment card number you provide to CoreOS, and (iii) understand that you may withdraw this consent by emailing CoreOS at support@quay.io. Accounts can be canceled at any time in the Plan and Usage section of your Account Settings. No refunds will be issued (unless expressly stated otherwise).

    +

    Notwithstanding any terms to the contrary in these Terms, CoreOS, at its sole discretion, may modify its pricing during any Subscription Period and such modifications will be effective as of the directly subsequent Subscription Period.

    +

    Interest on any late payments will accrue at the rate of 1.5% per month, or the highest rate permitted by law, whichever is lower, from the date such amount is due until the date such amount is paid in full. You will be responsible for, and will pay all sales and similar taxes on, all license fees and similar fees levied upon the provision of the Services provided under these Terms, excluding only taxes based solely on CoreOS’ net income. You will indemnify and hold CoreOS harmless from and against any and all such taxes and related amounts levied upon the provision of the Services and any costs associated with the collection or withholding thereof, including penalties and interest.

  16. - Without limiting any of those representations or warranties, DevTable has the right (though not the obligation) to, in DevTable’s sole discretion (i) refuse or remove any content that, in DevTable’s reasonable opinion, violates any DevTable policy or is in any way harmful or objectionable, or (ii) terminate or deny access to and use of the Website to any individual or entity for any reason, in DevTable’s sole discretion. DevTable will have no obligation to provide a refund of any amounts previously paid. + Disclaimer +

    COREOS DISCLAIMS ANY AND ALL REPRESENTATIONS OR WARRANTIES (EXPRESS OR IMPLIED, ORAL OR WRITTEN) WITH RESPECT TO THESE TERMS, SERVICES AND ANY OPEN SOURCE SOFTWARE (AS DEFINED BELOW), WHETHER ALLEGED TO ARISE BY OPERATION OF LAW, BY REASON OF CUSTOM OR USAGE IN THE TRADE, BY COURSE OF DEALING OR OTHERWISE. NOTWITHSTANDING ANY TERMS TO THE CONTRARY IN THESE TERMS, COMPANY ACKNOWLEDGES AND AGREES THAT COREOS MAY MODIFY THE FEATURES OF THE SERVICES FROM TIME-TO-TIME AT COREOS’ SOLE DISCRETION.

  17. - Payment and Renewal. -
    -
    General Terms.
    -
    Paid services beyond the initial trial are available on the Website (any such services, an “Account”). By maintaining an Account you agree to pay DevTable the monthly or annual subscription fees indicated for that service. Payments will be charged on a pre-pay basis on the day you sign up for a plan and will cover the use of that service for a monthly or annual subscription period as indicated. Account fees are not refundable.
    -
    Automatic Renewal.
    -
    Unless you notify DevTable before the end of the applicable subscription period that you want to cancel an Account, your Account subscription will automatically renew and you authorize us to collect the then-applicable annual or monthly subscription fee for such Account (as well as any taxes) using any credit card or other payment mechanism we have on record for you. Accounts can be canceled at any time in the Payment Information section of your User Settings.
    -
    + Indemnification Obligations +

    You agree, at your sole expense, to defend, indemnify and hold CoreOS (and its directors, officers, employees, consultants and agents) harmless from and against any and all actual or threatened suits, actions, proceedings (at law or in equity), claims, damages, payments, deficiencies, fines, judgments, settlements, liabilities, losses, costs and expenses (including, but not limited to, reasonable attorneys’ fees, costs, penalties, interest and disbursements) for any death, injury, property damage caused by, arising out of, resulting from, attributable to or in any way incidental to any of your Content or any actual or alleged breach of any of your obligations under these Terms (including, but not limited to, any actual or alleged breach of any of your representations or warranties as set forth in these Terms).

  18. - Responsibility of Website Visitors. DevTable has not reviewed, and cannot review, all of the material, including computer software, submitted to the Service, and cannot therefore be responsible for that material’s content, use or effects. By operating the Website, DevTable does not represent or imply that it endorses the material there posted, or that it believes such material to be accurate, useful or non-harmful. You are responsible for taking precautions as necessary to protect yourself and your computer systems from viruses, worms, Trojan horses, and other harmful or destructive content. The Website may contain content that is offensive, indecent, or otherwise objectionable, as well as content containing technical inaccuracies, typographical mistakes, and other errors. The Website may also contain material that violates the privacy or publicity rights, or infringes the intellectual property and other proprietary rights, of third parties, or the downloading, copying or use of which is subject to additional terms and conditions, stated or unstated. DevTable disclaims any responsibility for any harm resulting from the use by visitors of the Website, or from any downloading by those visitors of content there posted.
    19. + Limitation of Liability +

    IN NO EVENT WILL (A) COREOS’ TOTAL LIABILITY ARISING OUT OF OR RELATED TO THESE TERMS EXCEED THE TOTAL AMOUNT PAID BY YOU TO COREOS UNDER THESE TERMS THE SIX MONTHS IMMEDIATELY PRIOR TO THE ACCRUAL OF THE FIRST CLAIM, AND (B) COREOS BE LIABLE TO YOU OR ANY THIRD PARTY FOR ANY LOSS OF PROFITS, LOSS OF USE, LOSS OF REVENUE, LOSS OF GOODWILL, ANY INTERRUPTION OF BUSINESS, OR ANY INDIRECT, SPECIAL, INCIDENTAL, EXEMPLARY, PUNITIVE OR CONSEQUENTIAL DAMAGES OF ANY KIND ARISING OUT OF, OR IN CONNECTION WITH THESE TERMS, WHETHER IN CONTRACT, TORT, STRICT LIABILITY OR OTHERWISE, EVEN IF SUCH PARTY HAS BEEN ADVISED OR IS OTHERWISE AWARE OF THE POSSIBILITY OF SUCH DAMAGES. MULTIPLE CLAIMS WILL NOT EXPAND THIS LIMITATION. THIS SECTION (LIMITATION OF LIABILITY) WILL BE GIVEN FULL EFFECT EVEN IF ANY REMEDY SPECIFIED IN THESE TERMS IS DEEMED TO HAVE FAILED OF ITS ESSENTIAL PURPOSE.

  19. - Content Posted on Other Websites. We have not reviewed, and cannot review, all of the material, including computer software, made available through the websites and webpages to which Quay.io links, and that link to Quay.io. DevTable does not have any control over those non-DevTable websites and webpages, and is not responsible for their contents or their use. By linking to a non-DevTable website or webpage, DevTable does not represent or imply that it endorses such website or webpage. You are responsible for taking precautions as necessary to protect yourself and your computer systems from viruses, worms, Trojan horses, and other harmful or destructive content. DevTable disclaims any responsibility for any harm resulting from your use of non-DevTable websites and webpages.
    20. + Ownership +

    As between the parties and subject to Section 5 (Your Responsibilities), you own all right, title and interest in and to the Content and any and all Intellectual Property Rights (as defined below) embodied in or related to the foregoing. As between the parties and subject to Section 3 (Services), CoreOS owns all right, title and interest in and to the Services and any and all Intellectual Property Rights (as defined below) embodied in or related to the foregoing. CoreOS reserves all rights not expressly granted in these Terms, and no licenses are granted by CoreOS to you or any other party under these Terms, whether by implication, estoppel or otherwise, except as expressly set forth in these Terms. For the purpose of these Terms, “Intellectual Property Rights” means all patents, copyrights, moral rights, trademarks, trade secrets and any other form of intellectual property rights recognized in any jurisdiction, including applications and registrations for any of the foregoing.

  20. - Copyright Infringement and DMCA Policy. As DevTable asks others to respect its intellectual property rights, it respects the intellectual property rights of others. If you believe that material located on or linked to by Quay.io violates your copyright, you are encouraged to notify DevTable in accordance with the provisions of the Digital Millennium Copyright Act (“DMCA”). DevTable will respond to all such notices, including as required or appropriate by removing the infringing material or disabling all links to the infringing material. DevTable will terminate a visitor’s access to and use of the Website if, under appropriate circumstances, the visitor is determined to be a repeat infringer of the copyrights or other intellectual property rights of DevTable or others. In the case of such termination, DevTable will have no obligation to provide a refund of any amounts previously paid to DevTable.
    21. + Term, Termination and Effect of Termination +

    Unless earlier terminated as set forth in these Terms, the term of these Terms commences upon the Effective Date and continues for the Subscription Period, and thereafter the term of these Terms automatically renews for one or more additional Subscription Periods unless a party terminates these Terms with no less than 15 days advance written notice prior to the close of the then-current term. Further, CoreOS may terminate or deny access to and use of the Services if CoreOS reasonably believes you have violate any of the terms or conditions of these Terms. Upon any termination of these Terms, your rights to the Services will immediately cease.

  21. - Intellectual Property. This Agreement does not transfer from DevTable to you any DevTable or third party intellectual property, and all right, title and interest in and to such property will remain (as between the parties) solely with DevTable. DevTable, Quay.io, the Quay.io logo, and all other trademarks, service marks, graphics and logos used in connection with Quay.io, or the Website are trademarks or registered trademarks of DevTable or DevTable’s licensors. Other trademarks, service marks, graphics and logos used in connection with the Website may be the trademarks of other third parties. Your use of the Website grants you no right or license to reproduce or otherwise use any DevTable or third-party trademarks. + Copyright Policy +

    CoreOS users may report content that appears on/via the Site or Services to CoreOS that he/she thinks violates these Terms, and CoreOS may remove such content, suspend or terminate the account of the user who made posted such content and/or take additional action to enforce these Terms against such user.

    +

    Also, in accordance with the Digital Millennium Copyright Act (DMCA) and other applicable law, CoreOS has adopted a policy of terminating, in appropriate circumstances and at our discretion, account holders who are deemed to be repeat infringers. CoreOS also may, at its discretion, limit access to the Services and terminate the accounts of any users who infringe any intellectual property rights of others, whether or not there is any repeat infringement.

    +

    If you think that anything on the Services infringes upon any copyright that you own or control, you may file a notification with CoreOS’ Designated Agent as set forth below:

    + + + + + + +
    Designated Agent:DMCA Agent
    Address of Designated Agent:3043 Mission Street, San Francisco, CA 94110
    Telephone Number of Designated Agent:(800) 774-3507
    Fax Number of Designated Agent:(415) 580-7362
    Email Address of Designated Agent:support@quay.io
    +

    Please see 17 U.S.C. § 512(c)(3) for the requirements of a proper notification. If you knowingly misrepresent that any material or activity is infringing, you may be liable for any damages, including costs and attorneys’ fees, CoreOS or the alleged infringer incurs because we relied on the misrepresentation when removing or disabling access to the material or activity.

  22. - Changes. DevTable reserves the right, at its sole discretion, to modify or replace any part of this Agreement. It is your responsibility to check this Agreement periodically for changes. Your continued use of or access to the Website following the posting of any changes to this Agreement constitutes acceptance of those changes. DevTable may also, in the future, offer new services and/or features through the Website (including, the release of new tools and resources). Such new features and/or services shall be subject to the terms and conditions of this Agreement. + Feedback +

    Any suggestions, comments, or other feedback provided by you to CoreOS with respect to the Services or CoreOS (collectively, “Feedback”) will constitute confidential information of CoreOS. CoreOS will be free to use, disclose, reproduce, license, and otherwise distribute and exploit the Feedback provided to it as it sees fit, entirely without obligation or restriction of any kind, on account of intellectual property rights or otherwise.

  23. - Termination. DevTable may terminate your access to all or any part of the Website at any time, with or without cause, with or without notice, effective immediately. If you wish to terminate this Agreement or your Quay.io account (if you have one), you may simply discontinue using the Website. All provisions of this Agreement which by their nature should survive termination shall survive termination, including, without limitation, ownership provisions, warranty disclaimers, indemnity and limitations of liability. + Links +

    You are granted a limited, non-exclusive right to create a text hyperlink to the Services for noncommercial purposes, provided such link does not portray CoreOS or any of its products and services in a false, misleading, derogatory, or defamatory manner and that the linking site does not contain any material that is offensive, illegal, harassing, or otherwise objectionable. This limited right may be revoked at any time. CoreOS makes no claim or representation regarding, and accepts no responsibility for, the quality, content, nature, or reliability of third-party sites accessible by link from the Services or Site. CoreOS provides these links to you only as a convenience, and the inclusion of any link does not imply affiliation, endorsement, or adoption by CoreOS of the corresponding site or any information contained in (or made available via) that site. When you leave the Site, CoreOS’ terms and policies no longer govern. You should review the applicable terms and policies, including privacy and data-gathering practices, of any site to which you navigate from the Site.

  24. - Disclaimer of Warranties. The Website is provided “as is”. DevTable and its suppliers and licensors hereby disclaim all warranties of any kind, express or implied, including, without limitation, the warranties of merchantability, fitness for a particular purpose and non-infringement. Neither DevTable nor its suppliers and licensors, makes any warranty that the Website will be error free or that access thereto will be continuous or uninterrupted. You understand that you download from, or otherwise obtain content or services through, the Website at your own discretion and risk. + Trademarks +

    CoreOS’ name, trademarks, logos, and any other CoreOS product, service name, or slogan included in the Site are property of CoreOS and may not be copied, imitated, or used (in whole or in part) without CoreOS’ prior written consent. The look and feel of the Site, including all custom graphics, button icons, and scripts constitute service marks, trademarks, or trade dress of CoreOS and may not be copied, imitated, or used (in whole or in part) without CoreOS’ prior written consent. All other trademarks, registered trademarks, product names, and company names or logos mentioned in the Site (“Third-Party Trademarks”) are the property of their respective owners, and the use of such Third-Party Trademarks inures to the benefit of each owner. The use of such Third-Party Trademarks is intended to denote interoperability and does not constitute an affiliation by CoreOS and its licensors with such company or an endorsement or approval by such company of CoreOS or its licensors or their respective products or services.

  25. - Limitation of Liability. In no event will DevTable, or its suppliers or licensors, be liable with respect to any subject matter of this agreement under any contract, negligence, strict liability or other legal or equitable theory for: (i) any special, incidental or consequential damages; (ii) the cost of procurement for substitute products or services; (iii) for interruption of use or loss or corruption of data; or (iv) for any amounts that exceed the fees paid by you to DevTable under this agreement during the twelve (12) month period prior to the cause of action. DevTable shall have no liability for any failure or delay due to matters beyond their reasonable control. The foregoing shall not apply to the extent prohibited by applicable law. -
    26. -
  26. - General Representation and Warranty. You represent and warrant that (i) your use of the Website will be in strict accordance with the Quay.io Privacy Policy, with this Agreement and with all applicable laws and regulations (including without limitation any local laws or regulations in your country, state, city, or other governmental area, regarding online conduct and acceptable content, and including all applicable laws regarding the transmission of technical data exported from the United States or the country in which you reside) and (ii) your use of the Website will not infringe or misappropriate the intellectual property rights of any third party. -
    27. -
  27. - Indemnification. You agree to indemnify and hold harmless DevTable, its contractors, and its licensors, and their respective directors, officers, employees and agents from and against any and all claims and expenses, including attorneys’ fees, arising out of your use of the Website, including but not limited to your violation of this Agreement. -
    28. -
  28. - Miscellaneous. This Agreement constitutes the entire agreement between DevTable and you concerning the subject matter hereof, and they may only be modified by a written amendment signed by an authorized executive of DevTable, or by the posting by DevTable of a revised version. Except to the extent applicable law, if any, provides otherwise, this Agreement, any access to or use of the Website will be governed by the laws of the state of New York, U.S.A., excluding its conflict of law provisions, and the proper venue for any disputes arising out of or relating to any of the same will be the state and federal courts located in New York County, New York. The prevailing party in any action or proceeding to enforce this Agreement shall be entitled to costs and attorneys’ fees. If any part of this Agreement is held invalid or unenforceable, that part will be construed to reflect the parties’ original intent, and the remaining portions will remain in full force and effect. A waiver by either party of any term or condition of this Agreement or any breach thereof, in any one instance, will not waive such term or condition or any subsequent breach thereof. You may assign your rights under this Agreement to any party that consents to, and agrees to be bound by, its terms and conditions; DevTable may assign its rights under this Agreement without condition. This Agreement will be binding upon and will inure to the benefit of the parties, their successors and permitted assigns. + General Provisions +

    +

    +
    Entire Agreement
    +
    + These Terms (together with all terms incorporated in by reference) are the complete and exclusive statement of the mutual understanding of the parties and supersedes and cancels all previous written and oral agreements and communications relating to the subject matter of these Terms. +
    +
    Governing Law and Venue
    +
    + These Terms will be governed by and construed in accordance with the laws of the State of California applicable to agreements made and to be entirely performed within the State of California, without resort to its conflict of law provisions. The federal court in San Mateo County, California will be the jurisdiction in which any suits should be filed if they relate to these Terms. Prior to the filing or initiation of any action or proceeding relating to these Terms, the parties must participate in good faith mediation in San Mateo County, California. If a party initiates any proceeding regarding these Terms, the prevailing party to such proceeding is entitled to reasonable attorneys’ fees and costs for claims arising out of these Terms. +
    +
    Publicity
    +
    + You consent to CoreOS’ use of your name and/or logo on the CoreOS website, identifying you as a customer of CoreOS and describing your use of the Services notwithstanding any terms to the contrary in these Terms. You agree that CoreOS may issue a press release identifying you as customer of CoreOS. +
    +
    Assignment
    +
    + Neither these Terms nor any right or duty under these Terms may be transferred, assigned or delegated by you, by operation of law or otherwise, without the prior written consent of CoreOS, and any attempted transfer, assignment or delegation without such consent will be void and without effect. CoreOS may freely transfer, assign or delegate these Terms or its rights and duties under these Terms. Subject to the foregoing, these Terms will be binding upon and will inure to the benefit of the parties and their respective representatives, heirs, administrators, successors and permitted assigns. +
    +
    Amendments and Waivers
    +
    + Unless expressly stated otherwise stated in your standard service terms, no modification, addition or deletion, or waiver of any rights under these Terms will be binding on a party unless clearly understood by the parties to be a modification or waiver and signed by a duly authorized representative of each party. No failure or delay (in whole or in part) on the part of a party to exercise any right or remedy hereunder will operate as a waiver thereof or effect any other right or remedy. All rights and remedies hereunder are cumulative and are not exclusive of any other rights or remedies provided hereunder or by law. The waiver of one breach or default or any delay in exercising any rights will not constitute a waiver of any subsequent breach or default. +
    +
    Electronic Communications
    +
    + CoreOS may choose to electronically deliver all communications with you, which may include email to the email address you provide to CoreOS. CoreOS’ electronic communications to you may transmit or convey information about action taken on your request, portions of your request that may be incomplete or require additional explanation, any notices required under applicable law and any other notices. You agree to do business electronically with CoreOS and to receive electronically all current and future notices, disclosures, communications and information and that the aforementioned electronic communications satisfy any legal requirement that such communications be in writing. An electronic notice will be deemed to have been received on the day of receipt as evidenced by such email. +
    +
    Severability
    +
    + If any provision of these Terms is invalid, illegal, or incapable of being enforced by any rule of law or public policy, all other provisions of these Terms will nonetheless remain in full force and effect so long as the economic and legal substance of the transactions contemplated by these Terms is not affected in any manner adverse to any party. Upon such determination that any provision is invalid, illegal, or incapable of being enforced, the parties will negotiate in good faith to modify these Terms so as to effect the original intent of the parties as closely as possible in an acceptable manner to the end that the transactions contemplated hereby are fulfilled. +
    +
    Force Majeure
    +
    + Except for payments due under these Terms, neither party will be responsible for any failure to perform or delay attributable in whole or in part to any cause beyond its reasonable control, including, but not limited to, acts of God (fire, storm, floods, earthquakes, etc.), civil disturbances, disruption of telecommunications, disruption of power or other essential services, interruption or termination of service by any service providers being used by CoreOS to host the Services or to link its servers to the Internet, labor disturbances, vandalism, cable cut, computer viruses or other similar occurrences, or any malicious or unlawful acts of any third party. +
    +
    Notice for California Users
    +
    + If you are a California resident, you may have these Terms mailed to you electronically by sending a letter to the foregoing address with your electronic mail address and a request for these Terms. Under California Civil Code Section 1789.3, California Website users are entitled to the following specific consumer rights notice: The Complaint Assistance Unit of the Division of Consumer Services of the California Department of Consumer Affairs may be contacted in writing at 1625 N. Market Blvd., Suite S-202, Sacramento, California 95834, or by telephone at (800) 952-5210. +
    +
diff --git a/test/data/test.db b/test/data/test.db index 94762ac06..4da80c978 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/test/fulldbtest.sh b/test/fulldbtest.sh index 49ad1f999..d3fa0caa7 100755 --- a/test/fulldbtest.sh +++ b/test/fulldbtest.sh @@ -2,14 +2,14 @@ set -e up_mysql() { # Run a SQL database on port 3306 inside of Docker. - docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mysql + docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mysql:5.7 # Sleep for 5s to get MySQL get started. echo 'Sleeping for 10...' sleep 10 # Add the database to mysql. - docker run --rm --link mysql:mysql mysql sh -c 'echo "create database genschema" | mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -ppassword' + docker run --rm --link mysql:mysql mysql:5.7 sh -c 'echo "create database genschema" | mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -ppassword' } down_mysql() { diff --git a/test/test_api_security.py b/test/test_api_security.py index 97ec3950d..6cc790fbc 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -15,7 +15,7 @@ from endpoints.api.tag import RepositoryTagImages, RepositoryTag from endpoints.api.search import FindRepositories, EntitySearch from endpoints.api.image import RepositoryImageChanges, RepositoryImage, RepositoryImageList from endpoints.api.build import (FileDropResource, RepositoryBuildStatus, RepositoryBuildLogs, - RepositoryBuildList) + RepositoryBuildList, RepositoryBuildResource) from endpoints.api.robot import (UserRobotList, OrgRobot, OrgRobotList, UserRobot, RegenerateOrgRobot, RegenerateUserRobot) @@ -1571,6 +1571,60 @@ class TestRepositoryBuildStatusFg86BuynlargeOrgrepo(ApiTestCase): self._run_test('GET', 400, 'devtable', None) +class TestRepositoryBuildResourceFg86PublicPublicrepo(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(RepositoryBuildResource, build_uuid="FG86", repository="public/publicrepo") + + def test_delete_anonymous(self): + self._run_test('DELETE', 401, None, None) + + def test_delete_freshuser(self): + self._run_test('DELETE', 403, 'freshuser', None) + + def test_delete_reader(self): + self._run_test('DELETE', 403, 'reader', None) + + def test_delete_devtable(self): + self._run_test('DELETE', 403, 'devtable', None) + + +class TestRepositoryBuildResourceFg86DevtableShared(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(RepositoryBuildResource, build_uuid="FG86", repository="devtable/shared") + + def test_delete_anonymous(self): + self._run_test('DELETE', 401, None, None) + + def test_delete_freshuser(self): + self._run_test('DELETE', 403, 'freshuser', None) + + def test_delete_reader(self): + self._run_test('DELETE', 403, 'reader', None) + + def test_delete_devtable(self): + self._run_test('DELETE', 404, 'devtable', None) + + +class TestRepositoryBuildResourceFg86BuynlargeOrgrepo(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(RepositoryBuildResource, build_uuid="FG86", repository="buynlarge/orgrepo") + + def test_delete_anonymous(self): + self._run_test('DELETE', 401, None, None) + + def test_delete_freshuser(self): + self._run_test('DELETE', 403, 'freshuser', None) + + def test_delete_reader(self): + self._run_test('DELETE', 403, 'reader', None) + + def test_delete_devtable(self): + self._run_test('DELETE', 404, 'devtable', None) + + class TestRepositoryBuildLogsS5j8PublicPublicrepo(ApiTestCase): def setUp(self): ApiTestCase.setUp(self) @@ -3576,7 +3630,7 @@ class TestSuperUserLogs(ApiTestCase): self._set_url(SuperUserLogs) def test_get_anonymous(self): - self._run_test('GET', 403, None, None) + self._run_test('GET', 401, None, None) def test_get_freshuser(self): self._run_test('GET', 403, 'freshuser', None) diff --git a/test/test_api_usage.py b/test/test_api_usage.py index ff7ab5f10..c0cdf767f 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -17,7 +17,8 @@ from endpoints.api.team import TeamMember, TeamMemberList, TeamMemberInvite, Org from endpoints.api.tag import RepositoryTagImages, RepositoryTag from endpoints.api.search import FindRepositories, EntitySearch from endpoints.api.image import RepositoryImage, RepositoryImageList -from endpoints.api.build import RepositoryBuildStatus, RepositoryBuildLogs, RepositoryBuildList +from endpoints.api.build import (RepositoryBuildStatus, RepositoryBuildLogs, RepositoryBuildList, + RepositoryBuildResource) from endpoints.api.robot import (UserRobotList, OrgRobot, OrgRobotList, UserRobot, RegenerateUserRobot, RegenerateOrgRobot) from endpoints.api.trigger import (BuildTriggerActivate, BuildTriggerSources, BuildTriggerSubdirs, @@ -1303,6 +1304,103 @@ class TestGetRepository(ApiTestCase): self.assertEquals(True, json['is_organization']) + +class TestRepositoryBuildResource(ApiTestCase): + def test_cancel_invalidbuild(self): + self.login(ADMIN_ACCESS_USER) + + self.deleteResponse(RepositoryBuildResource, + params=dict(repository=ADMIN_ACCESS_USER + '/simple', build_uuid='invalid'), + expected_code=404) + + def test_cancel_waitingbuild(self): + self.login(ADMIN_ACCESS_USER) + + # Request a (fake) build. + json = self.postJsonResponse(RepositoryBuildList, + params=dict(repository=ADMIN_ACCESS_USER + '/simple'), + data=dict(file_id='foobarbaz'), + expected_code=201) + + uuid = json['id'] + + # Check for the build. + json = self.getJsonResponse(RepositoryBuildList, + params=dict(repository=ADMIN_ACCESS_USER + '/simple')) + + self.assertEquals(1, len(json['builds'])) + self.assertEquals(uuid, json['builds'][0]['id']) + + # Cancel the build. + self.deleteResponse(RepositoryBuildResource, + params=dict(repository=ADMIN_ACCESS_USER + '/simple', build_uuid=uuid), + expected_code=201) + + # Check for the build. + json = self.getJsonResponse(RepositoryBuildList, + params=dict(repository=ADMIN_ACCESS_USER + '/simple')) + + self.assertEquals(0, len(json['builds'])) + + + def test_attemptcancel_scheduledbuild(self): + self.login(ADMIN_ACCESS_USER) + + # Request a (fake) build. + json = self.postJsonResponse(RepositoryBuildList, + params=dict(repository=ADMIN_ACCESS_USER + '/simple'), + data=dict(file_id='foobarbaz'), + expected_code=201) + + uuid = json['id'] + + # Check for the build. + json = self.getJsonResponse(RepositoryBuildList, + params=dict(repository=ADMIN_ACCESS_USER + '/simple')) + + self.assertEquals(1, len(json['builds'])) + self.assertEquals(uuid, json['builds'][0]['id']) + + # Set queue item to be picked up. + qi = database.QueueItem.get(id=1) + qi.available = False + qi.save() + + # Try to cancel the build. + self.deleteResponse(RepositoryBuildResource, + params=dict(repository=ADMIN_ACCESS_USER + '/simple', build_uuid=uuid), + expected_code=400) + + + def test_attemptcancel_workingbuild(self): + self.login(ADMIN_ACCESS_USER) + + # Request a (fake) build. + json = self.postJsonResponse(RepositoryBuildList, + params=dict(repository=ADMIN_ACCESS_USER + '/simple'), + data=dict(file_id='foobarbaz'), + expected_code=201) + + uuid = json['id'] + + # Check for the build. + json = self.getJsonResponse(RepositoryBuildList, + params=dict(repository=ADMIN_ACCESS_USER + '/simple')) + + self.assertEquals(1, len(json['builds'])) + self.assertEquals(uuid, json['builds'][0]['id']) + + # Set the build to a different phase. + rb = database.RepositoryBuild.get(uuid=uuid) + rb.phase = database.BUILD_PHASE.BUILDING + rb.save() + + # Try to cancel the build. + self.deleteResponse(RepositoryBuildResource, + params=dict(repository=ADMIN_ACCESS_USER + '/simple', build_uuid=uuid), + expected_code=400) + + class TestRepoBuilds(ApiTestCase): def test_getrepo_nobuilds(self): self.login(ADMIN_ACCESS_USER) @@ -1965,6 +2063,9 @@ class TestOrgRobots(ApiTestCase): pull_robot = model.get_user(membername) model.create_build_trigger(repo, 'fakeservice', 'sometoken', user, pull_robot=pull_robot) + # Add some log entries for the robot. + model.log_action('pull_repo', ORGANIZATION, performer=pull_robot, repository=repo) + # Delete the robot and verify it works. self.deleteResponse(OrgRobot, params=dict(orgname=ORGANIZATION, robot_shortname='bender')) @@ -2175,7 +2276,7 @@ class FakeBuildTrigger(BuildTriggerBase): return [auth_token, 'foo', 'bar', config['somevalue']] def handle_trigger_request(self, request, auth_token, config): - return ('foo', ['bar'], 'build-name', 'subdir') + return ('foo', ['bar'], 'build-name', 'subdir', {'foo': 'bar'}) def is_active(self, config): return 'active' in config and config['active'] @@ -2189,7 +2290,7 @@ class FakeBuildTrigger(BuildTriggerBase): return config def manual_start(self, auth_token, config, run_parameters=None): - return ('foo', ['bar'], 'build-name', 'subdir') + return ('foo', ['bar'], 'build-name', 'subdir', {'foo': 'bar'}) def dockerfile_url(self, auth_token, config): return 'http://some/url' @@ -2400,6 +2501,9 @@ class TestBuildTriggers(ApiTestCase): self.assertEquals("build-name", start_json['display_name']) self.assertEquals(['bar'], start_json['job_config']['docker_tags']) + # Verify the metadata was added. + build_obj = database.RepositoryBuild.get(database.RepositoryBuild.uuid == start_json['id']) + self.assertEquals('bar', py_json.loads(build_obj.job_config)['trigger_metadata']['foo']) def test_invalid_robot_account(self): self.login(ADMIN_ACCESS_USER) diff --git a/test/test_buildman.py b/test/test_buildman.py new file mode 100644 index 000000000..543c5fc5d --- /dev/null +++ b/test/test_buildman.py @@ -0,0 +1,235 @@ +import unittest +import etcd +import os.path +import time +import json + +from trollius import coroutine, get_event_loop, From, Future, sleep, Return +from mock import Mock +from threading import Event +from urllib3.exceptions import ReadTimeoutError + +from buildman.manager.executor import BuilderExecutor +from buildman.manager.ephemeral import EphemeralBuilderManager, EtcdAction +from buildman.server import BuildJobResult +from buildman.component.buildcomponent import BuildComponent + + +BUILD_UUID = 'deadbeef-dead-beef-dead-deadbeefdead' +REALM_ID = '1234-realm' + + +def async_test(f): + def wrapper(*args, **kwargs): + coro = coroutine(f) + future = coro(*args, **kwargs) + loop = get_event_loop() + loop.run_until_complete(future) + return wrapper + +class TestEphemeral(unittest.TestCase): + def __init__(self, *args, **kwargs): + self.etcd_client_mock = None + self.etcd_wait_event = Event() + self.test_executor = None + super(TestEphemeral, self).__init__(*args, **kwargs) + + def _create_mock_etcd_client(self, *args, **kwargs): + def hang_until_event(*args, **kwargs): + time.sleep(.01) # 10ms to simulate network latency + self.etcd_wait_event.wait() + + self.etcd_client_mock = Mock(spec=etcd.Client, name='etcd.Client') + self.etcd_client_mock.watch = Mock(side_effect=hang_until_event) + return self.etcd_client_mock + + def _create_completed_future(self, result=None): + def inner(*args, **kwargs): + new_future = Future() + new_future.set_result(result) + return new_future + return inner + + def _create_mock_executor(self, *args, **kwargs): + self.test_executor = Mock(spec=BuilderExecutor) + self.test_executor.start_builder = Mock(side_effect=self._create_completed_future('123')) + self.test_executor.stop_builder = Mock(side_effect=self._create_completed_future()) + return self.test_executor + + def _create_build_job(self): + mock_job = Mock() + mock_job.job_details = { + 'build_uuid': BUILD_UUID, + } + mock_job.job_item = { + 'body': json.dumps(mock_job.job_details), + 'id': 1, + } + return mock_job + + def setUp(self): + EphemeralBuilderManager._executors['test'] = self._create_mock_executor + + self.old_etcd_client_klass = EphemeralBuilderManager._etcd_client_klass + EphemeralBuilderManager._etcd_client_klass = self._create_mock_etcd_client + self.etcd_wait_event.clear() + + self.register_component_callback = Mock() + self.unregister_component_callback = Mock() + self.job_heartbeat_callback = Mock() + self.job_complete_callback = Mock() + + self.manager = EphemeralBuilderManager( + self.register_component_callback, + self.unregister_component_callback, + self.job_heartbeat_callback, + self.job_complete_callback, + '127.0.0.1', + 30, + ) + + self.manager.initialize({'EXECUTOR': 'test'}) + + self.mock_job = self._create_build_job() + self.mock_job_key = os.path.join('building/', BUILD_UUID) + + def tearDown(self): + self.etcd_wait_event.set() + + self.manager.shutdown() + + del EphemeralBuilderManager._executors['test'] + EphemeralBuilderManager._etcd_client_klass = self.old_etcd_client_klass + + @coroutine + def _setup_job_for_managers(self): + # Test that we are watching the realm location before anything else happens + self.etcd_client_mock.watch.assert_any_call('realm/', recursive=True, timeout=0) + + self.etcd_client_mock.read = Mock(side_effect=KeyError) + test_component = Mock(spec=BuildComponent) + test_component.builder_realm = REALM_ID + test_component.start_build = Mock(side_effect=self._create_completed_future()) + self.register_component_callback.return_value = test_component + + # Ask for a builder to be scheduled + is_scheduled = yield From(self.manager.schedule(self.mock_job)) + + self.assertTrue(is_scheduled) + + self.etcd_client_mock.read.assert_called_once_with('building/', recursive=True) + self.assertEqual(self.test_executor.start_builder.call_count, 1) + self.assertEqual(self.etcd_client_mock.write.call_args_list[0][0][0], self.mock_job_key) + self.assertEqual(self.etcd_client_mock.write.call_args_list[1][0][0], self.mock_job_key) + + # Right now the job is not registered with any managers because etcd has not accepted the job + self.assertEqual(self.register_component_callback.call_count, 0) + + realm_created = Mock(spec=etcd.EtcdResult) + realm_created.action = EtcdAction.CREATE + realm_created.key = os.path.join('realm/', REALM_ID) + realm_created.value = json.dumps({ + 'realm': REALM_ID, + 'token': 'beef', + 'builder_id': '123', + 'job_queue_item': self.mock_job.job_item, + }) + + self.manager._handle_realm_change(realm_created) + + self.assertEqual(self.register_component_callback.call_count, 1) + + raise Return(test_component) + + @async_test + def test_schedule_and_complete(self): + # Test that a job is properly registered with all of the managers + test_component = yield From(self._setup_job_for_managers()) + + # Take the job ourselves + yield From(self.manager.build_component_ready(test_component)) + + self.etcd_client_mock.delete.assert_called_once_with(os.path.join('realm/', REALM_ID)) + self.etcd_client_mock.delete.reset_mock() + + # Finish the job + yield From(self.manager.job_completed(self.mock_job, BuildJobResult.COMPLETE, test_component)) + + self.assertEqual(self.test_executor.stop_builder.call_count, 1) + self.etcd_client_mock.delete.assert_called_once_with(self.mock_job_key) + + @async_test + def test_another_manager_takes_job(self): + # Prepare a job to be taken by another manager + test_component = yield From(self._setup_job_for_managers()) + + realm_deleted = Mock(spec=etcd.EtcdResult) + realm_deleted.action = EtcdAction.DELETE + realm_deleted.key = os.path.join('realm/', REALM_ID) + + realm_deleted._prev_node = Mock(spec=etcd.EtcdResult) + realm_deleted._prev_node.value = json.dumps({ + 'realm': REALM_ID, + 'token': 'beef', + 'builder_id': '123', + 'job_queue_item': self.mock_job.job_item, + }) + + self.manager._handle_realm_change(realm_deleted) + + self.unregister_component_callback.assert_called_once_with(test_component) + + @async_test + def test_expiring_worker(self): + # Test that we are watching before anything else happens + self.etcd_client_mock.watch.assert_any_call('building/', recursive=True, timeout=0) + + # Send a signal to the callback that a worker has expired + expired_result = Mock(spec=etcd.EtcdResult) + expired_result.action = EtcdAction.EXPIRE + expired_result.key = self.mock_job_key + expired_result._prev_node = Mock(spec=etcd.EtcdResult) + expired_result._prev_node.value = json.dumps({'builder_id': '1234'}) + + self.manager._handle_builder_expiration(expired_result) + + yield From(sleep(.01)) + + self.test_executor.stop_builder.assert_called_once_with('1234') + self.assertEqual(self.test_executor.stop_builder.call_count, 1) + + @async_test + def test_change_worker(self): + # Send a signal to the callback that a worker key has been changed + set_result = Mock(sepc=etcd.EtcdResult) + set_result.action = 'set' + set_result.key = self.mock_job_key + + self.manager._handle_builder_expiration(set_result) + + yield From(sleep(.01)) + + self.assertEquals(self.test_executor.stop_builder.call_count, 0) + + @async_test + def test_heartbeat_response(self): + expiration_timestamp = time.time() + 60 + builder_result = Mock(spec=etcd.EtcdResult) + builder_result.value = json.dumps({ + 'builder_id': '123', + 'expiration': expiration_timestamp, + 'max_expiration': expiration_timestamp, + }) + self.etcd_client_mock.read = Mock(return_value=builder_result) + + yield From(self.manager.job_heartbeat(self.mock_job)) + + # Wait for threads to complete + yield From(sleep(.01)) + + self.job_heartbeat_callback.assert_called_once_with(self.mock_job) + self.assertEqual(self.etcd_client_mock.write.call_count, 1) + self.assertEqual(self.etcd_client_mock.write.call_args_list[0][0][0], self.mock_job_key) + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_gc.py b/test/test_gc.py index 0ad8ce7bb..e538ff5bf 100644 --- a/test/test_gc.py +++ b/test/test_gc.py @@ -1,8 +1,6 @@ import unittest -import json as py_json +import time -from flask import url_for -from endpoints.api import api from app import app, storage from initdb import setup_database_for_testing, finished_database_for_testing from data import model, database @@ -13,8 +11,17 @@ PUBLIC_USER = 'public' REPO = 'somerepo' class TestGarbageColection(unittest.TestCase): + @staticmethod + def _set_tag_expiration_policy(namespace, expiration_s): + namespace_user = model.get_user(namespace) + model.change_user_tag_expiration(namespace_user, expiration_s) + def setUp(self): setup_database_for_testing(self) + + self._set_tag_expiration_policy(ADMIN_ACCESS_USER, 0) + self._set_tag_expiration_policy(PUBLIC_USER, 0) + self.app = app.test_client() self.ctx = app.test_request_context() self.ctx.__enter__() @@ -77,19 +84,21 @@ class TestGarbageColection(unittest.TestCase): model.garbage_collect_repository(repository.namespace_user.username, repository.name) def moveTag(self, repository, tag, docker_image_id): - model.create_or_update_tag(repository.namespace_user.username, repository.name, tag, docker_image_id) + model.create_or_update_tag(repository.namespace_user.username, repository.name, tag, + docker_image_id) model.garbage_collect_repository(repository.namespace_user.username, repository.name) def assertNotDeleted(self, repository, *args): for docker_image_id in args: - self.assertTrue(bool(model.get_image_by_id(repository.namespace_user.username, repository.name, docker_image_id))) + self.assertTrue(bool(model.get_image_by_id(repository.namespace_user.username, + repository.name, docker_image_id))) def assertDeleted(self, repository, *args): for docker_image_id in args: try: # Verify the image is missing when accessed by the repository. model.get_image_by_id(repository.namespace_user.username, repository.name, docker_image_id) - except model.DataModelException as ex: + except model.DataModelException: return self.fail('Expected image %s to be deleted' % docker_image_id) @@ -98,13 +107,13 @@ class TestGarbageColection(unittest.TestCase): def test_one_tag(self): """ Create a repository with a single tag, then remove that tag and verify that the repository is now empty. """ - repository = self.createRepository(latest = ['i1', 'i2', 'i3']) + repository = self.createRepository(latest=['i1', 'i2', 'i3']) self.deleteTag(repository, 'latest') self.assertDeleted(repository, 'i1', 'i2', 'i3') def test_two_tags_unshared_images(self): """ Repository has two tags with no shared images between them. """ - repository = self.createRepository(latest = ['i1', 'i2', 'i3'], other = ['f1', 'f2']) + repository = self.createRepository(latest=['i1', 'i2', 'i3'], other=['f1', 'f2']) self.deleteTag(repository, 'latest') self.assertDeleted(repository, 'i1', 'i2', 'i3') self.assertNotDeleted(repository, 'f1', 'f2') @@ -113,7 +122,7 @@ class TestGarbageColection(unittest.TestCase): """ Repository has two tags with shared images. Deleting the tag should only remove the unshared images. """ - repository = self.createRepository(latest = ['i1', 'i2', 'i3'], other = ['i1', 'f1']) + repository = self.createRepository(latest=['i1', 'i2', 'i3'], other=['i1', 'f1']) self.deleteTag(repository, 'latest') self.assertDeleted(repository, 'i2', 'i3') self.assertNotDeleted(repository, 'i1', 'f1') @@ -122,8 +131,8 @@ class TestGarbageColection(unittest.TestCase): """ Two repositories with different images. Removing the tag from one leaves the other's images intact. """ - repository1 = self.createRepository(latest = ['i1', 'i2', 'i3'], name = 'repo1') - repository2 = self.createRepository(latest = ['j1', 'j2', 'j3'], name = 'repo2') + repository1 = self.createRepository(latest=['i1', 'i2', 'i3'], name='repo1') + repository2 = self.createRepository(latest=['j1', 'j2', 'j3'], name='repo2') self.deleteTag(repository1, 'latest') @@ -134,8 +143,8 @@ class TestGarbageColection(unittest.TestCase): """ Two repositories with shared images. Removing the tag from one leaves the other's images intact. """ - repository1 = self.createRepository(latest = ['i1', 'i2', 'i3'], name = 'repo1') - repository2 = self.createRepository(latest = ['i1', 'i2', 'j1'], name = 'repo2') + repository1 = self.createRepository(latest=['i1', 'i2', 'i3'], name='repo1') + repository2 = self.createRepository(latest=['i1', 'i2', 'j1'], name='repo2') self.deleteTag(repository1, 'latest') @@ -146,8 +155,8 @@ class TestGarbageColection(unittest.TestCase): """ Two repositories under different namespaces should result in the images being deleted but not completely removed from the database. """ - repository1 = self.createRepository(namespace = ADMIN_ACCESS_USER, latest = ['i1', 'i2', 'i3']) - repository2 = self.createRepository(namespace = PUBLIC_USER, latest = ['i1', 'i2', 'i3']) + repository1 = self.createRepository(namespace=ADMIN_ACCESS_USER, latest=['i1', 'i2', 'i3']) + repository2 = self.createRepository(namespace=PUBLIC_USER, latest=['i1', 'i2', 'i3']) self.deleteTag(repository1, 'latest') self.assertDeleted(repository1, 'i1', 'i2', 'i3') @@ -158,34 +167,31 @@ class TestGarbageColection(unittest.TestCase): """ Repository has multiple tags with shared images. Selectively deleting the tags, and verifying at each step. """ - repository = self.createRepository( - latest = ['i1', 'i2', 'i3'], - other = ['i1', 'f1', 'f2'], - third = ['t1', 't2', 't3'], - fourth = ['i1', 'f1']) + repository = self.createRepository(latest=['i1', 'i2', 'i3'], other=['i1', 'f1', 'f2'], + third=['t1', 't2', 't3'], fourth=['i1', 'f1']) # Delete tag other. Should delete f2, since it is not shared. self.deleteTag(repository, 'other') self.assertDeleted(repository, 'f2') - self.assertNotDeleted(repository,'i1', 'i2', 'i3', 't1', 't2', 't3', 'f1') + self.assertNotDeleted(repository, 'i1', 'i2', 'i3', 't1', 't2', 't3', 'f1') # Move tag fourth to i3. This should remove f1 since it is no longer referenced. self.moveTag(repository, 'fourth', 'i3') self.assertDeleted(repository, 'f1') - self.assertNotDeleted(repository,'i1', 'i2', 'i3', 't1', 't2', 't3') + self.assertNotDeleted(repository, 'i1', 'i2', 'i3', 't1', 't2', 't3') # Delete tag 'latest'. This should do nothing since fourth is on the same branch. self.deleteTag(repository, 'latest') - self.assertNotDeleted(repository,'i1', 'i2', 'i3', 't1', 't2', 't3') + self.assertNotDeleted(repository, 'i1', 'i2', 'i3', 't1', 't2', 't3') # Delete tag 'third'. This should remove t1->t3. self.deleteTag(repository, 'third') self.assertDeleted(repository, 't1', 't2', 't3') - self.assertNotDeleted(repository,'i1', 'i2', 'i3') + self.assertNotDeleted(repository, 'i1', 'i2', 'i3') # Add tag to i1. self.moveTag(repository, 'newtag', 'i1') - self.assertNotDeleted(repository,'i1', 'i2', 'i3') + self.assertNotDeleted(repository, 'i1', 'i2', 'i3') # Delete tag 'fourth'. This should remove i2 and i3. self.deleteTag(repository, 'fourth') @@ -197,17 +203,40 @@ class TestGarbageColection(unittest.TestCase): self.assertDeleted(repository, 'i1') def test_empty_gc(self): - repository = self.createRepository( - latest = ['i1', 'i2', 'i3'], - other = ['i1', 'f1', 'f2'], - third = ['t1', 't2', 't3'], - fourth = ['i1', 'f1']) + repository = self.createRepository(latest=['i1', 'i2', 'i3'], other=['i1', 'f1', 'f2'], + third=['t1', 't2', 't3'], fourth=['i1', 'f1']) self.gcNow(repository) - self.assertNotDeleted(repository,'i1', 'i2', 'i3', 't1', 't2', 't3', 'f1', 'f2') + self.assertNotDeleted(repository, 'i1', 'i2', 'i3', 't1', 't2', 't3', 'f1', 'f2') + + def test_time_machine_no_gc(self): + """ Repository has two tags with shared images. Deleting the tag should not remove any images + """ + repository = self.createRepository(latest=['i1', 'i2', 'i3'], other=['i1', 'f1']) + self._set_tag_expiration_policy(repository.namespace_user.username, 60*60*24) + + self.deleteTag(repository, 'latest') + self.assertNotDeleted(repository, 'i2', 'i3') + self.assertNotDeleted(repository, 'i1', 'f1') + + def test_time_machine_gc(self): + """ Repository has two tags with shared images. Deleting the second tag should cause the images + for the first deleted tag to gc. + """ + repository = self.createRepository(latest=['i1', 'i2', 'i3'], other=['i1', 'f1']) + + self._set_tag_expiration_policy(repository.namespace_user.username, 1) + + self.deleteTag(repository, 'latest') + self.assertNotDeleted(repository, 'i2', 'i3') + self.assertNotDeleted(repository, 'i1', 'f1') + + time.sleep(2) + + self.deleteTag(repository, 'other') # This will cause the images associated with latest to gc + self.assertDeleted(repository, 'i2', 'i3') + self.assertNotDeleted(repository, 'i1', 'f1') - def test_gc_storage_empty(self): - model.garbage_collect_storage(set()) if __name__ == '__main__': unittest.main() diff --git a/test/test_imagetree.py b/test/test_imagetree.py new file mode 100644 index 000000000..d72eb6505 --- /dev/null +++ b/test/test_imagetree.py @@ -0,0 +1,96 @@ +import unittest + +from app import app +from util.imagetree import ImageTree +from initdb import setup_database_for_testing, finished_database_for_testing +from data import model + +NAMESPACE = 'devtable' +SIMPLE_REPO = 'simple' +COMPLEX_REPO = 'complex' + +class TestImageTree(unittest.TestCase): + def setUp(self): + setup_database_for_testing(self) + self.app = app.test_client() + self.ctx = app.test_request_context() + self.ctx.__enter__() + + def tearDown(self): + finished_database_for_testing(self) + self.ctx.__exit__(True, None, None) + + def _get_base_image(self, all_images): + for image in all_images: + if image.ancestors == '/': + return image + + return None + + def test_longest_path_simple_repo(self): + all_images = list(model.get_repository_images(NAMESPACE, SIMPLE_REPO)) + all_tags = list(model.list_repository_tags(NAMESPACE, SIMPLE_REPO)) + tree = ImageTree(all_images, all_tags) + + base_image = self._get_base_image(all_images) + tag_image = all_tags[0].image + + def checker(index, image): + return True + + ancestors = tag_image.ancestors.split('/')[2:-1] # Skip the first image. + result = tree.find_longest_path(base_image.id, checker) + self.assertEquals(3, len(result)) + for index in range(0, 2): + self.assertEquals(int(ancestors[index]), result[index].id) + + self.assertEquals('latest', tree.tag_containing_image(result[-1])) + + def test_longest_path_complex_repo(self): + all_images = list(model.get_repository_images(NAMESPACE, COMPLEX_REPO)) + all_tags = list(model.list_repository_tags(NAMESPACE, COMPLEX_REPO)) + tree = ImageTree(all_images, all_tags) + + base_image = self._get_base_image(all_images) + + def checker(index, image): + return True + + result = tree.find_longest_path(base_image.id, checker) + self.assertEquals(4, len(result)) + self.assertEquals('v2.0', tree.tag_containing_image(result[-1])) + + def test_filtering(self): + all_images = list(model.get_repository_images(NAMESPACE, COMPLEX_REPO)) + all_tags = list(model.list_repository_tags(NAMESPACE, COMPLEX_REPO)) + tree = ImageTree(all_images, all_tags, base_filter=1245) + + base_image = self._get_base_image(all_images) + + def checker(index, image): + return True + + result = tree.find_longest_path(base_image.id, checker) + self.assertEquals(0, len(result)) + + def test_find_tag_parent_image(self): + all_images = list(model.get_repository_images(NAMESPACE, COMPLEX_REPO)) + all_tags = list(model.list_repository_tags(NAMESPACE, COMPLEX_REPO)) + tree = ImageTree(all_images, all_tags) + + base_image = self._get_base_image(all_images) + + def checker(index, image): + return True + + result = tree.find_longest_path(base_image.id, checker) + self.assertEquals(4, len(result)) + + # Only use the first two images. They don't have tags, but the method should + # still return the tag that contains them. + self.assertEquals('v2.0', tree.tag_containing_image(result[0])) + + +if __name__ == '__main__': + unittest.main() + diff --git a/test/test_queue.py b/test/test_queue.py index 6c1660eb7..3d31978c8 100644 --- a/test/test_queue.py +++ b/test/test_queue.py @@ -162,3 +162,8 @@ class TestQueue(QueueTestCase): one = self.queue.get() self.assertNotEqual(None, one) self.assertEqual(self.TEST_MESSAGE_1, one.body) + + +if __name__ == '__main__': + unittest.main() + diff --git a/test/test_streamlayerformat.py b/test/test_streamlayerformat.py index 093bbaee4..6e65e2b34 100644 --- a/test/test_streamlayerformat.py +++ b/test/test_streamlayerformat.py @@ -34,11 +34,11 @@ class TestStreamLayerMerger(unittest.TestCase): def create_empty_layer(self): return '' - def squash_layers(self, layers): + def squash_layers(self, layers, path_prefix=None): def get_layers(): return [StringIO(layer) for layer in layers] - merger = StreamLayerMerger(get_layers) + merger = StreamLayerMerger(get_layers, path_prefix=path_prefix) merged_data = ''.join(merger.get_generator()) return merged_data @@ -395,5 +395,57 @@ class TestStreamLayerMerger(unittest.TestCase): except TarLayerReadException as ex: self.assertEquals('Could not read layer', ex.message) + def test_single_layer_with_prefix(self): + tar_layer = self.create_layer( + foo = 'some_file', + bar = 'another_file', + meh = 'third_file') + + squashed = self.squash_layers([tar_layer], path_prefix='foo/') + + self.assertHasFile(squashed, 'foo/some_file', 'foo') + self.assertHasFile(squashed, 'foo/another_file', 'bar') + self.assertHasFile(squashed, 'foo/third_file', 'meh') + + def test_multiple_layers_overwrite_with_prefix(self): + second_layer = self.create_layer( + foo = 'some_file', + bar = 'another_file', + meh = 'third_file') + + first_layer = self.create_layer( + top = 'another_file') + + squashed = self.squash_layers([first_layer, second_layer], path_prefix='foo/') + + self.assertHasFile(squashed, 'foo/some_file', 'foo') + self.assertHasFile(squashed, 'foo/third_file', 'meh') + self.assertHasFile(squashed, 'foo/another_file', 'top') + + + def test_superlong_filename(self): + tar_layer = self.create_layer( + meh = 'this_is_the_filename_that_never_ends_it_goes_on_and_on_my_friend_some_people_started') + + squashed = self.squash_layers([tar_layer], + path_prefix='foo/') + + self.assertHasFile(squashed, 'foo/this_is_the_filename_that_never_ends_it_goes_on_and_on_my_friend_some_people_started', 'meh') + + + def test_superlong_prefix(self): + tar_layer = self.create_layer( + foo = 'some_file', + bar = 'another_file', + meh = 'third_file') + + squashed = self.squash_layers([tar_layer], + path_prefix='foo/bar/baz/something/foo/bar/baz/anotherthing/whatever/this/is/a/really/long/filename/that/goes/here/') + + self.assertHasFile(squashed, 'foo/bar/baz/something/foo/bar/baz/anotherthing/whatever/this/is/a/really/long/filename/that/goes/here/some_file', 'foo') + self.assertHasFile(squashed, 'foo/bar/baz/something/foo/bar/baz/anotherthing/whatever/this/is/a/really/long/filename/that/goes/here/another_file', 'bar') + self.assertHasFile(squashed, 'foo/bar/baz/something/foo/bar/baz/anotherthing/whatever/this/is/a/really/long/filename/that/goes/here/third_file', 'meh') + + if __name__ == '__main__': unittest.main() diff --git a/test/test_suconfig_api.py b/test/test_suconfig_api.py new file mode 100644 index 000000000..ca05d8705 --- /dev/null +++ b/test/test_suconfig_api.py @@ -0,0 +1,186 @@ +from test.test_api_usage import ApiTestCase, READ_ACCESS_USER, ADMIN_ACCESS_USER +from endpoints.api.suconfig import (SuperUserRegistryStatus, SuperUserConfig, SuperUserConfigFile, + SuperUserCreateInitialSuperUser, SuperUserConfigValidate) +from app import CONFIG_PROVIDER +from data.database import User + +import unittest + + +class ConfigForTesting(object): + + def __enter__(self): + CONFIG_PROVIDER.reset_for_test() + return CONFIG_PROVIDER + + def __exit__(self, type, value, traceback): + CONFIG_PROVIDER.reset_for_test() + + +class TestSuperUserRegistryStatus(ApiTestCase): + def test_registry_status(self): + with ConfigForTesting(): + json = self.getJsonResponse(SuperUserRegistryStatus) + self.assertEquals('config-db', json['status']) + + +class TestSuperUserConfigFile(ApiTestCase): + def test_get_non_superuser(self): + with ConfigForTesting(): + # No user. + self.getResponse(SuperUserConfigFile, params=dict(filename='ssl.cert'), expected_code=403) + + # Non-superuser. + self.login(READ_ACCESS_USER) + self.getResponse(SuperUserConfigFile, params=dict(filename='ssl.cert'), expected_code=403) + + def test_get_superuser_invalid_filename(self): + with ConfigForTesting(): + self.login(ADMIN_ACCESS_USER) + self.getResponse(SuperUserConfigFile, params=dict(filename='somefile'), expected_code=404) + + def test_get_superuser(self): + with ConfigForTesting(): + self.login(ADMIN_ACCESS_USER) + result = self.getJsonResponse(SuperUserConfigFile, params=dict(filename='ssl.cert')) + self.assertFalse(result['exists']) + + def test_post_non_superuser(self): + with ConfigForTesting(): + # No user. + self.postResponse(SuperUserConfigFile, params=dict(filename='ssl.cert'), expected_code=403) + + # Non-superuser. + self.login(READ_ACCESS_USER) + self.postResponse(SuperUserConfigFile, params=dict(filename='ssl.cert'), expected_code=403) + + def test_post_superuser_invalid_filename(self): + with ConfigForTesting(): + self.login(ADMIN_ACCESS_USER) + self.postResponse(SuperUserConfigFile, params=dict(filename='somefile'), expected_code=404) + + def test_post_superuser(self): + with ConfigForTesting(): + self.login(ADMIN_ACCESS_USER) + self.postResponse(SuperUserConfigFile, params=dict(filename='ssl.cert'), expected_code=400) + + +class TestSuperUserCreateInitialSuperUser(ApiTestCase): + def test_no_config_file(self): + with ConfigForTesting(): + # If there is no config.yaml, then this method should security fail. + data = dict(username='cooluser', password='password', email='fake@example.com') + self.postResponse(SuperUserCreateInitialSuperUser, data=data, expected_code=403) + + def test_config_file_with_db_users(self): + with ConfigForTesting(): + # Write some config. + self.putJsonResponse(SuperUserConfig, data=dict(config={}, hostname='foobar')) + + # If there is a config.yaml, but existing DB users exist, then this method should security + # fail. + data = dict(username='cooluser', password='password', email='fake@example.com') + self.postResponse(SuperUserCreateInitialSuperUser, data=data, expected_code=403) + + def test_config_file_with_no_db_users(self): + with ConfigForTesting(): + # Write some config. + self.putJsonResponse(SuperUserConfig, data=dict(config={}, hostname='foobar')) + + # Delete all the users in the DB. + for user in list(User.select()): + user.delete_instance(recursive=True) + + # This method should now succeed. + data = dict(username='cooluser', password='password', email='fake@example.com') + result = self.postJsonResponse(SuperUserCreateInitialSuperUser, data=data) + self.assertTrue(result['status']) + + # Verify the superuser was created. + User.get(User.username == 'cooluser') + + # Verify the superuser was placed into the config. + result = self.getJsonResponse(SuperUserConfig) + self.assertEquals(['cooluser'], result['config']['SUPER_USERS']) + + +class TestSuperUserConfigValidate(ApiTestCase): + def test_nonsuperuser_noconfig(self): + with ConfigForTesting(): + self.login(ADMIN_ACCESS_USER) + result = self.postJsonResponse(SuperUserConfigValidate, params=dict(service='someservice'), + data=dict(config={})) + + self.assertFalse(result['status']) + + + def test_nonsuperuser_config(self): + with ConfigForTesting(): + # The validate config call works if there is no config.yaml OR the user is a superuser. + # Add a config, and verify it breaks when unauthenticated. + json = self.putJsonResponse(SuperUserConfig, data=dict(config={}, hostname='foobar')) + self.assertTrue(json['exists']) + + self.postResponse(SuperUserConfigValidate, params=dict(service='someservice'), + data=dict(config={}), + expected_code=403) + + # Now login as a superuser. + self.login(ADMIN_ACCESS_USER) + result = self.postJsonResponse(SuperUserConfigValidate, params=dict(service='someservice'), + data=dict(config={})) + + self.assertFalse(result['status']) + + +class TestSuperUserConfig(ApiTestCase): + def test_get_non_superuser(self): + with ConfigForTesting(): + # No user. + self.getResponse(SuperUserConfig, expected_code=401) + + # Non-superuser. + self.login(READ_ACCESS_USER) + self.getResponse(SuperUserConfig, expected_code=403) + + def test_get_superuser(self): + with ConfigForTesting(): + self.login(ADMIN_ACCESS_USER) + json = self.getJsonResponse(SuperUserConfig) + + # Note: We expect the config to be none because a config.yaml should never be checked into + # the directory. + self.assertIsNone(json['config']) + + def test_put(self): + with ConfigForTesting() as config: + # The update config call works if there is no config.yaml OR the user is a superuser. First + # try writing it without a superuser present. + json = self.putJsonResponse(SuperUserConfig, data=dict(config={}, hostname='foobar')) + self.assertTrue(json['exists']) + + # Verify the config file exists. + self.assertTrue(config.yaml_exists()) + + # Try writing it again. This should now fail, since the config.yaml exists. + self.putResponse(SuperUserConfig, data=dict(config={}, hostname='barbaz'), expected_code=403) + + # Login as a non-superuser. + self.login(READ_ACCESS_USER) + + # Try writing it again. This should fail. + self.putResponse(SuperUserConfig, data=dict(config={}, hostname='barbaz'), expected_code=403) + + # Login as a superuser. + self.login(ADMIN_ACCESS_USER) + + # This should succeed. + json = self.putJsonResponse(SuperUserConfig, data=dict(config={}, hostname='barbaz')) + self.assertTrue(json['exists']) + + json = self.getJsonResponse(SuperUserConfig) + self.assertIsNotNone(json['config']) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/test/testconfig.py b/test/testconfig.py index 9613f76d2..0b947fd64 100644 --- a/test/testconfig.py +++ b/test/testconfig.py @@ -1,6 +1,7 @@ import os from datetime import datetime, timedelta +from tempfile import NamedTemporaryFile from config import DefaultConfig @@ -13,10 +14,14 @@ class FakeTransaction(object): pass +TEST_DB_FILE = NamedTemporaryFile(delete=True) + + class TestConfig(DefaultConfig): TESTING = True + SECRET_KEY = 'a36c9d7d-25a9-4d3f-a586-3d2f8dc40a83' - DB_URI = os.environ.get('TEST_DATABASE_URI', 'sqlite:///:memory:') + DB_URI = os.environ.get('TEST_DATABASE_URI', 'sqlite:///{0}'.format(TEST_DB_FILE.name)) DB_CONNECTION_ARGS = { 'threadlocals': True, 'autorollback': True diff --git a/tools/sendresetemail.py b/tools/sendresetemail.py new file mode 100644 index 000000000..e977c654e --- /dev/null +++ b/tools/sendresetemail.py @@ -0,0 +1,27 @@ +from app import app + +from util.useremails import send_recovery_email + +from data import model + +import argparse + +from flask import Flask, current_app +from flask_mail import Mail + +def sendReset(username): + user = model.get_user(username) + if not user: + print 'No user found' + return + + + with app.app_context(): + code = model.create_reset_password_email_code(user.email) + send_recovery_email(user.email, code.code) + print 'Email sent to %s' % (user.email) + +parser = argparse.ArgumentParser(description='Sends a reset email') +parser.add_argument('username', help='The username') +args = parser.parse_args() +sendReset(args.username) diff --git a/util/analytics.py b/util/analytics.py index a7608aed8..7ecf8a9c4 100644 --- a/util/analytics.py +++ b/util/analytics.py @@ -1,8 +1,9 @@ import json import logging -from multiprocessing import Process, Queue -from mixpanel import Consumer, Mixpanel +from Queue import Queue +from threading import Thread +from mixpanel import BufferedConsumer, Mixpanel logger = logging.getLogger(__name__) @@ -17,24 +18,23 @@ class MixpanelQueingConsumer(object): self._mp_queue.put(json.dumps([endpoint, json_message])) -class SendToMixpanel(Process): +class SendToMixpanel(Thread): def __init__(self, request_queue): - Process.__init__(self) + Thread.__init__(self) + self.daemon = True self._mp_queue = request_queue - self._consumer = Consumer() - self.daemon = True + self._consumer = BufferedConsumer() def run(self): logger.debug('Starting mixpanel sender process.') while True: mp_request = self._mp_queue.get() - logger.debug('Got queued mixpanel reqeust.') + logger.debug('Got queued mixpanel request.') try: self._consumer.send(*json.loads(mp_request)) except: - # Make sure we don't crash if Mixpanel request fails. - pass + logger.exception('Failed to send Mixpanel request.') class FakeMixpanel(object): diff --git a/util/backfill_user_uuids.py b/util/backfill_user_uuids.py index ab9ca4567..e71ec82a2 100644 --- a/util/backfill_user_uuids.py +++ b/util/backfill_user_uuids.py @@ -15,7 +15,7 @@ def backfill_user_uuids(): # Check to see if any users are missing uuids. has_missing_uuids = True try: - User.select().where(User.uuid >> None).get() + User.select(User.id).where(User.uuid >> None).get() except User.DoesNotExist: has_missing_uuids = False @@ -39,9 +39,9 @@ def backfill_user_uuids(): for user_id in batch_user_ids: with app.config['DB_TRANSACTION_FACTORY'](db): try: - user = User.get(User.id == user_id) + user = User.select(User.id, User.uuid).where(User.id == user_id).get() user.uuid = str(uuid.uuid4()) - user.save() + user.save(only=[User.uuid]) except User.DoesNotExist: pass diff --git a/util/cloudwatch.py b/util/cloudwatch.py new file mode 100644 index 000000000..b75dadf31 --- /dev/null +++ b/util/cloudwatch.py @@ -0,0 +1,47 @@ +import logging +import boto + +from Queue import Queue +from threading import Thread + + +logger = logging.getLogger(__name__) + +def get_queue(app): + """ + Returns a queue to the CloudWatchSender. If a queue/sender do not exist, creates them. + """ + access_key = app.config['CLOUDWATCH_AWS_ACCESS_KEY'] + secret_key = app.config['CLOUDWATCH_AWS_SECRET_KEY'] + + queue = Queue() + sender = CloudWatchSender(queue, access_key, secret_key) + sender.start() + return queue + +class CloudWatchSender(Thread): + """ + CloudWatchSender loops indefinitely and pulls metrics off of a queue then sends it to CloudWatch. + """ + def __init__(self, request_queue, aws_access_key, aws_secret_key): + Thread.__init__(self) + self.daemon = True + + self._aws_access_key = aws_access_key + self._aws_secret_key = aws_secret_key + self._put_metrics_queue = request_queue + + def run(self): + try: + logger.debug('Starting CloudWatch sender process.') + connection = boto.connect_cloudwatch(self._aws_access_key, self._aws_secret_key) + except: + logger.exception('Failed to connect to CloudWatch.') + + while True: + put_metric_args, kwargs = self._put_metrics_queue.get() + logger.debug('Got queued put metrics request.') + try: + connection.put_metric_data(*put_metric_args, **kwargs) + except: + logger.exception('Failed to write to CloudWatch') diff --git a/util/config/__init__.py b/util/config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/util/config/configutil.py b/util/config/configutil.py new file mode 100644 index 000000000..7a31390c9 --- /dev/null +++ b/util/config/configutil.py @@ -0,0 +1,48 @@ +import yaml + +from random import SystemRandom + +def generate_secret_key(): + cryptogen = SystemRandom() + return str(cryptogen.getrandbits(256)) + + +def add_enterprise_config_defaults(config_obj, current_secret_key, hostname): + """ Adds/Sets the config defaults for enterprise registry config. """ + # These have to be false. + config_obj['TESTING'] = False + config_obj['USE_CDN'] = False + + # Default features that are on. + config_obj['FEATURE_USER_LOG_ACCESS'] = config_obj.get('FEATURE_USER_LOG_ACCESS', True) + config_obj['FEATURE_USER_CREATION'] = config_obj.get('FEATURE_USER_CREATION', True) + + # Default features that are off. + config_obj['FEATURE_MAILING'] = config_obj.get('FEATURE_MAILING', False) + config_obj['FEATURE_BUILD_SUPPORT'] = config_obj.get('FEATURE_BUILD_SUPPORT', False) + + # Default auth type. + if not 'AUTHENTICATION_TYPE' in config_obj: + config_obj['AUTHENTICATION_TYPE'] = 'Database' + + # Default secret key. + if not 'SECRET_KEY' in config_obj: + config_obj['SECRET_KEY'] = current_secret_key + + # Default storage configuration. + if not 'DISTRIBUTED_STORAGE_CONFIG' in config_obj: + config_obj['DISTRIBUTED_STORAGE_PREFERENCE'] = ['local'] + config_obj['DISTRIBUTED_STORAGE_CONFIG'] = { + 'local': ['LocalStorage', {'storage_path': '/datastorage/registry'}] + } + + config_obj['USERFILES_LOCATION'] = 'local' + config_obj['USERFILES_PATH'] = 'userfiles/' + + if not 'SERVER_HOSTNAME' in config_obj: + config_obj['SERVER_HOSTNAME'] = hostname + + # Misc configuration. + config_obj['PREFERRED_URL_SCHEME'] = config_obj.get('PREFERRED_URL_SCHEME', 'http') + config_obj['ENTERPRISE_LOGO_URL'] = config_obj.get('ENTERPRISE_LOGO_URL', + '/static/img/quay-logo.png') diff --git a/util/config/provider.py b/util/config/provider.py new file mode 100644 index 000000000..5a2d92757 --- /dev/null +++ b/util/config/provider.py @@ -0,0 +1,171 @@ +import os +import yaml +import logging +import json +from StringIO import StringIO + +logger = logging.getLogger(__name__) + +def _import_yaml(config_obj, config_file): + with open(config_file) as f: + c = yaml.safe_load(f) + if not c: + logger.debug('Empty YAML config file') + return + + if isinstance(c, str): + raise Exception('Invalid YAML config file: ' + str(c)) + + for key in c.iterkeys(): + if key.isupper(): + config_obj[key] = c[key] + + return config_obj + + +def _export_yaml(config_obj, config_file): + with open(config_file, 'w') as f: + f.write(yaml.safe_dump(config_obj, encoding='utf-8', allow_unicode=True)) + + +class BaseProvider(object): + """ A configuration provider helps to load, save, and handle config override in the application. + """ + + def update_app_config(self, app_config): + """ Updates the given application config object with the loaded override config. """ + raise NotImplementedError + + def get_yaml(self): + """ Returns the contents of the YAML config override file, or None if none. """ + raise NotImplementedError + + def save_yaml(self, config_object): + """ Updates the contents of the YAML config override file to those given. """ + raise NotImplementedError + + def yaml_exists(self): + """ Returns true if a YAML config override file exists in the config volume. """ + raise NotImplementedError + + def volume_exists(self): + """ Returns whether the config override volume exists. """ + raise NotImplementedError + + def volume_file_exists(self, filename): + """ Returns whether the file with the given name exists under the config override volume. """ + raise NotImplementedError + + def get_volume_file(self, filename, mode='r'): + """ Returns a Python file referring to the given name under the config override volumne. """ + raise NotImplementedError + + def save_volume_file(self, filename, flask_file): + """ Saves the given flask file to the config override volume, with the given + filename. + """ + raise NotImplementedError + + def requires_restart(self, app_config): + """ If true, the configuration loaded into memory for the app does not match that on disk, + indicating that this container requires a restart. + """ + raise NotImplementedError + + +class FileConfigProvider(BaseProvider): + """ Implementation of the config provider that reads the data from the file system. """ + def __init__(self, config_volume, yaml_filename, py_filename): + self.config_volume = config_volume + self.yaml_filename = yaml_filename + self.py_filename = py_filename + + self.yaml_path = os.path.join(config_volume, yaml_filename) + self.py_path = os.path.join(config_volume, py_filename) + + def update_app_config(self, app_config): + if os.path.exists(self.py_path): + logger.debug('Applying config file: %s', self.py_path) + app_config.from_pyfile(self.py_path) + + if os.path.exists(self.yaml_path): + logger.debug('Applying config file: %s', self.yaml_path) + _import_yaml(app_config, self.yaml_path) + + def get_yaml(self): + if not os.path.exists(self.yaml_path): + return None + + config_obj = {} + _import_yaml(config_obj, self.yaml_path) + return config_obj + + def save_yaml(self, config_obj): + _export_yaml(config_obj, self.yaml_path) + + def yaml_exists(self): + return self.volume_file_exists(self.yaml_filename) + + def volume_exists(self): + return os.path.exists(self.config_volume) + + def volume_file_exists(self, filename): + return os.path.exists(os.path.join(self.config_volume, filename)) + + def get_volume_file(self, filename, mode='r'): + return open(os.path.join(self.config_volume, filename), mode) + + def save_volume_file(self, filename, flask_file): + flask_file.save(os.path.join(self.config_volume, filename)) + + def requires_restart(self, app_config): + file_config = self.get_yaml() + if not file_config: + return False + + for key in file_config: + if app_config.get(key) != file_config[key]: + return True + + return False + +class TestConfigProvider(BaseProvider): + """ Implementation of the config provider for testing. Everything is kept in-memory instead on + the real file system. """ + def __init__(self): + self.files = {} + self._config = None + + def update_app_config(self, app_config): + self._config = app_config + + def get_yaml(self): + if not 'config.yaml' in self.files: + return None + + return json.loads(self.files.get('config.yaml', '{}')) + + def save_yaml(self, config_obj): + self.files['config.yaml'] = json.dumps(config_obj) + + def yaml_exists(self): + return 'config.yaml' in self.files + + def volume_exists(self): + return True + + def volume_file_exists(self, filename): + return filename in self.files + + def save_volume_file(self, filename, flask_file): + self.files[filename] = '' + + def get_volume_file(self, filename, mode='r'): + return StringIO(self.files[filename]) + + def requires_restart(self, app_config): + return False + + def reset_for_test(self): + self._config['SUPER_USERS'] = ['devtable'] + self.files = {} diff --git a/util/config/superusermanager.py b/util/config/superusermanager.py new file mode 100644 index 000000000..5930da9cf --- /dev/null +++ b/util/config/superusermanager.py @@ -0,0 +1,38 @@ +from multiprocessing.sharedctypes import Value, Array +from util.validation import MAX_LENGTH + +class SuperUserManager(object): + """ In-memory helper class for quickly accessing (and updating) the valid + set of super users. This class communicates across processes to ensure + that the shared set is always the same. + """ + + def __init__(self, app): + usernames = app.config.get('SUPER_USERS', []) + usernames_str = ','.join(usernames) + + self._max_length = len(usernames_str) + MAX_LENGTH + 1 + self._array = Array('c', self._max_length, lock=True) + self._array.value = usernames_str + + def is_superuser(self, username): + """ Returns if the given username represents a super user. """ + usernames = self._array.value.split(',') + return username in usernames + + def register_superuser(self, username): + """ Registers a new username as a super user for the duration of the container. + Note that this does *not* change any underlying config files. + """ + usernames = self._array.value.split(',') + usernames.append(username) + new_string = ','.join(usernames) + + if len(new_string) <= self._max_length: + self._array.value = new_string + else: + raise Exception('Maximum superuser count reached. Please report this to support.') + + def has_superusers(self): + """ Returns whether there are any superusers defined. """ + return bool(self._array.value) diff --git a/util/config/validator.py b/util/config/validator.py new file mode 100644 index 000000000..3c1671c2b --- /dev/null +++ b/util/config/validator.py @@ -0,0 +1,253 @@ +import redis +import os +import json +import ldap +import peewee +import OpenSSL +import logging + +from fnmatch import fnmatch +from data.users import LDAPConnection +from flask import Flask +from flask.ext.mail import Mail, Message +from data.database import validate_database_url, User +from storage import get_storage_driver +from app import app, CONFIG_PROVIDER +from auth.auth_context import get_authenticated_user +from util.oauth import GoogleOAuthConfig, GithubOAuthConfig + +logger = logging.getLogger(__name__) + +SSL_FILENAMES = ['ssl.cert', 'ssl.key'] + +def get_storage_provider(config): + parameters = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).get('local', ['LocalStorage', {}]) + try: + return get_storage_driver(parameters) + except TypeError: + raise Exception('Missing required storage configuration parameter(s)') + +def validate_service_for_config(service, config): + """ Attempts to validate the configuration for the given service. """ + if not service in _VALIDATORS: + return { + 'status': False + } + + try: + _VALIDATORS[service](config) + return { + 'status': True + } + except Exception as ex: + logger.exception('Validation exception') + return { + 'status': False, + 'reason': str(ex) + } + + +def _validate_database(config): + """ Validates connecting to the database. """ + try: + validate_database_url(config['DB_URI']) + except peewee.OperationalError as ex: + if ex.args and len(ex.args) > 1: + raise Exception(ex.args[1]) + else: + raise ex + + +def _validate_redis(config): + """ Validates connecting to redis. """ + redis_config = config.get('BUILDLOGS_REDIS', {}) + if not 'host' in redis_config: + raise Exception('Missing redis hostname') + + client = redis.StrictRedis(socket_connect_timeout=5, **redis_config) + client.ping() + + +def _validate_registry_storage(config): + """ Validates registry storage. """ + driver = get_storage_provider(config) + + # Put and remove a temporary file. + driver.put_content('_verify', 'testing 123') + driver.remove('_verify') + + # Run setup on the driver if the read/write succeeded. + try: + driver.setup() + except Exception as ex: + raise Exception('Could not prepare storage: %s' % str(ex)) + + +def _validate_mailing(config): + """ Validates sending email. """ + test_app = Flask("mail-test-app") + test_app.config.update(config) + test_app.config.update({ + 'MAIL_FAIL_SILENTLY': False, + 'TESTING': False + }) + + test_mail = Mail(test_app) + test_msg = Message("Test e-mail from %s" % app.config['REGISTRY_TITLE']) + test_msg.add_recipient(get_authenticated_user().email) + test_mail.send(test_msg) + + +def _validate_github(config_key): + return lambda config: _validate_github_with_key(config_key, config) + + +def _validate_github_with_key(config_key, config): + """ Validates the OAuth credentials and API endpoint for a Github service. """ + github_config = config.get(config_key) + if not github_config: + raise Exception('Missing Github client id and client secret') + + endpoint = github_config.get('GITHUB_ENDPOINT') + if not endpoint: + raise Exception('Missing Github Endpoint') + + if endpoint.find('http://') != 0 and endpoint.find('https://') != 0: + raise Exception('Github Endpoint must start with http:// or https://') + + if not github_config.get('CLIENT_ID'): + raise Exception('Missing Client ID') + + if not github_config.get('CLIENT_SECRET'): + raise Exception('Missing Client Secret') + + client = app.config['HTTPCLIENT'] + oauth = GithubOAuthConfig(config, config_key) + result = oauth.validate_client_id_and_secret(client) + if not result: + raise Exception('Invalid client id or client secret') + + +def _validate_google_login(config): + """ Validates the Google Login client ID and secret. """ + google_login_config = config.get('GOOGLE_LOGIN_CONFIG') + if not google_login_config: + raise Exception('Missing client ID and client secret') + + if not google_login_config.get('CLIENT_ID'): + raise Exception('Missing Client ID') + + if not google_login_config.get('CLIENT_SECRET'): + raise Exception('Missing Client Secret') + + client = app.config['HTTPCLIENT'] + oauth = GoogleOAuthConfig(config, 'GOOGLE_LOGIN_CONFIG') + result = oauth.validate_client_id_and_secret(client) + if not result: + raise Exception('Invalid client id or client secret') + + +def _validate_ssl(config): + """ Validates the SSL configuration (if enabled). """ + if config.get('PREFERRED_URL_SCHEME', 'http') != 'https': + return + + for filename in SSL_FILENAMES: + if not CONFIG_PROVIDER.volume_file_exists(filename): + raise Exception('Missing required SSL file: %s' % filename) + + with CONFIG_PROVIDER.get_volume_file(SSL_FILENAMES[0]) as f: + cert_contents = f.read() + + # Validate the certificate. + try: + cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert_contents) + except: + raise Exception('Could not parse certificate file. Is it a valid PEM certificate?') + + if cert.has_expired(): + raise Exception('The specified SSL certificate has expired.') + + private_key_path = None + with CONFIG_PROVIDER.get_volume_file(SSL_FILENAMES[1]) as f: + private_key_path = f.name + + if not private_key_path: + # Only in testing. + return + + # Validate the private key with the certificate. + context = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD) + context.use_certificate(cert) + + try: + context.use_privatekey_file(private_key_path) + except: + raise Exception('Could not parse key file. Is it a valid PEM private key?') + + try: + context.check_privatekey() + except OpenSSL.SSL.Error as e: + raise Exception('SSL key failed to validate: %s' % str(e)) + + # Verify the hostname matches the name in the certificate. + common_name = cert.get_subject().commonName + if common_name is None: + raise Exception('Missing CommonName (CN) from SSL certificate') + + # Build the list of allowed host patterns. + hosts = set([common_name]) + + # Find the DNS extension, if any. + for i in range(0, cert.get_extension_count()): + ext = cert.get_extension(i) + if ext.get_short_name() == 'subjectAltName': + value = str(ext) + hosts.update([host.strip()[4:] for host in value.split(',')]) + + # Check each host. + for host in hosts: + if fnmatch(config['SERVER_HOSTNAME'], host): + return + + raise Exception('Supported names "%s" in SSL cert do not match server hostname "%s"' % + (', '.join(list(hosts)), config['SERVER_HOSTNAME'])) + + + +def _validate_ldap(config): + """ Validates the LDAP connection. """ + if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP': + return + + # Note: raises ldap.INVALID_CREDENTIALS on failure + admin_dn = config.get('LDAP_ADMIN_DN') + admin_passwd = config.get('LDAP_ADMIN_PASSWD') + + if not admin_dn: + raise Exception('Missing Admin DN for LDAP configuration') + + if not admin_passwd: + raise Exception('Missing Admin Password for LDAP configuration') + + ldap_uri = config.get('LDAP_URI', 'ldap://localhost') + + try: + with LDAPConnection(ldap_uri, admin_dn, admin_passwd): + pass + except ldap.LDAPError as ex: + values = ex.args[0] if ex.args else {} + raise Exception(values.get('desc', 'Unknown error')) + + +_VALIDATORS = { + 'database': _validate_database, + 'redis': _validate_redis, + 'registry-storage': _validate_registry_storage, + 'mail': _validate_mailing, + 'github-login': _validate_github('GITHUB_LOGIN_CONFIG'), + 'github-trigger': _validate_github('GITHUB_TRIGGER_CONFIG'), + 'google-login': _validate_google_login, + 'ssl': _validate_ssl, + 'ldap': _validate_ldap, +} \ No newline at end of file diff --git a/util/dockerloadformat.py b/util/dockerloadformat.py deleted file mode 100644 index b4a8393c3..000000000 --- a/util/dockerloadformat.py +++ /dev/null @@ -1,132 +0,0 @@ -from util.gzipwrap import GzipWrap, GZIP_BUFFER_SIZE -from util.streamlayerformat import StreamLayerMerger -from app import app - -import copy -import json -import tarfile - -class FileEstimationException(Exception): - """ Exception raised by build_docker_load_stream if the estimated size of the layer TAR - was lower than the actual size. This means the sent TAR header is wrong, and we have - to fail. - """ - pass - - -def build_docker_load_stream(namespace, repository, tag, synthetic_image_id, - layer_json, get_image_iterator, get_layer_iterator): - """ Builds and streams a synthetic .tar.gz that represents a squashed version - of the given layers, in `docker load` V1 format. - """ - return GzipWrap(_import_format_generator(namespace, repository, tag, - synthetic_image_id, layer_json, - get_image_iterator, get_layer_iterator)) - - -def _import_format_generator(namespace, repository, tag, synthetic_image_id, - layer_json, get_image_iterator, get_layer_iterator): - # Docker import V1 Format (.tar): - # repositories - JSON file containing a repo -> tag -> image map - # {image ID folder}: - # json - The layer JSON - # layer.tar - The TARed contents of the layer - # VERSION - The docker import version: '1.0' - layer_merger = StreamLayerMerger(get_layer_iterator) - - # Yield the repositories file: - synthetic_layer_info = {} - synthetic_layer_info[tag + '.squash'] = synthetic_image_id - - hostname = app.config['SERVER_HOSTNAME'] - repositories = {} - repositories[hostname + '/' + namespace + '/' + repository] = synthetic_layer_info - - yield _tar_file('repositories', json.dumps(repositories)) - - # Yield the image ID folder. - yield _tar_folder(synthetic_image_id) - - # Yield the JSON layer data. - layer_json = _build_layer_json(layer_json, synthetic_image_id) - yield _tar_file(synthetic_image_id + '/json', json.dumps(layer_json)) - - # Yield the VERSION file. - yield _tar_file(synthetic_image_id + '/VERSION', '1.0') - - # Yield the merged layer data's header. - estimated_file_size = 0 - for image in get_image_iterator(): - estimated_file_size += image.storage.uncompressed_size - - yield _tar_file_header(synthetic_image_id + '/layer.tar', estimated_file_size) - - # Yield the contents of the merged layer. - yielded_size = 0 - for entry in layer_merger.get_generator(): - yield entry - yielded_size += len(entry) - - # If the yielded size is more than the estimated size (which is unlikely but possible), then - # raise an exception since the tar header will be wrong. - if yielded_size > estimated_file_size: - raise FileEstimationException() - - # If the yielded size is less than the estimated size (which is likely), fill the rest with - # zeros. - if yielded_size < estimated_file_size: - to_yield = estimated_file_size - yielded_size - while to_yield > 0: - yielded = min(to_yield, GZIP_BUFFER_SIZE) - yield '\0' * yielded - to_yield -= yielded - - # Yield any file padding to 512 bytes that is necessary. - yield _tar_file_padding(estimated_file_size) - - # Last two records are empty in TAR spec. - yield '\0' * 512 - yield '\0' * 512 - - -def _build_layer_json(layer_json, synthetic_image_id): - updated_json = copy.deepcopy(layer_json) - updated_json['id'] = synthetic_image_id - - if 'parent' in updated_json: - del updated_json['parent'] - - if 'config' in updated_json and 'Image' in updated_json['config']: - updated_json['config']['Image'] = synthetic_image_id - - if 'container_config' in updated_json and 'Image' in updated_json['container_config']: - updated_json['container_config']['Image'] = synthetic_image_id - - return updated_json - - -def _tar_file(name, contents): - length = len(contents) - tar_data = _tar_file_header(name, length) - tar_data += contents - tar_data += _tar_file_padding(length) - return tar_data - - -def _tar_file_padding(length): - if length % 512 != 0: - return '\0' * (512 - (length % 512)) - - return '' - -def _tar_file_header(name, file_size): - info = tarfile.TarInfo(name=name) - info.type = tarfile.REGTYPE - info.size = file_size - return info.tobuf() - - -def _tar_folder(name): - info = tarfile.TarInfo(name=name) - info.type = tarfile.DIRTYPE - return info.tobuf() diff --git a/util/gzipstream.py b/util/gzipstream.py index e4f39c6da..0f9ce1e6b 100644 --- a/util/gzipstream.py +++ b/util/gzipstream.py @@ -3,6 +3,7 @@ Defines utility methods for working with gzip streams. """ import zlib +import time # Window size for decompressing GZIP streams. # This results in ZLIB automatically detecting the GZIP headers. @@ -35,4 +36,8 @@ def calculate_size_handler(): size_info.uncompressed_size += len(decompressor.decompress(current_data, CHUNK_SIZE)) current_data = decompressor.unconsumed_tail + # Make sure we allow the scheduler to do other work if we get stuck in this tight loop. + if len(current_data) > 0: + time.sleep(0) + return size_info, fn diff --git a/util/imagetree.py b/util/imagetree.py new file mode 100644 index 000000000..39cd5c3c9 --- /dev/null +++ b/util/imagetree.py @@ -0,0 +1,103 @@ +class ImageTreeNode(object): + """ A node in the image tree. """ + def __init__(self, image): + self.image = image + self.parent = None + self.children = [] + self.tags = [] + + def add_child(self, child): + self.children.append(child) + child.parent = self + + def add_tag(self, tag): + self.tags.append(tag) + + +class ImageTree(object): + """ In-memory tree for easy traversal and lookup of images in a repository. """ + + def __init__(self, all_images, all_tags, base_filter=None): + self._tag_map = {} + self._image_map = {} + + self._build(all_images, all_tags, base_filter) + + def _build(self, all_images, all_tags, base_filter=None): + # Build nodes for each of the images. + for image in all_images: + ancestors = image.ancestors.split('/')[1:-1] + + # Filter any unneeded images. + if base_filter is not None: + if image.id != base_filter and not str(base_filter) in ancestors: + continue + + self._image_map[image.id] = ImageTreeNode(image) + + # Connect the nodes to their parents. + for image_node in self._image_map.values(): + image = image_node.image + parent_image_id = image.ancestors.split('/')[-2] if image.ancestors else None + if not parent_image_id: + continue + + parent_node = self._image_map.get(int(parent_image_id)) + if parent_node is not None: + parent_node.add_child(image_node) + + # Build the tag map. + for tag in all_tags: + image_node = self._image_map.get(tag.image.id) + if not image_node: + continue + + self._tag_map = image_node + image_node.add_tag(tag.name) + + + def find_longest_path(self, image_id, checker): + """ Returns a list of images representing the longest path that matches the given + checker function, starting from the given image_id *exclusive*. + """ + start_node = self._image_map.get(image_id) + if not start_node: + return [] + + return self._find_longest_path(start_node, checker, -1)[1:] + + + def _find_longest_path(self, image_node, checker, index): + found_path = [] + + for child_node in image_node.children: + if not checker(index + 1, child_node.image): + continue + + found = self._find_longest_path(child_node, checker, index + 1) + if found and len(found) > len(found_path): + found_path = found + + return [image_node.image] + found_path + + + def tag_containing_image(self, image): + """ Returns the name of the closest tag containing the given image. """ + if not image: + return None + + # Check the current image for a tag. + image_node = self._image_map.get(image.id) + if image_node is None: + return None + + if image_node.tags: + return image_node.tags[0] + + # Check any deriving images for a tag. + for child_node in image_node.children: + found = self.tag_containing_image(child_node.image) + if found is not None: + return found + + return None \ No newline at end of file diff --git a/util/invoice.tmpl b/util/invoice.tmpl index 95cb5a23a..fac657ef5 100644 --- a/util/invoice.tmpl +++ b/util/invoice.tmpl @@ -8,10 +8,10 @@

Quay.io

- DevTable, LLC
- https://devtable.com
- PO Box 48
- New York, NY 10009 + CoreOS, Inc
+ https://coreos.com
+ PO Box 7775 #55097
+ San Francisco, CA 94120-7775

diff --git a/util/migrateslackwebhook.py b/util/migrateslackwebhook.py new file mode 100644 index 000000000..9d4c3ba61 --- /dev/null +++ b/util/migrateslackwebhook.py @@ -0,0 +1,52 @@ +import logging +import json + +from app import app +from data.database import configure, RepositoryNotification, ExternalNotificationMethod + +configure(app.config) + +logger = logging.getLogger(__name__) + +def run_slackwebhook_migration(): + slack_method = ExternalNotificationMethod.get(ExternalNotificationMethod.name == "slack") + + encountered = set() + while True: + found = list(RepositoryNotification.select().where( + RepositoryNotification.method == slack_method, + RepositoryNotification.config_json ** "%subdomain%", + ~(RepositoryNotification.config_json ** "%url%"))) + + found = [f for f in found if not f.uuid in encountered] + + if not found: + logger.debug('No additional records found') + return + + logger.debug('Found %s records to be changed', len(found)) + for notification in found: + encountered.add(notification.uuid) + + try: + config = json.loads(notification.config_json) + except: + logging.error("Cannot parse config for noticification %s", notification.uuid) + continue + + logger.debug("Checking notification %s", notification.uuid) + if 'subdomain' in config and 'token' in config: + subdomain = config['subdomain'] + token = config['token'] + new_url = 'https://%s.slack.com/services/hooks/incoming-webhook?token=%s' % (subdomain, token) + config['url'] = new_url + + logger.debug("Updating notification %s to URL: %s", notification.uuid, new_url) + notification.config_json = json.dumps(config) + notification.save() + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + logging.getLogger('boto').setLevel(logging.CRITICAL) + + run_slackwebhook_migration() diff --git a/util/names.py b/util/names.py index 31546d450..f37f0135d 100644 --- a/util/names.py +++ b/util/names.py @@ -4,16 +4,25 @@ from functools import wraps from uuid import uuid4 -def parse_namespace_repository(repository): +def parse_namespace_repository(repository, include_tag=False): parts = repository.rstrip('/').split('/', 1) if len(parts) < 2: namespace = 'library' repository = parts[0] else: (namespace, repository) = parts - repository = urllib.quote_plus(repository) - return (namespace, repository) + if include_tag: + parts = repository.split(':', 1) + if len(parts) < 2: + tag = 'latest' + else: + (repository, tag) = parts + + repository = urllib.quote_plus(repository) + if include_tag: + return (namespace, repository, tag) + return (namespace, repository) def parse_repository_name(f): @wraps(f) @@ -22,6 +31,13 @@ def parse_repository_name(f): return f(namespace, repository, *args, **kwargs) return wrapper +def parse_repository_name_and_tag(f): + @wraps(f) + def wrapper(repository, *args, **kwargs): + namespace, repository, tag = parse_namespace_repository(repository, include_tag=True) + return f(namespace, repository, tag, *args, **kwargs) + return wrapper + def format_robot_username(parent_username, robot_shortname): return '%s+%s' % (parent_username, robot_shortname) diff --git a/util/oauth.py b/util/oauth.py index e0d38d395..ede8823aa 100644 --- a/util/oauth.py +++ b/util/oauth.py @@ -1,9 +1,9 @@ import urlparse class OAuthConfig(object): - def __init__(self, app, key_name): + def __init__(self, config, key_name): self.key_name = key_name - self.config = app.config.get(key_name) or {} + self.config = config.get(key_name) or {} def service_name(self): raise NotImplementedError @@ -17,6 +17,9 @@ class OAuthConfig(object): def login_endpoint(self): raise NotImplementedError + def validate_client_id_and_secret(self, http_client): + raise NotImplementedError + def client_id(self): return self.config.get('CLIENT_ID') @@ -31,8 +34,8 @@ class OAuthConfig(object): class GithubOAuthConfig(OAuthConfig): - def __init__(self, app, key_name): - super(GithubOAuthConfig, self).__init__(app, key_name) + def __init__(self, config, key_name): + super(GithubOAuthConfig, self).__init__(config, key_name) def service_name(self): return 'GitHub' @@ -63,6 +66,31 @@ class GithubOAuthConfig(OAuthConfig): api_endpoint = self._api_endpoint() return self._get_url(api_endpoint, 'user/emails') + def validate_client_id_and_secret(self, http_client): + # First: Verify that the github endpoint is actually Github by checking for the + # X-GitHub-Request-Id here. + api_endpoint = self._api_endpoint() + result = http_client.get(api_endpoint, auth=(self.client_id(), self.client_secret()), timeout=5) + if not 'X-GitHub-Request-Id' in result.headers: + raise Exception('Endpoint is not a Github (Enterprise) installation') + + # Next: Verify the client ID and secret. + # Note: The following code is a hack until such time as Github officially adds an API endpoint + # for verifying a {client_id, client_secret} pair. That being said, this hack was given to us + # *by a Github Engineer*, so I think it is okay for the time being :) + # + # TODO(jschorr): Replace with the real API call once added. + # + # Hitting the endpoint applications/{client_id}/tokens/foo will result in the following + # behavior IF the client_id is given as the HTTP username and the client_secret as the HTTP + # password: + # - If the {client_id, client_secret} pair is invalid in some way, we get a 401 error. + # - If the pair is valid, then we get a 404 because the 'foo' token does not exists. + validate_endpoint = self._get_url(api_endpoint, 'applications/%s/tokens/foo' % self.client_id()) + result = http_client.get(validate_endpoint, auth=(self.client_id(), self.client_secret()), + timeout=5) + return result.status_code == 404 + def get_public_config(self): return { 'CLIENT_ID': self.client_id(), @@ -73,8 +101,8 @@ class GithubOAuthConfig(OAuthConfig): class GoogleOAuthConfig(OAuthConfig): - def __init__(self, app, key_name): - super(GoogleOAuthConfig, self).__init__(app, key_name) + def __init__(self, config, key_name): + super(GoogleOAuthConfig, self).__init__(config, key_name) def service_name(self): return 'Google' @@ -88,6 +116,23 @@ class GoogleOAuthConfig(OAuthConfig): def user_endpoint(self): return 'https://www.googleapis.com/oauth2/v1/userinfo' + def validate_client_id_and_secret(self, http_client): + # To verify the Google client ID and secret, we hit the + # https://www.googleapis.com/oauth2/v3/token endpoint with an invalid request. If the client + # ID or secret are invalid, we get returned a 403 Unauthorized. Otherwise, we get returned + # another response code. + url = 'https://www.googleapis.com/oauth2/v3/token' + data = { + 'code': 'fakecode', + 'client_id': self.client_id(), + 'client_secret': self.client_secret(), + 'grant_type': 'authorization_code', + 'redirect_uri': 'http://example.com' + } + + result = http_client.post(url, data=data, timeout=5) + return result.status_code != 401 + def get_public_config(self): return { 'CLIENT_ID': self.client_id(), diff --git a/util/queuemetrics.py b/util/queuemetrics.py index ec1444c16..9e0a549f4 100644 --- a/util/queuemetrics.py +++ b/util/queuemetrics.py @@ -1,7 +1,7 @@ import logging -import boto -from multiprocessing import Process, Queue +from util.cloudwatch import get_queue + logger = logging.getLogger(__name__) @@ -11,7 +11,8 @@ class NullReporter(object): pass -class QueueingCloudWatchReporter(object): +class CloudWatchReporter(object): + """ CloudWatchReporter reports work queue metrics to CloudWatch """ def __init__(self, request_queue, namespace, need_capacity_name, build_percent_name): self._namespace = namespace self._need_capacity_name = need_capacity_name @@ -33,54 +34,23 @@ class QueueingCloudWatchReporter(object): self._send_to_queue(self._namespace, self._build_percent_name, building_percent, unit='Percent') - -class SendToCloudWatch(Process): - def __init__(self, request_queue, aws_access_key, aws_secret_key): - Process.__init__(self) - self._aws_access_key = aws_access_key - self._aws_secret_key = aws_secret_key - self._put_metrics_queue = request_queue - self.daemon = True - - def run(self): - logger.debug('Starting cloudwatch sender process.') - connection = boto.connect_cloudwatch(self._aws_access_key, self._aws_secret_key) - while True: - put_metric_args, kwargs = self._put_metrics_queue.get() - logger.debug('Got queued put metrics reqeust.') - connection.put_metric_data(*put_metric_args, **kwargs) - - class QueueMetrics(object): + """ + QueueMetrics initializes a reporter for recording metrics of work queues. + """ def __init__(self, app=None): - self.app = app + self._app = app + self._reporter = NullReporter() if app is not None: - self.state = self.init_app(app) - else: - self.state = None + reporter_type = app.config.get('QUEUE_METRICS_TYPE', 'Null') + if reporter_type == 'CloudWatch': + namespace = app.config['QUEUE_METRICS_NAMESPACE'] + req_capacity_name = app.config['QUEUE_METRICS_CAPACITY_SHORTAGE_NAME'] + build_percent_name = app.config['QUEUE_METRICS_BUILD_PERCENT_NAME'] - def init_app(self, app): - analytics_type = app.config.get('QUEUE_METRICS_TYPE', 'Null') - - if analytics_type == 'CloudWatch': - access_key = app.config.get('QUEUE_METRICS_AWS_ACCESS_KEY') - secret_key = app.config.get('QUEUE_METRICS_AWS_SECRET_KEY') - namespace = app.config.get('QUEUE_METRICS_NAMESPACE') - req_capacity_name = app.config.get('QUEUE_METRICS_CAPACITY_SHORTAGE_NAME') - build_percent_name = app.config.get('QUEUE_METRICS_BUILD_PERCENT_NAME') - - request_queue = Queue() - reporter = QueueingCloudWatchReporter(request_queue, namespace, req_capacity_name, - build_percent_name) - sender = SendToCloudWatch(request_queue, access_key, secret_key) - sender.start() - else: - reporter = NullReporter() - - # register extension with app - app.extensions = getattr(app, 'extensions', {}) - app.extensions['queuemetrics'] = reporter - return reporter + request_queue = get_queue(app) + self._reporter = CloudWatchReporter(request_queue, namespace, req_capacity_name, + build_percent_name) def __getattr__(self, name): - return getattr(self.state, name, None) + return getattr(self._reporter, name, None) diff --git a/util/signing.py b/util/signing.py new file mode 100644 index 000000000..a57e4ebd7 --- /dev/null +++ b/util/signing.py @@ -0,0 +1,69 @@ +import gpgme +import os +from StringIO import StringIO + +class GPG2Signer(object): + """ Helper class for signing data using GPG2. """ + def __init__(self, app, key_directory): + if not app.config.get('GPG2_PRIVATE_KEY_NAME'): + raise Exception('Missing configuration key GPG2_PRIVATE_KEY_NAME') + + if not app.config.get('GPG2_PRIVATE_KEY_FILENAME'): + raise Exception('Missing configuration key GPG2_PRIVATE_KEY_FILENAME') + + if not app.config.get('GPG2_PUBLIC_KEY_FILENAME'): + raise Exception('Missing configuration key GPG2_PUBLIC_KEY_FILENAME') + + self._ctx = gpgme.Context() + self._ctx.armor = True + self._private_key_name = app.config['GPG2_PRIVATE_KEY_NAME'] + self._public_key_path = os.path.join(key_directory, app.config['GPG2_PUBLIC_KEY_FILENAME']) + + key_file = os.path.join(key_directory, app.config['GPG2_PRIVATE_KEY_FILENAME']) + if not os.path.exists(key_file): + raise Exception('Missing key file %s' % key_file) + + with open(key_file, 'rb') as fp: + self._ctx.import_(fp) + + @property + def name(self): + return 'gpg2' + + @property + def public_key_path(self): + return self._public_key_path + + def detached_sign(self, stream): + """ Signs the given stream, returning the signature. """ + ctx = self._ctx + ctx.signers = [ctx.get_key(self._private_key_name)] + signature = StringIO() + new_sigs = ctx.sign(stream, signature, gpgme.SIG_MODE_DETACH) + + signature.seek(0) + return signature.getvalue() + + +class Signer(object): + def __init__(self, app=None, key_directory=None): + self.app = app + if app is not None: + self.state = self.init_app(app, key_directory) + else: + self.state = None + + def init_app(self, app, key_directory): + preference = app.config.get('SIGNING_ENGINE', None) + if preference is None: + return None + + return SIGNING_ENGINES[preference](app, key_directory) + + def __getattr__(self, name): + return getattr(self.state, name, None) + + +SIGNING_ENGINES = { + 'gpg2': GPG2Signer +} \ No newline at end of file diff --git a/util/streamlayerformat.py b/util/streamlayerformat.py index 914dea4a2..686c16204 100644 --- a/util/streamlayerformat.py +++ b/util/streamlayerformat.py @@ -11,8 +11,8 @@ AUFS_WHITEOUT_PREFIX_LENGTH = len(AUFS_WHITEOUT) class StreamLayerMerger(TarLayerFormat): """ Class which creates a generator of the combined TAR data for a set of Docker layers. """ - def __init__(self, layer_iterator): - super(StreamLayerMerger, self).__init__(layer_iterator) + def __init__(self, layer_iterator, path_prefix=None): + super(StreamLayerMerger, self).__init__(layer_iterator, path_prefix) self.path_trie = marisa_trie.Trie() self.path_encountered = [] diff --git a/util/systemlogs.py b/util/systemlogs.py new file mode 100644 index 000000000..e2705b2d6 --- /dev/null +++ b/util/systemlogs.py @@ -0,0 +1,15 @@ +import tarfile +import os +import cStringIO + +def build_logs_archive(app): + """ Builds a .tar.gz with the contents of the system logs found for the given app and returns + the binary contents. + """ + path = app.config['SYSTEM_LOGS_PATH'] + buf = cStringIO.StringIO() + + with tarfile.open(mode="w:gz", fileobj=buf) as tar: + tar.add(path, arcname=os.path.basename(path)) + + return buf.getvalue() \ No newline at end of file diff --git a/util/tarlayerformat.py b/util/tarlayerformat.py index 3468678c5..111e0f731 100644 --- a/util/tarlayerformat.py +++ b/util/tarlayerformat.py @@ -1,5 +1,6 @@ import os import tarfile +import copy class TarLayerReadException(Exception): """ Exception raised when reading a layer has failed. """ @@ -8,8 +9,9 @@ class TarLayerReadException(Exception): class TarLayerFormat(object): """ Class which creates a generator of the combined TAR data. """ - def __init__(self, tar_iterator): + def __init__(self, tar_iterator, path_prefix=None): self.tar_iterator = tar_iterator + self.path_prefix = path_prefix def get_generator(self): for current_tar in self.tar_iterator(): @@ -36,7 +38,20 @@ class TarLayerFormat(object): continue # Yield the tar header. - yield tar_info.tobuf() + if self.path_prefix: + # Note: We use a copy here because we need to make sure we copy over all the internal + # data of the tar header. We cannot use frombuf(tobuf()), however, because it doesn't + # properly handle large filenames. + clone = copy.deepcopy(tar_info) + clone.name = os.path.join(self.path_prefix, clone.name) + + # If the entry is a *hard* link, then prefix it as well. Soft links are relative. + if clone.linkname and clone.type == tarfile.LNKTYPE: + clone.linkname = os.path.join(self.path_prefix, clone.linkname) + + yield clone.tobuf() + else: + yield tar_info.tobuf() # Try to extract any file contents for the tar. If found, we yield them as well. if tar_info.isreg(): diff --git a/util/useremails.py b/util/useremails.py index 0b8790b3b..fb562fde4 100644 --- a/util/useremails.py +++ b/util/useremails.py @@ -1,5 +1,6 @@ import logging import traceback +import json from flask.ext.mail import Message @@ -13,7 +14,42 @@ template_env = get_template_env("emails") class CannotSendEmailException(Exception): pass -def send_email(recipient, subject, template_file, parameters): +class GmailAction(object): + """ Represents an action that can be taken in Gmail in response to the email. """ + def __init__(self, metadata): + self.metadata = metadata + + @staticmethod + def confirm(name, url, description): + return GmailAction({ + "@context": "http://schema.org", + "@type": "EmailMessage", + "action": { + "@type": 'ConfirmAction', + "name": name, + "handler": { + "@type": "HttpActionHandler", + "url": get_app_url() + '/' + url + } + }, + "description": description + }) + + @staticmethod + def view(name, url, description): + return GmailAction({ + "@context": "http://schema.org", + "@type": "EmailMessage", + "action": { + "@type": 'ViewAction', + "name": name, + "url": get_app_url() + '/' + url + }, + "description": description + }) + + +def send_email(recipient, subject, template_file, parameters, action=None): app_title = app.config['REGISTRY_TITLE_SHORT'] app_url = get_app_url() @@ -29,7 +65,8 @@ def send_email(recipient, subject, template_file, parameters): 'app_logo': 'https://quay.io/static/img/quay-logo.png', # TODO: make this pull from config 'app_url': app_url, 'app_title': app_title, - 'app_link': app_link_handler + 'app_link': app_link_handler, + 'action_metadata': json.dumps(action.metadata) if action else None }) rendered_html = template_env.get_template(template_file + '.html').render(parameters) @@ -61,25 +98,34 @@ def send_change_email(username, email, token): }) def send_confirmation_email(username, email, token): + action = GmailAction.confirm('Confirm E-mail', 'confirm?code=' + token, + 'Verification of e-mail address') + send_email(email, 'Please confirm your e-mail address', 'confirmemail', { 'username': username, 'token': token - }) + }, action=action) def send_repo_authorization_email(namespace, repository, email, token): + action = GmailAction.confirm('Verify E-mail', 'authrepoemail?code=' + token, + 'Verification of e-mail address') + subject = 'Please verify your e-mail address for repository %s/%s' % (namespace, repository) send_email(email, subject, 'repoauthorizeemail', { 'namespace': namespace, 'repository': repository, 'token': token - }) + }, action=action) def send_recovery_email(email, token): + action = GmailAction.view('Recover Account', 'recovery?code=' + token, + 'Recovery of an account') + subject = 'Account recovery' send_email(email, subject, 'recovery', { 'email': email, 'token': token - }) + }, action=action) def send_payment_failed(email, username): send_email(email, 'Subscription Payment Failure', 'paymentfailure', { @@ -87,12 +133,15 @@ def send_payment_failed(email, username): }) def send_org_invite_email(member_name, member_email, orgname, team, adder, code): + action = GmailAction.view('Join %s' % team, 'confirminvite?code=' + code, + 'Invitation to join a team') + send_email(member_email, 'Invitation to join team', 'teaminvite', { 'inviter': adder, 'token': code, 'organization': orgname, 'teamname': team - }) + }, action=action) def send_invoice_email(email, contents): diff --git a/web.py b/web.py index 6db09bec7..7c945cc45 100644 --- a/web.py +++ b/web.py @@ -9,7 +9,6 @@ from endpoints.webhooks import webhooks from endpoints.realtime import realtime from endpoints.callbacks import callback - application.register_blueprint(web) application.register_blueprint(callback, url_prefix='/oauth2') application.register_blueprint(api_bp, url_prefix='/api') diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py deleted file mode 100644 index 1991bc9fc..000000000 --- a/workers/dockerfilebuild.py +++ /dev/null @@ -1,704 +0,0 @@ -import logging.config - -if __name__ == "__main__": - logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False) - -import logging -import argparse -import os -import requests -import re -import json -import shutil -import tarfile - -from docker import Client -from docker.utils import kwargs_from_env -from docker.errors import APIError -from tempfile import TemporaryFile, mkdtemp -from zipfile import ZipFile -from functools import partial -from datetime import datetime, timedelta -from threading import Event -from uuid import uuid4 -from collections import defaultdict -from requests.exceptions import ConnectionError - -from data import model -from data.database import BUILD_PHASE -from workers.worker import Worker, WorkerUnhealthyException, JobException -from app import userfiles as user_files, build_logs, sentry, dockerfile_build_queue -from endpoints.notificationhelper import spawn_notification -from util.safetar import safe_extractall -from util.dockerfileparse import parse_dockerfile, ParsedDockerfile, serialize_dockerfile - - -logger = logging.getLogger(__name__) - -TIMEOUT_PERIOD_MINUTES = 20 -CACHE_EXPIRATION_PERIOD_HOURS = 24 -NO_TAGS = [':'] -RESERVATION_TIME = (TIMEOUT_PERIOD_MINUTES + 5) * 60 - -def build_docker_args(): - args = kwargs_from_env() - if 'tls' in args and os.environ.get('IGNORE_TLS_ISSUES', False): - args['tls'].verify = False - return args - - -def matches_system_error(status_str): - """ Returns true if the given status string matches a known system error in the - Docker builder. - """ - KNOWN_MATCHES = ['lxc-start: invalid', 'lxc-start: failed to', 'lxc-start: Permission denied'] - - for match in KNOWN_MATCHES: - # 10 because we might have a Unix control code at the start. - found = status_str.find(match[0:len(match) + 10]) - if found >= 0 and found <= 10: - return True - - return False - - -class StatusWrapper(object): - def __init__(self, build_uuid): - self._uuid = build_uuid - self._status = { - 'total_commands': None, - 'current_command': None, - 'push_completion': 0.0, - 'pull_completion': 0.0, - } - - self.__exit__(None, None, None) - - def __enter__(self): - return self._status - - def __exit__(self, exc_type, value, traceback): - build_logs.set_status(self._uuid, self._status) - - -class _IncompleteJsonError(Exception): - def __init__(self, start_from): - self.start_from = start_from - - -class _StreamingJSONDecoder(json.JSONDecoder): - FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL - WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) - - def decode(self, s, _w=WHITESPACE.match): - """Return the Python representation of ``s`` (a ``str`` or ``unicode`` - instance containing a JSON document) - - """ - start_from = 0 - while start_from < len(s): - try: - obj, end = self.raw_decode(s[start_from:], idx=_w(s[start_from:], 0).end()) - except ValueError: - raise _IncompleteJsonError(start_from) - end = _w(s[start_from:], end).end() - start_from += end - yield obj - - -class StreamingDockerClient(Client): - def _stream_helper(self, response): - """Generator for data coming from a chunked-encoded HTTP response.""" - content_buf = '' - for content in response.iter_content(chunk_size=256): - content_buf += content - try: - for val in json.loads(content_buf, cls=_StreamingJSONDecoder): - yield val - content_buf = '' - except _IncompleteJsonError as exc: - content_buf = content_buf[exc.start_from:] - - -class DockerfileBuildContext(object): - def __init__(self, build_context_dir, dockerfile_subdir, repo, tag_names, - push_token, build_uuid, cache_size_gb, pull_credentials=None): - self._build_dir = build_context_dir - self._dockerfile_subdir = dockerfile_subdir - self._repo = repo - self._tag_names = tag_names - self._push_token = push_token - self._status = StatusWrapper(build_uuid) - self._build_logger = partial(build_logs.append_log_message, build_uuid) - self._pull_credentials = pull_credentials - self._cache_size_gb = cache_size_gb - - # Note: We have two different clients here because we (potentially) login - # with both, but with different credentials that we do not want shared between - # the build and push operations. - self._push_cl = StreamingDockerClient(timeout=1200, **build_docker_args()) - self._build_cl = StreamingDockerClient(timeout=1200, **build_docker_args()) - - dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir, - 'Dockerfile') - if not os.path.exists(dockerfile_path): - raise RuntimeError('Build job did not contain a Dockerfile.') - - # Compute the number of steps - with open(dockerfile_path, 'r') as dockerfileobj: - self._parsed_dockerfile = parse_dockerfile(dockerfileobj.read()) - - self.__inject_quay_repo_env(self._parsed_dockerfile, repo) - self._num_steps = len(self._parsed_dockerfile.commands) - - with open(dockerfile_path, 'w') as dockerfileobj: - dockerfileobj.write(serialize_dockerfile(self._parsed_dockerfile)) - - logger.debug('Will build and push to repo %s with tags named: %s', self._repo, - self._tag_names) - - def __enter__(self): - try: - self.__cleanup_containers() - self.__cleanup_images() - self.__prune_cache() - except APIError: - sentry.client.captureException() - message = 'Docker installation is no longer healthy.' - logger.exception(message) - raise WorkerUnhealthyException(message) - - return self - - def __exit__(self, exc_type, value, traceback): - shutil.rmtree(self._build_dir) - - try: - self.__cleanup_containers() - except APIError: - sentry.client.captureException() - message = 'Docker installation is no longer healthy.' - logger.exception(message) - raise WorkerUnhealthyException(message) - - @staticmethod - def __inject_quay_repo_env(parsed_dockerfile, quay_reponame): - env_command = { - 'command': 'ENV', - 'parameters': 'QUAY_REPOSITORY %s' % quay_reponame - } - for index, command in reversed(list(enumerate(parsed_dockerfile.commands))): - if command['command'] == 'FROM': - new_command_index = index + 1 - logger.debug('Injecting env command at dockerfile index: %s', new_command_index) - parsed_dockerfile.commands.insert(new_command_index, env_command) - break - - @staticmethod - def __total_completion(statuses, total_images): - percentage_with_sizes = float(len(statuses.values()))/total_images - sent_bytes = sum([status['current'] for status in statuses.values()]) - total_bytes = sum([status['total'] for status in statuses.values()]) - return float(sent_bytes)/total_bytes*percentage_with_sizes - - @staticmethod - def __monitor_completion(status_stream, required_message, status_updater, status_completion_key, - num_images=0): - images = {} - for status in status_stream: - logger.debug('%s: %s', status_completion_key, status) - if 'status' in status: - status_msg = status['status'] - - if status_msg == required_message: - if 'progressDetail' in status and 'id' in status: - image_id = status['id'] - detail = status['progressDetail'] - - if 'current' in detail and 'total' in detail: - images[image_id] = detail - with status_updater as status_update: - status_update[status_completion_key] = \ - DockerfileBuildContext.__total_completion(images, max(len(images), num_images)) - - elif 'errorDetail' in status: - message = 'Error pushing image.' - if 'message' in status['errorDetail']: - message = str(status['errorDetail']['message']) - - raise RuntimeError(message) - - def pull(self): - image_and_tag_tuple = self._parsed_dockerfile.get_image_and_tag() - if image_and_tag_tuple is None or image_and_tag_tuple[0] is None: - self._build_logger('Missing FROM command in Dockerfile', build_logs.ERROR) - raise JobException('Missing FROM command in Dockerfile') - - image_and_tag = ':'.join(image_and_tag_tuple) - - # Login with the specified credentials (if any). - if self._pull_credentials: - logger.debug('Logging in with pull credentials: %s@%s', - self._pull_credentials['username'], self._pull_credentials['registry']) - - self._build_logger('Pulling base image: %s' % image_and_tag, log_data={ - 'phasestep': 'login', - 'username': self._pull_credentials['username'], - 'registry': self._pull_credentials['registry'] - }) - - self._build_cl.login(self._pull_credentials['username'], self._pull_credentials['password'], - registry=self._pull_credentials['registry'], reauth=True) - else: - self._build_logger('Pulling base image: %s' % image_and_tag, log_data={ - 'phasestep': 'pull', - 'repo_url': image_and_tag - }) - - pull_status = self._build_cl.pull(image_and_tag, stream=True) - - self.__monitor_completion(pull_status, 'Downloading', self._status, 'pull_completion') - - def build(self, reservation_extension_method): - # Start the build itself. - logger.debug('Starting build.') - - with self._status as status: - status['total_commands'] = self._num_steps - - logger.debug('Building to tags named: %s', self._tag_names) - context_path = os.path.join(self._build_dir, self._dockerfile_subdir) - - logger.debug('Final context path: %s exists: %s', context_path, - os.path.exists(context_path)) - - build_status = self._build_cl.build(path=context_path, stream=True) - - current_step = 0 - built_image = None - for status in build_status: - fully_unwrapped = "" - if isinstance(status, dict): - keys_to_extract = ['error', 'status', 'stream'] - for key in keys_to_extract: - if key in status: - fully_unwrapped = status[key] - break - - if not fully_unwrapped: - logger.debug('Status dict did not have any extractable keys and was: %s', status) - elif isinstance(status, basestring): - fully_unwrapped = status - - status_str = str(fully_unwrapped.encode('utf-8')) - - # Check for system errors when building. - # DISABLED: LXC is super flaky, but this is causing build nodes to spasm. - #if matches_system_error(status_str): - # raise WorkerUnhealthyException(status_str) - - logger.debug('Status: %s', status_str) - step_increment = re.search(r'Step ([0-9]+) :', status_str) - if step_increment: - self._build_logger(status_str, build_logs.COMMAND) - current_step = int(step_increment.group(1)) - logger.debug('Step now: %s/%s', current_step, self._num_steps) - with self._status as status_update: - status_update['current_command'] = current_step - - # Tell the queue that we're making progress every time we advance a step - reservation_extension_method(RESERVATION_TIME) - continue - else: - self._build_logger(status_str) - - complete = re.match(r'Successfully built ([a-z0-9]+)$', status_str) - if complete: - built_image = complete.group(1) - logger.debug('Final image ID is: %s', built_image) - continue - - # Get the image count - if not built_image: - return - - return built_image - - def push(self, built_image): - # Login to the registry - host = re.match(r'([a-z0-9.:]+)/.+/.+$', self._repo) - if not host: - raise RuntimeError('Invalid repo name: %s' % self._repo) - - for protocol in ['https', 'http']: - registry_endpoint = '%s://%s/v1/' % (protocol, host.group(1)) - logger.debug('Attempting login to registry: %s', registry_endpoint) - - try: - self._push_cl.login('$token', self._push_token, registry=registry_endpoint) - break - except APIError: - pass # Probably the wrong protocol - - for tag in self._tag_names: - logger.debug('Tagging image %s as %s:%s', built_image, self._repo, tag) - self._push_cl.tag(built_image, self._repo, tag) - - history = self._push_cl.history(built_image) - num_images = len(history) - - logger.debug('Pushing to repo %s', self._repo) - resp = self._push_cl.push(self._repo, stream=True) - self.__monitor_completion(resp, 'Pushing', self._status, 'push_completion', num_images) - - def __cleanup_containers(self): - # First clean up any containers that might be holding the images - for running in self._build_cl.containers(quiet=True): - logger.debug('Killing container: %s', running['Id']) - self._build_cl.kill(running['Id']) - - # Next, remove all of the containers (which should all now be killed) - for container in self._build_cl.containers(all=True, quiet=True): - logger.debug('Removing container: %s', container['Id']) - self._build_cl.remove_container(container['Id']) - - def __cleanup_images(self): - """ Remove tags on internal nodes, and remove images older than the expiratino time. """ - ids_to_images, ids_to_children = self.__compute_image_graph() - - # Untag all internal nodes, which are usually the base images - for internal_id in ids_to_children.keys(): - internal = ids_to_images[internal_id] - if internal['RepoTags'] != NO_TAGS: - for tag_name in internal['RepoTags']: - self._build_cl.remove_image(tag_name) - - # Make sure all of the leaves have gibberish tags, and remove those older than our expiration - leaves = set(ids_to_images.keys()) - set(ids_to_children.keys()) - now = datetime.now() - for leaf_id in leaves: - leaf = ids_to_images[leaf_id] - - created = datetime.fromtimestamp(leaf['Created']) - expiration = created + timedelta(hours=CACHE_EXPIRATION_PERIOD_HOURS) - if expiration > now: - # Assign a new tag as a uuid to preserve this image - new_tag = str(uuid4()) - self._build_cl.tag(leaf['Id'], new_tag) - - # Remove all of the existing tags - if leaf['RepoTags'] != NO_TAGS: - for tag_name in leaf['RepoTags']: - self._build_cl.remove_image(tag_name) - - def __prune_cache(self): - """ Remove the oldest leaf image until the cache size is the desired size. """ - - logger.debug('Pruning cache to size(gb): %s', self._cache_size_gb) - while self.__compute_cache_size_gb() > self._cache_size_gb: - logger.debug('Locating the oldest image in the cache to prune.') - # Find the oldest tagged image and remove it - oldest_creation_time = datetime.max - oldest_image = None - for image in self._build_cl.images(): - created = datetime.fromtimestamp(image['Created']) - if created < oldest_creation_time: - oldest_creation_time = created - oldest_image = image - - logger.debug('Removing oldest image from cache: %s', oldest_image['Id']) - # Remove all tags on the oldest image - if oldest_image['RepoTags'] == NO_TAGS: - # Remove the image id directly since there are no tags - self._build_cl.remove_image(oldest_image['Id']) - else: - # Remove all tags - for tag_name in oldest_image['RepoTags']: - self._build_cl.remove_image(tag_name) - - def __compute_cache_size_gb(self): - all_images = self._build_cl.images(all=True) - size_in_bytes = sum([img['Size'] for img in all_images]) - size_in_gb = float(size_in_bytes)/1024/1024/1024 - logger.debug('Computed cache size(gb) of: %s', size_in_gb) - return size_in_gb - - def __compute_image_graph(self): - all_images = self._build_cl.images(all=True) - - ids_to_images = {} - ids_to_children = defaultdict(list) - for image in all_images: - if image['ParentId'] != '': - ids_to_children[image['ParentId']].append(image) - ids_to_images[image['Id']] = image - - return (ids_to_images, ids_to_children) - - -class DockerfileBuildWorker(Worker): - def __init__(self, cache_size_gb, *vargs, **kwargs): - super(DockerfileBuildWorker, self).__init__(*vargs, **kwargs) - - self._mime_processors = { - 'application/zip': DockerfileBuildWorker.__prepare_zip, - 'application/x-zip-compressed': DockerfileBuildWorker.__prepare_zip, - 'text/plain': DockerfileBuildWorker.__prepare_dockerfile, - 'application/octet-stream': DockerfileBuildWorker.__prepare_dockerfile, - 'application/x-tar': DockerfileBuildWorker.__prepare_tarball, - 'application/gzip': DockerfileBuildWorker.__prepare_tarball, - 'application/x-gzip': DockerfileBuildWorker.__prepare_tarball, - } - - self._timeout = Event() - self._cache_size_gb = cache_size_gb - - @staticmethod - def __prepare_zip(request_file): - build_dir = mkdtemp(prefix='docker-build-') - - # Save the zip file to temp somewhere - with TemporaryFile() as zip_file: - zip_file.write(request_file.content) - to_extract = ZipFile(zip_file) - to_extract.extractall(build_dir) - - return build_dir - - @staticmethod - def __prepare_dockerfile(request_file): - build_dir = mkdtemp(prefix='docker-build-') - dockerfile_path = os.path.join(build_dir, "Dockerfile") - with open(dockerfile_path, 'w') as dockerfile: - dockerfile.write(request_file.content) - - return build_dir - - @staticmethod - def __prepare_tarball(request_file): - build_dir = mkdtemp(prefix='docker-build-') - - # Save the zip file to temp somewhere - with tarfile.open(mode='r|*', fileobj=request_file.raw) as tar_stream: - safe_extractall(tar_stream, build_dir) - - return build_dir - - def watchdog(self): - logger.debug('Running build watchdog code.') - try: - docker_cl = Client(**build_docker_args()) - - # Iterate the running containers and kill ones that have been running more than 20 minutes - for container in docker_cl.containers(): - start_time = datetime.fromtimestamp(container['Created']) - running_time = datetime.now() - start_time - if running_time > timedelta(minutes=TIMEOUT_PERIOD_MINUTES): - logger.warning('Container has been running too long: %s with command: %s', - container['Id'], container['Command']) - docker_cl.kill(container['Id']) - self._timeout.set() - - except ConnectionError as exc: - logger.exception('Watchdog exception') - raise WorkerUnhealthyException(exc.message) - - def process_queue_item(self, job_details): - self._timeout.clear() - - # Make sure we have more information for debugging problems - sentry.client.user_context(job_details) - - repository_build = model.get_repository_build(job_details['build_uuid']) - - pull_credentials = job_details.get('pull_credentials', None) - - job_config = json.loads(repository_build.job_config) - - resource_url = user_files.get_file_url(repository_build.resource_key, requires_cors=False) - tag_names = job_config['docker_tags'] - build_subdir = job_config['build_subdir'] - - # TODO remove the top branch when there are no more jobs with a repository config - if 'repository' in job_config: - repo = job_config['repository'] - else: - repo = '%s/%s/%s' % (job_config['registry'], - repository_build.repository.namespace_user.username, - repository_build.repository.name) - - access_token = repository_build.access_token.code - - log_appender = partial(build_logs.append_log_message, repository_build.uuid) - - # Lookup and save the version of docker being used. - try: - docker_cl = Client(**build_docker_args()) - docker_version = docker_cl.version().get('Version', '') - except ConnectionError as exc: - logger.exception('Initial connection exception') - raise WorkerUnhealthyException(exc.message) - - dash = docker_version.find('-') - - # Strip any -tutum or whatever off of the version. - if dash > 0: - docker_version = docker_version[:dash] - - log_appender('initializing', build_logs.PHASE, log_data={ - 'docker_version': docker_version - }) - - log_appender('Docker version: %s' % docker_version) - - start_msg = ('Starting job with resource url: %s repo: %s' % (resource_url, repo)) - logger.debug(start_msg) - - docker_resource = requests.get(resource_url, stream=True) - c_type = docker_resource.headers['content-type'] - - if ';' in c_type: - c_type = c_type.split(';')[0] - - filetype_msg = ('Request to build type: %s with repo: %s and tags: %s' % - (c_type, repo, tag_names)) - logger.info(filetype_msg) - log_appender(filetype_msg) - - # Spawn a notification that the build has started. - event_data = { - 'build_id': repository_build.uuid, - 'build_name': repository_build.display_name, - 'docker_tags': tag_names, - 'trigger_id': repository_build.trigger.uuid, - 'trigger_kind': repository_build.trigger.service.name - } - - spawn_notification(repository_build.repository, 'build_start', event_data, - subpage='build?current=%s' % repository_build.uuid, - pathargs=['build', repository_build.uuid]) - - - # Setup a handler for spawning failure messages. - def spawn_failure(message, event_data): - event_data['error_message'] = message - spawn_notification(repository_build.repository, 'build_failure', event_data, - subpage='build?current=%s' % repository_build.uuid, - pathargs=['build', repository_build.uuid]) - - if c_type not in self._mime_processors: - log_appender('error', build_logs.PHASE) - repository_build.phase = BUILD_PHASE.ERROR - repository_build.save() - message = 'Unknown mime-type: %s' % c_type - log_appender(message, build_logs.ERROR) - spawn_failure(message, event_data) - raise JobException(message) - - # Try to build the build directory package from the buildpack. - log_appender('unpacking', build_logs.PHASE) - repository_build.phase = BUILD_PHASE.UNPACKING - repository_build.save() - - build_dir = None - try: - build_dir = self._mime_processors[c_type](docker_resource) - except Exception as ex: - cur_message = ex.message or 'Error while unpacking build package' - log_appender(cur_message, build_logs.ERROR) - spawn_failure(cur_message, event_data) - raise JobException(cur_message) - - # Start the build process. - try: - with DockerfileBuildContext(build_dir, build_subdir, repo, tag_names, access_token, - repository_build.uuid, self._cache_size_gb, - pull_credentials) as build_ctxt: - log_appender('pulling', build_logs.PHASE) - repository_build.phase = BUILD_PHASE.PULLING - repository_build.save() - build_ctxt.pull() - - self.extend_processing(RESERVATION_TIME) - - log_appender('building', build_logs.PHASE) - repository_build.phase = BUILD_PHASE.BUILDING - repository_build.save() - built_image = build_ctxt.build(self.extend_processing) - - if not built_image: - log_appender('error', build_logs.PHASE) - repository_build.phase = BUILD_PHASE.ERROR - repository_build.save() - - message = 'Unable to build dockerfile.' - if self._timeout.is_set(): - message = 'Build step was terminated after %s minutes.' % TIMEOUT_PERIOD_MINUTES - - log_appender(message, build_logs.ERROR) - raise JobException(message) - - self.extend_processing(RESERVATION_TIME) - - log_appender('pushing', build_logs.PHASE) - repository_build.phase = BUILD_PHASE.PUSHING - repository_build.save() - - build_ctxt.push(built_image) - - log_appender('complete', build_logs.PHASE) - repository_build.phase = BUILD_PHASE.COMPLETE - repository_build.save() - - # Spawn a notification that the build has completed. - spawn_notification(repository_build.repository, 'build_success', event_data, - subpage='build?current=%s' % repository_build.uuid, - pathargs=['build', repository_build.uuid]) - - except WorkerUnhealthyException as exc: - # Spawn a notification that the build has failed. - log_appender('Worker has become unhealthy. Will retry shortly.', build_logs.ERROR) - spawn_failure(exc.message, event_data) - - # Raise the exception to the queue. - raise exc - - except JobException as exc: - # Spawn a notification that the build has failed. - spawn_failure(exc.message, event_data) - - # Raise the exception to the queue. - raise exc - - except ConnectionError as exc: - # A connection exception means the worker has become unhealthy (Docker is down) - # so we re-raise as that exception. - logger.exception('Build connection exception') - log_appender('Docker daemon has gone away. Will retry shortly.', build_logs.ERROR) - raise WorkerUnhealthyException(exc.message) - - except Exception as exc: - # Spawn a notification that the build has failed. - spawn_failure(exc.message, event_data) - - # Write the error to the logs. - sentry.client.captureException() - log_appender('error', build_logs.PHASE) - logger.exception('Exception when processing request.') - repository_build.phase = BUILD_PHASE.ERROR - repository_build.save() - log_appender(str(exc), build_logs.ERROR) - - # Raise the exception to the queue. - raise JobException(str(exc)) - -if __name__ == "__main__": - desc = 'Worker daemon to monitor dockerfile build' - parser = argparse.ArgumentParser(description=desc) - parser.add_argument('--cachegb', default=20, type=float, - help='Maximum cache size in gigabytes.') - args = parser.parse_args() - - worker = DockerfileBuildWorker(args.cachegb, dockerfile_build_queue, - reservation_seconds=RESERVATION_TIME) - worker.start(start_status_server_port=8000) diff --git a/workers/worker.py b/workers/worker.py index 506a16f97..66ab38ba4 100644 --- a/workers/worker.py +++ b/workers/worker.py @@ -11,6 +11,7 @@ from threading import Thread from time import sleep from data.model import db +from data.queue import WorkQueue logger = logging.getLogger(__name__)