diff --git a/.gitlab-ci.jsonnet b/.gitlab-ci.jsonnet index e39933e8f..8ea50c70f 100644 --- a/.gitlab-ci.jsonnet +++ b/.gitlab-ci.jsonnet @@ -85,7 +85,7 @@ local jobs = { }, 'registry-tests': unittest_stage { - script: [ pytest_cmd + ' ./test/registry_tests.py'], + script: [ pytest_cmd + ' ./test/registry/registry_tests.py'], coverage: @"/^TOTAL.*\s+(\d+\%)\s*$/", }, diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 18f69eb0f..66c898294 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -292,7 +292,7 @@ registry-tests: coverage: /^TOTAL.*\s+(\d+\%)\s*$/ image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} script: - - py.test --cov="." --cov-report=html --cov-report=term-missing --timeout=3600 --verbose -x --color=no --show-count ./test/registry_tests.py + - py.test --cov="." --cov-report=html --cov-report=term-missing --timeout=3600 --verbose -x --color=no --show-count ./test/registry/registry_tests.py stage: tests tags: - kubernetes diff --git a/Dockerfile.old b/Dockerfile.old deleted file mode 100644 index 191a5cafe..000000000 --- a/Dockerfile.old +++ /dev/null @@ -1,189 +0,0 @@ -# vim:ft=dockerfile - -FROM phusion/baseimage:0.9.19 - -ENV DEBIAN_FRONTEND noninteractive -ENV HOME /root -ENV QUAYCONF /quay/conf -ENV QUAYDIR /quay -ENV QUAYPATH "." - -RUN mkdir $QUAYDIR -WORKDIR $QUAYDIR - -# This is so we don't break http golang/go#17066 -# When Ubuntu has nginx >= 1.11.0 we can switch back. -RUN add-apt-repository ppa:nginx/development - -# Add Yarn repository until it is officially added to Ubuntu -RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list - -# Install system packages -RUN apt-get update && apt-get upgrade -y # 26MAY2017 -RUN apt-get install -y \ - dnsmasq \ - g++ \ - gdb \ - gdebi-core \ - git \ - jpegoptim \ - libevent-2.0.5 \ - libevent-dev \ - libffi-dev \ - libfreetype6-dev \ - libgpgme11 \ - libgpgme11-dev \ - libjpeg62 \ - libjpeg62-dev \ - libjpeg8 \ - libldap-2.4-2 \ - libldap2-dev \ - libmagic1 \ - libpq-dev \ - libpq5 \ - libsasl2-dev \ - libsasl2-modules \ - monit \ - nginx \ - nodejs \ - optipng \ - openssl \ - python-dbg \ - python-dev \ - python-pip \ - python-virtualenv \ - yarn=0.22.0-1 \ - w3m - -# Install python dependencies - -ADD requirements.txt requirements.txt -RUN virtualenv --distribute venv -RUN venv/bin/pip install -r requirements.txt # 07SEP2016 -RUN venv/bin/pip freeze - -# Check python dependencies for the GPL -# Due to the following bug, pip results must be piped to a file before grepping: -# https://github.com/pypa/pip/pull/3304 -RUN cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs venv/bin/pip --disable-pip-version-check show > pipinfo.txt && \ - test -z $(cat pipinfo.txt | grep GPL | grep -v LGPL) && \ - rm pipinfo.txt - -# Install cfssl -RUN mkdir /gocode -ENV GOPATH /gocode -RUN curl -O https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz && \ - tar -xvf go1.6.linux-amd64.tar.gz && \ - mv go /usr/local && \ - rm -rf go1.6.linux-amd64.tar.gz && \ - /usr/local/go/bin/go get -u github.com/cloudflare/cfssl/cmd/cfssl && \ - /usr/local/go/bin/go get -u github.com/cloudflare/cfssl/cmd/cfssljson && \ - cp /gocode/bin/cfssljson /bin/cfssljson && \ - cp /gocode/bin/cfssl /bin/cfssl && \ - rm -rf /gocode && rm -rf /usr/local/go - -# Install jwtproxy -RUN curl -L -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.1/jwtproxy-linux-x64 -RUN chmod +x /usr/local/bin/jwtproxy - -# Install prometheus-aggregator -RUN curl -L -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator -RUN chmod +x /usr/local/bin/prometheus-aggregator - -# Install front-end dependencies -RUN ln -s /usr/bin/nodejs /usr/bin/node -ADD package.json package.json -ADD tsconfig.json tsconfig.json -ADD webpack.config.js webpack.config.js -ADD yarn.lock yarn.lock -RUN yarn install --ignore-engines - -# Add static files -ADD static static - -# Run Webpack -RUN yarn build - -# Optimize our images -ADD static/img static/img -RUN jpegoptim static/img/**/*.jpg -RUN optipng -clobber -quiet static/img/**/*.png - -RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev libgpgme11-dev nodejs jpegoptim optipng w3m -RUN apt-get autoremove -y -RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# Set up the init system -ADD conf/init/copy_config_files.sh /etc/my_init.d/ -ADD conf/init/doupdatelimits.sh /etc/my_init.d/ -ADD conf/init/copy_syslog_config.sh /etc/my_init.d/ -ADD conf/init/certs_create.sh /etc/my_init.d/ -ADD conf/init/certs_install.sh /etc/my_init.d/ -ADD conf/init/nginx_conf_create.sh /etc/my_init.d/ -ADD conf/init/runmigration.sh /etc/my_init.d/ -ADD conf/init/syslog-ng.conf /etc/syslog-ng/ -ADD conf/init/zz_boot.sh /etc/my_init.d/ -ADD conf/init/service/ /etc/service/ -RUN rm -rf /etc/service/syslog-forwarder - -ADD conf/kill-buildmanager.sh /usr/local/bin/kill-buildmanager.sh -ADD conf/monitrc /etc/monit/monitrc -RUN chmod 0600 /etc/monit/monitrc - -# remove after phusion/baseimage-docker#338 is fixed -ADD conf/init/logrotate.conf /etc/logrotate.conf - -# TODO(ssewell): only works on a detached head, make work with ref -ADD .git/HEAD GIT_HEAD - -# Add all of the files! -ADD . . -RUN mkdir static/fonts static/ldn - -# Download any external libs. -RUN venv/bin/python -m external_libraries -RUN mkdir -p /usr/local/nginx/logs/ - - -RUN pyclean . - -# Cleanup any NPM-related stuff. -RUN rm -rf /root/.npm -RUN rm -rf .npm -RUN rm -rf /usr/local/lib/node_modules -RUN rm -rf /usr/share/yarn/node_modules -RUN rm -rf /root/node_modules -RUN rm -rf node_modules -RUN rm -rf grunt -RUN rm package.json yarn.lock - -# Run the tests -ARG RUN_TESTS=true -ENV RUN_TESTS ${RUN_TESTS} - -ENV RUN_ACI_TESTS False -ADD requirements-tests.txt requirements-tests.txt - -RUN if [ "$RUN_TESTS" = true ]; then \ - venv/bin/pip install -r requirements-tests.txt ;\ - fi - -RUN if [ "$RUN_TESTS" = true ]; then \ - TEST=true PYTHONPATH="." venv/bin/py.test --timeout=7200 --verbose \ - --show-count -x --color=no ./ && rm -rf /var/tmp/; \ - fi - -RUN if [ "$RUN_TESTS" = true ]; then \ - TEST=true PYTHONPATH="." venv/bin/py.test --timeout=7200 --verbose \ - --show-count -x --color=no test/registry_tests.py && rm -rf /var/tmp/;\ - - fi - -RUN rm -rf /root/.cache - -RUN PYTHONPATH=. venv/bin/alembic heads | grep -E '^[0-9a-f]+ \(head\)$' > ALEMBIC_HEAD - -VOLUME ["/conf/stack", "/var/log", "/datastorage", "/tmp", "/conf/etcd"] - -EXPOSE 443 8443 80 diff --git a/Makefile b/Makefile index 780ca4e77..a0f187ce3 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,7 @@ registry-test: TEST=true PYTHONPATH="." py.test \ --cov="." --cov-report=html --cov-report=term-missing \ --timeout=3600 --verbose --show-count -x \ - test/registry_tests.py + test/registry/registry_tests.py test: unit-test registry-test diff --git a/README.md b/README.md index 43f1db291..26a0ff388 100644 --- a/README.md +++ b/README.md @@ -274,7 +274,7 @@ TEST=true python -m test.test_api_usage -f SuiteName TEST=true PYTHONPATH="." py.test --verbose # To run a specific test module -TEST=true PYTHONPATH="." py.test --verbose test/registry_tests.py +TEST=true PYTHONPATH="." py.test --verbose test/registry/registry_tests.py # To run a specific test unique test TEST=true PYTHONPATH="." py.test --verbose test/test_api_usage.py::TestDeleteNamespace diff --git a/ci/tasks/registry.yaml b/ci/tasks/registry.yaml index e27ba3cea..09095a73b 100644 --- a/ci/tasks/registry.yaml +++ b/ci/tasks/registry.yaml @@ -13,4 +13,4 @@ run: pip install --quiet -r quay-pull-request/requirements-tests.txt cd quay-pull-request PYTHONPATH="." py.test --timeout=7200 --verbose \ - --show-count -x test/registry_tests.py + --show-count -x test/registry/registry_tests.py diff --git a/data/cache/impl.py b/data/cache/impl.py index a851dbe07..982e950e9 100644 --- a/data/cache/impl.py +++ b/data/cache/impl.py @@ -40,7 +40,7 @@ class NoopDataModelCache(DataModelCache): class InMemoryDataModelCache(DataModelCache): """ Implementation of the data model cache backed by an in-memory dictionary. """ def __init__(self): - self.cache = ExpiresDict(rebuilder=lambda: {}) + self.cache = ExpiresDict() def retrieve(self, cache_key, loader, should_cache=is_not_none): not_found = [None] diff --git a/endpoints/decorators.py b/endpoints/decorators.py index b075fa1bd..787d5648d 100644 --- a/endpoints/decorators.py +++ b/endpoints/decorators.py @@ -9,7 +9,7 @@ import features from app import app from auth.auth_context import get_authenticated_context -from util.names import parse_namespace_repository +from util.names import parse_namespace_repository, ImplicitLibraryNamespaceNotAllowed from util.http import abort logger = logging.getLogger(__name__) @@ -26,9 +26,14 @@ def parse_repository_name(include_tag=False, def inner(func): @wraps(func) def wrapper(*args, **kwargs): - repo_name_components = parse_namespace_repository(kwargs[incoming_repo_kwarg], - app.config['LIBRARY_NAMESPACE'], - include_tag=include_tag) + try: + repo_name_components = parse_namespace_repository(kwargs[incoming_repo_kwarg], + app.config['LIBRARY_NAMESPACE'], + include_tag=include_tag, + allow_library=features.LIBRARY_SUPPORT) + except ImplicitLibraryNamespaceNotAllowed: + abort(400) + del kwargs[incoming_repo_kwarg] kwargs[ns_kwarg_name] = repo_name_components[0] kwargs[repo_kwarg_name] = repo_name_components[1] diff --git a/test/fixtures.py b/test/fixtures.py index 67a973992..68e461510 100644 --- a/test/fixtures.py +++ b/test/fixtures.py @@ -6,15 +6,18 @@ import pytest import shutil from flask import Flask, jsonify from flask_login import LoginManager +from flask_principal import identity_loaded, Permission, Identity, identity_changed, Principal from peewee import SqliteDatabase, savepoint, InternalError from app import app as application +from auth.permissions import on_identity_loaded from data import model from data.database import close_db_filter, db, configure from data.model.user import LoginWrappedDBUser from endpoints.api import api_bp from endpoints.appr import appr_bp from endpoints.web import web +from endpoints.v1 import v1_bp from endpoints.v2 import v2_bp from endpoints.verbs import verbs as verbs_bp from endpoints.webhooks import webhooks @@ -118,6 +121,9 @@ def appconfig(database_uri): 'autorollback': True, }, "DB_TRANSACTION_FACTORY": _create_transaction, + "DATA_MODEL_CACHE_CONFIG": { + 'engine': 'inmemory', + }, } return conf @@ -167,6 +173,12 @@ def app(appconfig, initialized_db): def load_user(user_uuid): return LoginWrappedDBUser(user_uuid) + @identity_loaded.connect_via(app) + def on_identity_loaded_for_test(sender, identity): + on_identity_loaded(sender, identity) + + Principal(app, use_sessions=False) + app.url_map.converters['regex'] = RegexConverter app.url_map.converters['apirepopath'] = APIRepositoryPathConverter app.url_map.converters['repopath'] = RepositoryPathConverter @@ -175,6 +187,7 @@ def app(appconfig, initialized_db): app.register_blueprint(appr_bp, url_prefix='/cnr') app.register_blueprint(web, url_prefix='/') app.register_blueprint(verbs_bp, url_prefix='/c1') + app.register_blueprint(v1_bp, url_prefix='/v1') app.register_blueprint(v2_bp, url_prefix='/v2') app.register_blueprint(webhooks, url_prefix='/webhooks') diff --git a/test/registry/__init__.py b/test/registry/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/registry/fixtures.py b/test/registry/fixtures.py new file mode 100644 index 000000000..bb079d56f --- /dev/null +++ b/test/registry/fixtures.py @@ -0,0 +1,200 @@ +import copy +import logging.config +import json +import os +import shutil + +from tempfile import NamedTemporaryFile + +import pytest + +from Crypto import Random +from flask import jsonify, g +from flask_principal import Identity + +from app import storage +from data.database import close_db_filter, configure, DerivedStorageForImage, QueueItem, Image +from data import model +from endpoints.csrf import generate_csrf_token +from util.log import logfile_path + +from test.registry.liveserverfixture import LiveServerExecutor + + +@pytest.fixture() +def registry_server_executor(app): + def generate_csrf(): + return generate_csrf_token() + + def set_supports_direct_download(enabled): + storage.put_content(['local_us'], 'supports_direct_download', 'true' if enabled else 'false') + return 'OK' + + def delete_image(image_id): + image = Image.get(docker_image_id=image_id) + image.docker_image_id = 'DELETED' + image.save() + return 'OK' + + def get_storage_replication_entry(image_id): + image = Image.get(docker_image_id=image_id) + QueueItem.select().where(QueueItem.queue_name ** ('%' + image.storage.uuid + '%')).get() + return 'OK' + + def set_feature(feature_name, value): + import features + old_value = features._FEATURES[feature_name].value + features._FEATURES[feature_name].value = value + return jsonify({'old_value': old_value}) + + def clear_derived_cache(): + DerivedStorageForImage.delete().execute() + return 'OK' + + def clear_uncompressed_size(image_id): + image = model.image.get_image_by_id('devtable', 'newrepo', image_id) + image.storage.uncompressed_size = None + image.storage.save() + return 'OK' + + def add_token(): + another_token = model.token.create_delegate_token('devtable', 'newrepo', 'my-new-token', + 'write') + another_token.code = 'somecooltokencode' + another_token.save() + return another_token.code + + def break_database(): + # Close any existing connection. + close_db_filter(None) + + # Reload the database config with an invalid connection. + config = copy.copy(app.config) + config['DB_URI'] = 'sqlite:///not/a/valid/database' + configure(config) + + return 'OK' + + def reload_app(server_hostname): + # Close any existing connection. + close_db_filter(None) + + # Reload the database config. + app.config['SERVER_HOSTNAME'] = server_hostname[len('http://'):] + configure(app.config) + + # Reload random after the process split, as it cannot be used uninitialized across forks. + Random.atfork() + + # Required for anonymous calls to not exception. + g.identity = Identity(None, 'none') + + if os.environ.get('DEBUGLOG') == 'true': + logging.config.fileConfig(logfile_path(debug=True), disable_existing_loggers=False) + + return 'OK' + + def create_app_repository(namespace, name): + user = model.user.get_user(namespace) + model.repository.create_repository(namespace, name, user, repo_kind='application') + return 'OK' + + executor = LiveServerExecutor() + executor.register('generate_csrf', generate_csrf) + executor.register('set_supports_direct_download', set_supports_direct_download) + executor.register('delete_image', delete_image) + executor.register('get_storage_replication_entry', get_storage_replication_entry) + executor.register('set_feature', set_feature) + executor.register('clear_derived_cache', clear_derived_cache) + executor.register('clear_uncompressed_size', clear_uncompressed_size) + executor.register('add_token', add_token) + executor.register('break_database', break_database) + executor.register('reload_app', reload_app) + executor.register('create_app_repository', create_app_repository) + return executor + + +@pytest.fixture() +def liveserver_app(app, registry_server_executor, init_db_path): + registry_server_executor.apply_blueprint_to_app(app) + + if os.environ.get('DEBUG', 'false').lower() == 'true': + app.config['DEBUG'] = True + + # Copy the clean database to a new path. We cannot share the DB created by the + # normal app fixture, as it is already open in the local process. + local_db_file = NamedTemporaryFile(delete=True) + local_db_file.close() + + shutil.copy2(init_db_path, local_db_file.name) + app.config['DB_URI'] = 'sqlite:///{0}'.format(local_db_file.name) + return app + + +@pytest.fixture() +def app_reloader(liveserver, registry_server_executor): + registry_server_executor.on(liveserver).reload_app(liveserver.url) + yield + + +class FeatureFlagValue(object): + """ Helper object which temporarily sets the value of a feature flag. + + Usage: + + with FeatureFlagValue('ANONYMOUS_ACCESS', False, registry_server_executor.on(liveserver)): + ... Features.ANONYMOUS_ACCESS is False in this context ... + """ + + def __init__(self, feature_flag, test_value, executor): + self.feature_flag = feature_flag + self.test_value = test_value + self.executor = executor + + self.old_value = None + + def __enter__(self): + result = self.executor.set_feature(self.feature_flag, self.test_value) + self.old_value = result.json()['old_value'] + + def __exit__(self, type, value, traceback): + self.executor.set_feature(self.feature_flag, self.old_value) + + +class ApiCaller(object): + def __init__(self, liveserver_session, registry_server_executor): + self.liveserver_session = liveserver_session + self.csrf_token = registry_server_executor.on_session(liveserver_session).generate_csrf() + + def conduct_auth(self, username, password): + r = self.post('/api/v1/signin', + data=json.dumps(dict(username=username, password=password)), + headers={'Content-Type': 'application/json'}) + assert r.status_code == 200 + + def _adjust_params(self, kwargs): + if 'params' not in kwargs: + kwargs['params'] = {} + + kwargs['params'].update({ + '_csrf_token': self.csrf_token, + }) + return kwargs + + def get(self, url, **kwargs): + kwargs = self._adjust_params(kwargs) + return self.liveserver_session.get(url, **kwargs) + + def post(self, url, **kwargs): + kwargs = self._adjust_params(kwargs) + return self.liveserver_session.post(url, **kwargs) + + def change_repo_visibility(self, namespace, repository, visibility): + self.post('/api/v1/repository/%s/%s/changevisibility' % (namespace, repository), + data=json.dumps(dict(visibility=visibility)), + headers={'Content-Type': 'application/json'}) + + +@pytest.fixture(scope="function") +def api_caller(liveserver, registry_server_executor): + return ApiCaller(liveserver.new_session(), registry_server_executor) diff --git a/test/registry/liveserverfixture.py b/test/registry/liveserverfixture.py new file mode 100644 index 000000000..18185a5d1 --- /dev/null +++ b/test/registry/liveserverfixture.py @@ -0,0 +1,272 @@ +import inspect +import json +import multiprocessing +import socket +import socketserver +import time + +from contextlib import contextmanager +from urlparse import urlparse, urljoin + +import pytest +import requests + +from flask import request +from flask.blueprints import Blueprint + + +class liveFlaskServer(object): + """ Helper class for spawning a live Flask server for testing. + + Based on https://github.com/jarus/flask-testing/blob/master/flask_testing/utils.py#L421 + """ + def __init__(self, app, port_value): + self.app = app + self._port_value = port_value + self._process = None + + def get_server_url(self): + """ + Return the url of the test server + """ + return 'http://localhost:%s' % self._port_value.value + + def terminate_live_server(self): + if self._process: + self._process.terminate() + + def spawn_live_server(self): + self._process = None + port_value = self._port_value + + def worker(app, port): + # Based on solution: http://stackoverflow.com/a/27598916 + # Monkey-patch the server_bind so we can determine the port bound by Flask. + # This handles the case where the port specified is `0`, which means that + # the OS chooses the port. This is the only known way (currently) of getting + # the port out of Flask once we call `run`. + original_socket_bind = socketserver.TCPServer.server_bind + def socket_bind_wrapper(self): + ret = original_socket_bind(self) + + # Get the port and save it into the port_value, so the parent process + # can read it. + (_, port) = self.socket.getsockname() + port_value.value = port + socketserver.TCPServer.server_bind = original_socket_bind + return ret + + socketserver.TCPServer.server_bind = socket_bind_wrapper + app.run(port=port, use_reloader=False) + + self._process = multiprocessing.Process(target=worker, args=(self.app, 0)) + self._process.start() + + # We must wait for the server to start listening, but give up + # after a specified maximum timeout + timeout = self.app.config.get('LIVESERVER_TIMEOUT', 5) + start_time = time.time() + + while True: + time.sleep(0.1) + + elapsed_time = (time.time() - start_time) + if elapsed_time > timeout: + raise RuntimeError("Failed to start the server after %d seconds. " % timeout) + + if self._can_connect(): + break + + def _can_connect(self): + host, port = self._get_server_address() + if port == 0: + # Port specified by the user was 0, and the OS has not yet assigned + # the proper port. + return False + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + sock.connect((host, port)) + except socket.error: + success = False + else: + success = True + finally: + sock.close() + + return success + + def _get_server_address(self): + """ + Gets the server address used to test the connection with a socket. + Respects both the LIVESERVER_PORT config value and overriding + get_server_url() + """ + parts = urlparse(self.get_server_url()) + + host = parts.hostname + port = parts.port + + if port is None: + if parts.scheme == 'http': + port = 80 + elif parts.scheme == 'https': + port = 443 + else: + raise RuntimeError("Unsupported server url scheme: %s" % parts.scheme) + + return host, port + + +class LiveFixtureServerSession(object): + """ Helper class for calling the live server via a single requests Session. """ + def __init__(self, base_url): + self.base_url = base_url + self.session = requests.Session() + + def _get_url(self, url): + return urljoin(self.base_url, url) + + def get(self, url, **kwargs): + return self.session.get(self._get_url(url), **kwargs) + + def post(self, url, **kwargs): + return self.session.post(self._get_url(url), **kwargs) + + def put(self, url, **kwargs): + return self.session.put(self._get_url(url), **kwargs) + + def delete(self, url, **kwargs): + return self.session.delete(self._get_url(url), **kwargs) + + def request(self, method, url, **kwargs): + return self.session.request(method, self._get_url(url), **kwargs) + + +class LiveFixtureServer(object): + """ Helper for interacting with a live server. """ + def __init__(self, url): + self.url = url + + @contextmanager + def session(self): + """ Yields a session for speaking to the live server. """ + yield LiveFixtureServerSession(self.url) + + def new_session(self): + """ Returns a new session for speaking to the live server. """ + return LiveFixtureServerSession(self.url) + + +@pytest.fixture(scope='function') +def liveserver(liveserver_app): + """ Runs a live Flask server for the app for the duration of the test. + + Based on https://github.com/jarus/flask-testing/blob/master/flask_testing/utils.py#L421 + """ + context = liveserver_app.test_request_context() + context.push() + + port = multiprocessing.Value('i', 0) + live_server = liveFlaskServer(liveserver_app, port) + + try: + live_server.spawn_live_server() + yield LiveFixtureServer(live_server.get_server_url()) + finally: + context.pop() + live_server.terminate_live_server() + + +@pytest.fixture(scope='function') +def liveserver_session(liveserver, liveserver_app): + """ Fixtures which instantiates a liveserver and returns a single session for + interacting with that server. + """ + return LiveFixtureServerSession(liveserver.url) + + +class LiveServerExecutor(object): + """ Helper class which can be used to register functions to be executed in the + same process as the live server. This is necessary because the live server + runs in a different process and, therefore, in order to execute state changes + outside of the server's normal flows (i.e. via code), it must be executed + *in-process* via an HTTP call. The LiveServerExecutor class abstracts away + all the setup for this process. + + Usage: + def _perform_operation(first_param, second_param): + ... do some operation in the app ... + return 'some value' + + @pytest.fixture(scope="session") + def my_server_executor(): + executor = LiveServerExecutor() + executor.register('performoperation', _perform_operation) + return executor + + @pytest.fixture() + def liveserver_app(app, my_server_executor): + ... other app setup here ... + my_server_executor.apply_blueprint_to_app(app) + return app + + def test_mytest(liveserver, my_server_executor): + # Invokes 'performoperation' in the liveserver's process. + my_server_executor.on(liveserver).performoperation('first', 'second') + """ + def __init__(self): + self.funcs = {} + + def register(self, fn_name, fn): + """ Registers the given function under the given name. """ + self.funcs[fn_name] = fn + + def apply_blueprint_to_app(self, app): + """ Applies a blueprint to the app, to support invocation from this executor. """ + testbp = Blueprint('testbp', __name__) + + def build_invoker(fn_name, fn): + path = '/' + fn_name + @testbp.route(path, methods=['POST'], endpoint=fn_name) + def _(**kwargs): + arg_values = request.get_json()['args'] + return fn(*arg_values) + + for fn_name, fn in self.funcs.iteritems(): + build_invoker(fn_name, fn) + + app.register_blueprint(testbp, url_prefix='/__test') + + def on(self, server): + """ Returns an invoker for the given live server. """ + return liveServerExecutorInvoker(self.funcs, server) + + def on_session(self, server_session): + """ Returns an invoker for the given live server session. """ + return liveServerExecutorInvoker(self.funcs, server_session) + + +class liveServerExecutorInvoker(object): + def __init__(self, funcs, server_or_session): + self._funcs = funcs + self._server_or_session = server_or_session + + def __getattribute__(self, name): + if name.startswith('_'): + return object.__getattribute__(self, name) + + if name not in self._funcs: + raise AttributeError('Unknown function: %s' % name) + + def invoker(*args): + path = '/__test/%s' % name + headers = {'Content-Type': 'application/json'} + + if isinstance(self._server_or_session, LiveFixtureServerSession): + return self._server_or_session.post(path, data=json.dumps({'args': args}), headers=headers) + else: + with self._server_or_session.session() as session: + return session.post(path, data=json.dumps({'args': args}), headers=headers) + + return invoker diff --git a/test/registry/protocol_fixtures.py b/test/registry/protocol_fixtures.py new file mode 100644 index 000000000..44ad89964 --- /dev/null +++ b/test/registry/protocol_fixtures.py @@ -0,0 +1,77 @@ +import random +import string + +import pytest + +from Crypto.PublicKey import RSA +from jwkest.jwk import RSAKey + +from test.registry.protocols import Image, layer_bytes_for_contents +from test.registry.protocol_v1 import V1Protocol +from test.registry.protocol_v2 import V2Protocol + + +@pytest.fixture(scope="session") +def basic_images(): + """ Returns basic images for push and pull testing. """ + # Note: order is from base layer down to leaf. + parent_bytes = layer_bytes_for_contents('parent contents') + image_bytes = layer_bytes_for_contents('some contents') + return [ + Image(id='parentid', bytes=parent_bytes, parent_id=None), + Image(id='someid', bytes=image_bytes, parent_id='parentid'), + ] + + +@pytest.fixture(scope="session") +def sized_images(): + """ Returns basic images (with sizes) for push and pull testing. """ + # Note: order is from base layer down to leaf. + parent_bytes = layer_bytes_for_contents('parent contents', mode='') + image_bytes = layer_bytes_for_contents('some contents', mode='') + return [ + Image(id='parentid', bytes=parent_bytes, parent_id=None, size=len(parent_bytes)), + Image(id='someid', bytes=image_bytes, parent_id='parentid', size=len(image_bytes)), + ] + + +@pytest.fixture(scope="session") +def jwk(): + return RSAKey(key=RSA.generate(2048)) + + +@pytest.fixture(params=[V2Protocol]) +def v2_protocol(request, jwk): + return request.param(jwk) + + +@pytest.fixture(params=[V1Protocol]) +def v1_protocol(request, jwk): + return request.param(jwk) + + +@pytest.fixture(params=[V2Protocol]) +def manifest_protocol(request, jwk): + return request.param(jwk) + + +@pytest.fixture(params=[V1Protocol, V2Protocol]) +def loginer(request, jwk): + return request.param(jwk) + + +@pytest.fixture(params=[V1Protocol, V2Protocol]) +def pusher(request, jwk): + return request.param(jwk) + + +@pytest.fixture(params=[V1Protocol, V2Protocol]) +def puller(request, jwk): + return request.param(jwk) + + +@pytest.fixture(scope="session") +def random_layer_data(): + size = 4096 + contents = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(size)) + return layer_bytes_for_contents(contents) diff --git a/test/registry/protocol_v1.py b/test/registry/protocol_v1.py new file mode 100644 index 000000000..a3da149b2 --- /dev/null +++ b/test/registry/protocol_v1.py @@ -0,0 +1,179 @@ +import json + +from cStringIO import StringIO +from enum import Enum, unique + +from digest.checksums import compute_simple +from test.registry.protocols import (RegistryProtocol, Failures, ProtocolOptions, PushResult, + PullResult) + +@unique +class V1ProtocolSteps(Enum): + """ Defines the various steps of the protocol, for matching failures. """ + PUT_IMAGES = 'put-images' + GET_IMAGES = 'get-images' + PUT_TAG = 'put-tag' + + +class V1Protocol(RegistryProtocol): + FAILURE_CODES = { + V1ProtocolSteps.PUT_IMAGES: { + Failures.UNAUTHENTICATED: 401, + Failures.UNAUTHORIZED: 403, + Failures.APP_REPOSITORY: 405, + Failures.INVALID_REPOSITORY: 404, + Failures.DISALLOWED_LIBRARY_NAMESPACE: 400, + }, + V1ProtocolSteps.GET_IMAGES: { + Failures.UNAUTHENTICATED: 403, + Failures.UNAUTHORIZED: 403, + Failures.APP_REPOSITORY: 405, + Failures.ANONYMOUS_NOT_ALLOWED: 401, + Failures.DISALLOWED_LIBRARY_NAMESPACE: 400, + }, + V1ProtocolSteps.PUT_TAG: { + Failures.MISSING_TAG: 404, + Failures.INVALID_TAG: 400, + Failures.INVALID_IMAGES: 400, + }, + } + + def __init__(self, jwk): + pass + + def _auth_for_credentials(self, credentials): + if credentials is None: + return None + + return credentials + + def ping(self, session): + assert session.get('/v1/_ping').status_code == 200 + + def login(self, session, username, password, scopes, expect_success): + data = { + 'username': username, + 'password': password, + } + + response = self.conduct(session, 'POST', '/v1/users/', json_data=data, expected_status=400) + assert (response.text == '"Username or email already exists"') == expect_success + + def pull(self, session, namespace, repo_name, tag_names, images, credentials=None, + expected_failure=None, options=None): + options = options or ProtocolOptions() + auth = self._auth_for_credentials(credentials) + tag_names = [tag_names] if isinstance(tag_names, str) else tag_names + prefix = '/v1/repositories/%s/' % self.repo_name(namespace, repo_name) + + # Ping! + self.ping(session) + + # GET /v1/repositories/{namespace}/{repository}/images + headers = {'X-Docker-Token': 'true'} + result = self.conduct(session, 'GET', prefix + 'images', auth=auth, headers=headers, + expected_status=(200, expected_failure, V1ProtocolSteps.GET_IMAGES)) + if expected_failure is not None: + return + + headers = {} + if credentials is not None: + headers['Authorization'] = 'token ' + result.headers['www-authenticate'] + else: + assert not 'www-authenticate' in result.headers + + # GET /v1/repositories/{namespace}/{repository}/tags + image_ids = self.conduct(session, 'GET', prefix + 'tags', headers=headers).json() + assert len(image_ids.values()) == len(tag_names) + + for tag_name in tag_names: + tag_image_id = image_ids[tag_name] + if not options.munge_shas: + # Ensure we have a matching image ID. + known_ids = {image.id for image in images} + assert tag_image_id in known_ids + + # Retrieve the ancestry of the tagged image. + image_prefix = '/v1/images/%s/' % tag_image_id + ancestors = self.conduct(session, 'GET', image_prefix + 'ancestry', headers=headers).json() + + assert len(ancestors) == len(images) + for index, image_id in enumerate(reversed(ancestors)): + # /v1/images/{imageID}/{ancestry, json, layer} + image_prefix = '/v1/images/%s/' % image_id + self.conduct(session, 'GET', image_prefix + 'ancestry', headers=headers) + + result = self.conduct(session, 'GET', image_prefix + 'json', headers=headers) + assert result.json()['id'] == image_id + + # Ensure we can HEAD the image layer. + self.conduct(session, 'HEAD', image_prefix + 'layer', headers=headers) + + # And retrieve the layer data. + result = self.conduct(session, 'GET', image_prefix + 'layer', headers=headers) + assert result.content == images[index].bytes + + return PullResult(manifests=None, image_ids=image_ids) + + def push(self, session, namespace, repo_name, tag_names, images, credentials=None, + expected_failure=None, options=None): + auth = self._auth_for_credentials(credentials) + tag_names = [tag_names] if isinstance(tag_names, str) else tag_names + + # Ping! + self.ping(session) + + # PUT /v1/repositories/{namespace}/{repository}/ + result = self.conduct(session, 'PUT', + '/v1/repositories/%s/' % self.repo_name(namespace, repo_name), + expected_status=(201, expected_failure, V1ProtocolSteps.PUT_IMAGES), + json_data={}, + auth=auth) + + if expected_failure is not None: + return + + headers = {} + headers['Authorization'] = 'token ' + result.headers['www-authenticate'] + + for image in images: + # PUT /v1/images/{imageID}/json + image_json_data = {'id': image.id} + if image.size is not None: + image_json_data['Size'] = image.size + + if image.parent_id is not None: + image_json_data['parent'] = image.parent_id + + if image.config is not None: + image_json_data['config'] = image.config + + self.conduct(session, 'PUT', '/v1/images/%s/json' % image.id, + json_data=image_json_data, headers=headers) + + # PUT /v1/images/{imageID}/layer + self.conduct(session, 'PUT', '/v1/images/%s/layer' % image.id, + data=StringIO(image.bytes), headers=headers) + + # PUT /v1/images/{imageID}/checksum + checksum = compute_simple(StringIO(image.bytes), json.dumps(image_json_data)) + checksum_headers = {'X-Docker-Checksum-Payload': checksum} + checksum_headers.update(headers) + + self.conduct(session, 'PUT', '/v1/images/%s/checksum' % image.id, + headers=checksum_headers) + + # PUT /v1/repositories/{namespace}/{repository}/tags/latest + for tag_name in tag_names: + self.conduct(session, 'PUT', + '/v1/repositories/%s/tags/%s' % (self.repo_name(namespace, repo_name), tag_name), + data='"%s"' % images[-1].id, + headers=headers, + expected_status=(200, expected_failure, V1ProtocolSteps.PUT_TAG)) + + # PUT /v1/repositories/{namespace}/{repository}/images + self.conduct(session, 'PUT', + '/v1/repositories/%s/images' % self.repo_name(namespace, repo_name), + expected_status=204, headers=headers) + + return PushResult(checksums=None, manifests=None, headers=headers) diff --git a/test/registry/protocol_v2.py b/test/registry/protocol_v2.py new file mode 100644 index 000000000..9c4be8dfb --- /dev/null +++ b/test/registry/protocol_v2.py @@ -0,0 +1,371 @@ +import hashlib +import json + +from enum import Enum, unique + +from image.docker.schema1 import DockerSchema1ManifestBuilder, DockerSchema1Manifest +from test.registry.protocols import (RegistryProtocol, Failures, ProtocolOptions, PushResult, + PullResult) + + +@unique +class V2ProtocolSteps(Enum): + """ Defines the various steps of the protocol, for matching failures. """ + AUTH = 'auth' + BLOB_HEAD_CHECK = 'blob-head-check' + GET_MANIFEST = 'get-manifest' + PUT_MANIFEST = 'put-manifest' + + +class V2Protocol(RegistryProtocol): + FAILURE_CODES = { + V2ProtocolSteps.AUTH: { + Failures.UNAUTHENTICATED: 401, + Failures.INVALID_REGISTRY: 400, + Failures.APP_REPOSITORY: 405, + Failures.ANONYMOUS_NOT_ALLOWED: 401, + Failures.INVALID_REPOSITORY: 400, + }, + V2ProtocolSteps.GET_MANIFEST: { + Failures.UNKNOWN_TAG: 404, + Failures.UNAUTHORIZED: 403, + Failures.DISALLOWED_LIBRARY_NAMESPACE: 400, + }, + V2ProtocolSteps.PUT_MANIFEST: { + Failures.DISALLOWED_LIBRARY_NAMESPACE: 400, + Failures.MISSING_TAG: 404, + Failures.INVALID_TAG: 400, + Failures.INVALID_IMAGES: 400, + Failures.INVALID_BLOB: 400, + Failures.UNSUPPORTED_CONTENT_TYPE: 415, + }, + } + + def __init__(self, jwk): + self.jwk = jwk + + def ping(self, session): + result = session.get('/v2/') + assert result.status_code == 401 + assert result.headers['Docker-Distribution-API-Version'] == 'registry/2.0' + + def login(self, session, username, password, scopes, expect_success): + scopes = scopes if isinstance(scopes, list) else [scopes] + params = { + 'account': username, + 'service': 'localhost:5000', + 'scope': scopes, + } + + auth = (username, password) + if not username or not password: + auth = None + + response = session.get('/v2/auth', params=params, auth=auth) + if expect_success: + assert response.status_code / 100 == 2 + else: + assert response.status_code / 100 == 4 + + return response + + def auth(self, session, credentials, namespace, repo_name, scopes=None, + expected_failure=None): + """ + Performs the V2 Auth flow, returning the token (if any) and the response. + + Spec: https://docs.docker.com/registry/spec/auth/token/ + """ + + scopes = scopes or [] + auth = None + username = None + + if credentials is not None: + username, _ = credentials + auth = credentials + + params = { + 'account': username, + 'service': 'localhost:5000', + } + + if scopes: + params['scope'] = 'repository:%s:%s' % (self.repo_name(namespace, repo_name), + ','.join(scopes)) + + response = self.conduct(session, 'GET', '/v2/auth', params=params, auth=auth, + expected_status=(200, expected_failure, V2ProtocolSteps.AUTH)) + + if expected_failure is None: + assert response.json().get('token') is not None + return response.json().get('token'), response + + return None, response + + def push(self, session, namespace, repo_name, tag_names, images, credentials=None, + expected_failure=None, options=None): + options = options or ProtocolOptions() + scopes = options.scopes or ['push', 'pull'] + tag_names = [tag_names] if isinstance(tag_names, str) else tag_names + + # Ping! + self.ping(session) + + # Perform auth and retrieve a token. + token, _ = self.auth(session, credentials, namespace, repo_name, scopes=scopes, + expected_failure=expected_failure) + if token is None: + return + + headers = { + 'Authorization': 'Bearer ' + token, + } + + # Build fake manifests. + manifests = {} + for tag_name in tag_names: + builder = DockerSchema1ManifestBuilder(namespace, repo_name, tag_name) + + for image in reversed(images): + checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest() + + # If invalid blob references were requested, just make it up. + if options.manifest_invalid_blob_references: + checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest() + + layer_dict = {'id': image.id, 'parent': image.parent_id} + if image.config is not None: + layer_dict['config'] = image.config + + if image.size is not None: + layer_dict['Size'] = image.size + + builder.add_layer(checksum, json.dumps(layer_dict)) + + # Build the manifest. + manifests[tag_name] = builder.build(self.jwk) + + # Push the layer data. + checksums = {} + for image in reversed(images): + checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest() + checksums[image.id] = checksum + + if not options.skip_head_checks: + # Layer data should not yet exist. + self.conduct(session, 'HEAD', + '/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name), checksum), + expected_status=(404, expected_failure, V2ProtocolSteps.BLOB_HEAD_CHECK), + headers=headers) + + # Start a new upload of the layer data. + response = self.conduct(session, 'POST', + '/v2/%s/blobs/uploads/' % self.repo_name(namespace, repo_name), + expected_status=202, + headers=headers) + + upload_uuid = response.headers['Docker-Upload-UUID'] + new_upload_location = response.headers['Location'] + assert new_upload_location.startswith('http://localhost:5000') + + # We need to make this relative just for the tests because the live server test + # case modifies the port. + location = response.headers['Location'][len('http://localhost:5000'):] + + # PATCH the image data into the layer. + if options.chunks_for_upload is None: + self.conduct(session, 'PATCH', location, data=image.bytes, expected_status=204, + headers=headers) + else: + # If chunked upload is requested, upload the data as a series of chunks, checking + # status at every point. + for chunk_data in options.chunks_for_upload: + if len(chunk_data) == 3: + (start_byte, end_byte, expected_code) = chunk_data + else: + (start_byte, end_byte) = chunk_data + expected_code = 204 + + patch_headers = {'Range': 'bytes=%s-%s' % (start_byte, end_byte)} + patch_headers.update(headers) + + contents_chunk = image.bytes[start_byte:end_byte] + self.conduct(session, 'PATCH', location, data=contents_chunk, + expected_status=expected_code, + headers=patch_headers) + if expected_code != 204: + return + + # Retrieve the upload status at each point, and ensure it is valid. + status_url = '/v2/%s/blobs/uploads/%s' % (self.repo_name(namespace, repo_name), + upload_uuid) + response = self.conduct(session, 'GET', status_url, expected_status=204, headers=headers) + assert response.headers['Docker-Upload-UUID'] == upload_uuid + assert response.headers['Range'] == "bytes=0-%s" % end_byte + + if options.cancel_blob_upload: + self.conduct(session, 'DELETE', location, params=dict(digest=checksum), expected_status=204, + headers=headers) + + # Ensure the upload was canceled. + status_url = '/v2/%s/blobs/uploads/%s' % (self.repo_name(namespace, repo_name), upload_uuid) + self.conduct(session, 'GET', status_url, expected_status=404, headers=headers) + return + + # Finish the layer upload with a PUT. + response = self.conduct(session, 'PUT', location, params=dict(digest=checksum), + expected_status=201, headers=headers) + assert response.headers['Docker-Content-Digest'] == checksum + + # Ensure the layer exists now. + response = self.conduct(session, 'HEAD', + '/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name), checksum), + expected_status=200, headers=headers) + + assert response.headers['Docker-Content-Digest'] == checksum + assert response.headers['Content-Length'] == str(len(image.bytes)) + + # And retrieve the layer data. + result = self.conduct(session, 'GET', + '/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name), checksum), + headers=headers, expected_status=200) + assert result.content == image.bytes + + # Write a manifest for each tag. + for tag_name in tag_names: + manifest = manifests[tag_name] + + # Write the manifest. If we expect it to be invalid, we expect a 404 code. Otherwise, we + # expect a 202 response for success. + put_code = 404 if options.manifest_invalid_blob_references else 202 + manifest_headers = {'Content-Type': 'application/json'} + manifest_headers.update(headers) + + if options.manifest_content_type is not None: + manifest_headers['Content-Type'] = options.manifest_content_type + + self.conduct(session, 'PUT', + '/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name), tag_name), + data=manifest.bytes, + expected_status=(put_code, expected_failure, V2ProtocolSteps.PUT_MANIFEST), + headers=manifest_headers) + + return PushResult(checksums=checksums, manifests=manifests, headers=headers) + + + def delete(self, session, namespace, repo_name, tag_names, credentials=None, + expected_failure=None, options=None): + options = options or ProtocolOptions() + scopes = options.scopes or ['*'] + tag_names = [tag_names] if isinstance(tag_names, str) else tag_names + + # Ping! + self.ping(session) + + # Perform auth and retrieve a token. + token, _ = self.auth(session, credentials, namespace, repo_name, scopes=scopes, + expected_failure=expected_failure) + if token is None: + return None + + headers = { + 'Authorization': 'Bearer ' + token, + } + + for tag_name in tag_names: + self.conduct(session, 'DELETE', + '/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name), tag_name), + headers=headers, + expected_status=202) + + + def pull(self, session, namespace, repo_name, tag_names, images, credentials=None, + expected_failure=None, options=None): + options = options or ProtocolOptions() + scopes = options.scopes or ['pull'] + tag_names = [tag_names] if isinstance(tag_names, str) else tag_names + + # Ping! + self.ping(session) + + # Perform auth and retrieve a token. + token, _ = self.auth(session, credentials, namespace, repo_name, scopes=scopes, + expected_failure=expected_failure) + if token is None: + return None + + headers = { + 'Authorization': 'Bearer ' + token, + } + + manifests = {} + image_ids = {} + for tag_name in tag_names: + # Retrieve the manifest for the tag or digest. + response = self.conduct(session, 'GET', + '/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name), + tag_name), + expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST), + headers=headers) + if expected_failure is not None: + return None + + # Ensure the manifest returned by us is valid. + manifest = DockerSchema1Manifest(response.text) + manifests[tag_name] = manifest + image_ids[tag_name] = manifest.leaf_layer.v1_metadata.image_id + + # Verify the layers. + for index, layer in enumerate(manifest.layers): + result = self.conduct(session, 'GET', + '/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name), + layer.digest), + expected_status=200, + headers=headers) + assert result.content == images[index].bytes + + return PullResult(manifests=manifests, image_ids=image_ids) + + + def catalog(self, session, page_size=2, credentials=None, options=None, expected_failure=None, + namespace=None, repo_name=None): + options = options or ProtocolOptions() + scopes = options.scopes or [] + + # Ping! + self.ping(session) + + # Perform auth and retrieve a token. + headers = {} + if credentials is not None: + token, _ = self.auth(session, credentials, namespace, repo_name, scopes=scopes, + expected_failure=expected_failure) + if token is None: + return None + + headers = { + 'Authorization': 'Bearer ' + token, + } + + results = [] + url = '/v2/_catalog' + params = {} + if page_size is not None: + params['n'] = page_size + + while True: + response = self.conduct(session, 'GET', url, headers=headers, params=params) + data = response.json() + + assert len(data['repositories']) <= page_size + results.extend(data['repositories']) + + if not response.headers.get('Link'): + return results + + link_url = response.headers['Link'] + v2_index = link_url.find('/v2/') + url = link_url[v2_index:] + + return results diff --git a/test/registry/protocols.py b/test/registry/protocols.py new file mode 100644 index 000000000..3b4edba33 --- /dev/null +++ b/test/registry/protocols.py @@ -0,0 +1,111 @@ +import json +import tarfile + +from abc import ABCMeta, abstractmethod +from collections import namedtuple +from cStringIO import StringIO +from enum import Enum, unique +from six import add_metaclass + +Image = namedtuple('Image', ['id', 'parent_id', 'bytes', 'size', 'config']) +Image.__new__.__defaults__ = (None, None) + +PushResult = namedtuple('PushResult', ['checksums', 'manifests', 'headers']) +PullResult = namedtuple('PullResult', ['manifests', 'image_ids']) + + +def layer_bytes_for_contents(contents, mode='|gz'): + layer_data = StringIO() + + def add_file(name, contents): + tar_file_info = tarfile.TarInfo(name=name) + tar_file_info.type = tarfile.REGTYPE + tar_file_info.size = len(contents) + tar_file_info.mtime = 1 + + tar_file = tarfile.open(fileobj=layer_data, mode='w' + mode) + tar_file.addfile(tar_file_info, StringIO(contents)) + tar_file.close() + + add_file('contents', contents) + + layer_bytes = layer_data.getvalue() + layer_data.close() + return layer_bytes + + +@unique +class Failures(Enum): + """ Defines the various forms of expected failure. """ + UNAUTHENTICATED = 'unauthenticated' + UNAUTHORIZED = 'unauthorized' + INVALID_REGISTRY = 'invalid-registry' + INVALID_REPOSITORY = 'invalid-repository' + APP_REPOSITORY = 'app-repository' + UNKNOWN_TAG = 'unknown-tag' + ANONYMOUS_NOT_ALLOWED = 'anonymous-not-allowed' + DISALLOWED_LIBRARY_NAMESPACE = 'disallowed-library-namespace' + MISSING_TAG = 'missing-tag' + INVALID_TAG = 'invalid-tag' + INVALID_IMAGES = 'invalid-images' + UNSUPPORTED_CONTENT_TYPE = 'unsupported-content-type' + INVALID_BLOB = 'invalid-blob' + + +class ProtocolOptions(object): + def __init__(self): + self.munge_shas = False + self.scopes = None + self.cancel_blob_upload = False + self.manifest_invalid_blob_references = False + self.chunks_for_upload = None + self.skip_head_checks = False + self.manifest_content_type = None + + +@add_metaclass(ABCMeta) +class RegistryProtocol(object): + """ Interface for protocols. """ + FAILURE_CODES = {} + + @abstractmethod + def login(self, session, username, password, scopes, expect_success): + """ Performs the login flow with the given credentials, over the given scopes. """ + + @abstractmethod + def pull(self, session, namespace, repo_name, tag_names, images, credentials=None, + expected_failure=None, options=None): + """ Pulls the given tag via the given session, using the given credentials, and + ensures the given images match. + """ + + @abstractmethod + def push(self, session, namespace, repo_name, tag_names, images, credentials=None, + expected_failure=None, options=None): + """ Pushes the specified images as the given tag via the given session, using + the given credentials. + """ + + def repo_name(self, namespace, repo_name): + if namespace: + return '%s/%s' % (namespace, repo_name) + + return repo_name + + def conduct(self, session, method, url, expected_status=200, params=None, data=None, + json_data=None, headers=None, auth=None): + if json_data is not None: + data = json.dumps(json_data) + headers = headers or {} + headers['Content-Type'] = 'application/json' + + if isinstance(expected_status, tuple): + expected_status, expected_failure, protocol_step = expected_status + if expected_failure is not None: + failures = self.__class__.FAILURE_CODES.get(protocol_step, {}) + expected_status = failures.get(expected_failure, expected_status) + + result = session.request(method, url, params=params, data=data, headers=headers, auth=auth) + msg = "Expected response %s, got %s" % (expected_status, result.status_code) + assert result.status_code == expected_status, msg + return result diff --git a/test/registry/registry_tests.py b/test/registry/registry_tests.py new file mode 100644 index 000000000..030528075 --- /dev/null +++ b/test/registry/registry_tests.py @@ -0,0 +1,892 @@ +# pylint: disable=W0401, W0621, W0613, W0614, R0913 +import hashlib +import tarfile + +from cStringIO import StringIO + +import binascii +import bencode +import resumablehashlib + +from test.fixtures import * +from test.registry.liveserverfixture import * +from test.registry.fixtures import * +from test.registry.protocol_fixtures import * + +from test.registry.protocols import Failures, Image, layer_bytes_for_contents, ProtocolOptions + +from app import instance_keys +from util.security.registry_jwt import decode_bearer_header +from util.timedeltastring import convert_to_timedelta + + +def test_basic_push_pull(pusher, puller, basic_images, liveserver_session, app_reloader): + """ Test: Basic push and pull of an image to a new repository. """ + credentials = ('devtable', 'password') + + # Push a new repository. + pusher.push(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images, + credentials=credentials) + + # Pull the repository to verify. + puller.pull(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images, + credentials=credentials) + + +def test_basic_push_pull_by_manifest(manifest_protocol, basic_images, liveserver_session, + app_reloader): + """ Test: Basic push and pull-by-manifest of an image to a new repository. """ + credentials = ('devtable', 'password') + + # Push a new repository. + result = manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images, + credentials=credentials) + + # Pull the repository by digests to verify. + digests = [str(manifest.digest) for manifest in result.manifests.values()] + manifest_protocol.pull(liveserver_session, 'devtable', 'newrepo', digests, basic_images, + credentials=credentials) + + +def test_push_invalid_credentials(pusher, basic_images, liveserver_session, app_reloader): + """ Test: Ensure we get auth errors when trying to push with invalid credentials. """ + invalid_credentials = ('devtable', 'notcorrectpassword') + + pusher.push(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images, + credentials=invalid_credentials, expected_failure=Failures.UNAUTHENTICATED) + + +def test_pull_invalid_credentials(puller, basic_images, liveserver_session, app_reloader): + """ Test: Ensure we get auth errors when trying to pull with invalid credentials. """ + invalid_credentials = ('devtable', 'notcorrectpassword') + + puller.pull(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images, + credentials=invalid_credentials, expected_failure=Failures.UNAUTHENTICATED) + + +def test_push_pull_formerly_bad_repo_name(pusher, puller, basic_images, liveserver_session, + app_reloader): + """ Test: Basic push and pull of an image to a new repository with a name that formerly + failed. """ + credentials = ('devtable', 'password') + + # Push a new repository. + pusher.push(liveserver_session, 'devtable', 'foo.bar', 'latest', basic_images, + credentials=credentials) + + # Pull the repository to verify. + puller.pull(liveserver_session, 'devtable', 'foo.bar', 'latest', basic_images, + credentials=credentials) + + +def test_application_repo(pusher, puller, basic_images, liveserver_session, app_reloader, + registry_server_executor, liveserver): + """ Test: Attempting to push or pull from an *application* repository raises a 405. """ + credentials = ('devtable', 'password') + registry_server_executor.on(liveserver).create_app_repository('devtable', 'someapprepo') + + # Attempt to push to the repository. + pusher.push(liveserver_session, 'devtable', 'someapprepo', 'latest', basic_images, + credentials=credentials, expected_failure=Failures.APP_REPOSITORY) + + # Attempt to pull from the repository. + puller.pull(liveserver_session, 'devtable', 'someapprepo', 'latest', basic_images, + credentials=credentials, expected_failure=Failures.APP_REPOSITORY) + + +def test_middle_layer_different_sha(manifest_protocol, v1_protocol, liveserver_session, + app_reloader): + """ Test: Pushing of a 3-layer image with the *same* V1 ID's, but the middle layer having + different bytes, must result in new IDs being generated for the leaf layer, as + they point to different "images". + """ + credentials = ('devtable', 'password') + first_images = [ + Image(id='baseimage', parent_id=None, size=None, bytes=layer_bytes_for_contents('base')), + Image(id='middleimage', parent_id='baseimage', size=None, + bytes=layer_bytes_for_contents('middle')), + Image(id='leafimage', parent_id='middleimage', size=None, + bytes=layer_bytes_for_contents('leaf')), + ] + + # First push and pull the images, to ensure we have the basics setup and working. + manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', first_images, + credentials=credentials) + first_pull_result = manifest_protocol.pull(liveserver_session, 'devtable', 'newrepo', 'latest', + first_images, credentials=credentials) + first_manifest = first_pull_result.manifests['latest'] + assert set([image.id for image in first_images]) == set(first_manifest.image_ids) + assert first_pull_result.image_ids['latest'] == 'leafimage' + + # Next, create an image list with the middle image's *bytes* changed. + second_images = list(first_images) + second_images[1] = Image(id='middleimage', parent_id='baseimage', size=None, + bytes=layer_bytes_for_contents('different middle bytes')) + + # Push and pull the image, ensuring that the produced ID for the middle and leaf layers + # are synthesized. + options = ProtocolOptions() + options.munge_shas = True + options.skip_head_checks = True + + manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', second_images, + credentials=credentials, options=options) + second_pull_result = v1_protocol.pull(liveserver_session, 'devtable', 'newrepo', 'latest', + second_images, credentials=credentials, options=options) + + assert second_pull_result.image_ids['latest'] != 'leafimage' + + +def add_robot(api_caller, _): + api_caller.conduct_auth('devtable', 'password') + resp = api_caller.get('/api/v1/organization/buynlarge/robots/ownerbot') + return ('buynlarge+ownerbot', resp.json()['token']) + + +def add_token(_, executor): + return ('$token', executor.add_token().text) + + +@pytest.mark.parametrize('credentials, namespace, expected_performer', [ + (('devtable', 'password'), 'devtable', 'devtable'), + (add_robot, 'buynlarge', 'buynlarge+ownerbot'), + (('$oauthtoken', 'test'), 'devtable', 'devtable'), + (('$app', 'test'), 'devtable', 'devtable'), + (add_token, 'devtable', None), +]) +def test_push_pull_logging(credentials, namespace, expected_performer, pusher, puller, basic_images, + liveserver_session, liveserver, api_caller, app_reloader, + registry_server_executor): + """ Test: Basic push and pull, ensuring that logs are added for each operation. """ + + # Create the repository before the test push. + start_images = [Image(id='startimage', parent_id=None, size=None, + bytes=layer_bytes_for_contents('start image'))] + pusher.push(liveserver_session, namespace, 'newrepo', 'latest', start_images, + credentials=('devtable', 'password')) + + # Retrieve the credentials to use. This must be after the repo is created, because + # some credentials creation code adds the new entity to the repository. + if not isinstance(credentials, tuple): + credentials = credentials(api_caller, registry_server_executor.on(liveserver)) + + # Push to the repository with the specified credentials. + pusher.push(liveserver_session, namespace, 'newrepo', 'latest', basic_images, + credentials=credentials) + + # Check the logs for the push. + api_caller.conduct_auth('devtable', 'password') + + result = api_caller.get('/api/v1/repository/%s/newrepo/logs' % namespace) + logs = result.json()['logs'] + assert len(logs) == 2 + assert logs[0]['kind'] == 'push_repo' + assert logs[0]['metadata']['namespace'] == namespace + assert logs[0]['metadata']['repo'] == 'newrepo' + + if expected_performer is not None: + assert logs[0]['performer']['name'] == expected_performer + + # Pull the repository to verify. + puller.pull(liveserver_session, namespace, 'newrepo', 'latest', basic_images, + credentials=credentials) + + # Check the logs for the pull. + result = api_caller.get('/api/v1/repository/%s/newrepo/logs' % namespace) + logs = result.json()['logs'] + assert len(logs) == 3 + assert logs[0]['kind'] == 'pull_repo' + assert logs[0]['metadata']['namespace'] == namespace + assert logs[0]['metadata']['repo'] == 'newrepo' + + if expected_performer is not None: + assert logs[0]['performer']['name'] == expected_performer + + +def test_pull_publicrepo_anonymous(pusher, puller, basic_images, liveserver_session, + app_reloader, api_caller, liveserver): + """ Test: Pull a public repository anonymously. """ + # Add a new repository under the public user, so we have a repository to pull. + pusher.push(liveserver_session, 'public', 'newrepo', 'latest', basic_images, + credentials=('public', 'password')) + + # First try to pull the (currently private) repo anonymously, which should fail (since it is + # private) + puller.pull(liveserver_session, 'public', 'newrepo', 'latest', basic_images, + expected_failure=Failures.UNAUTHORIZED) + + # Using a non-public user should also fail. + puller.pull(liveserver_session, 'public', 'newrepo', 'latest', basic_images, + credentials=('devtable', 'password'), + expected_failure=Failures.UNAUTHORIZED) + + # Make the repository public. + api_caller.conduct_auth('public', 'password') + api_caller.change_repo_visibility('public', 'newrepo', 'public') + + # Pull the repository anonymously, which should succeed because the repository is public. + puller.pull(liveserver_session, 'public', 'newrepo', 'latest', basic_images) + + +def test_pull_publicrepo_no_anonymous_access(pusher, puller, basic_images, liveserver_session, + app_reloader, api_caller, liveserver, + registry_server_executor): + """ Test: Attempts to pull a public repository anonymously, with the feature flag disabled. """ + # Add a new repository under the public user, so we have a repository to pull. + pusher.push(liveserver_session, 'public', 'newrepo', 'latest', basic_images, + credentials=('public', 'password')) + + # First try to pull the (currently private) repo anonymously, which should fail (since it is + # private) + puller.pull(liveserver_session, 'public', 'newrepo', 'latest', basic_images, + expected_failure=Failures.UNAUTHORIZED) + + # Using a non-public user should also fail. + puller.pull(liveserver_session, 'public', 'newrepo', 'latest', basic_images, + credentials=('devtable', 'password'), + expected_failure=Failures.UNAUTHORIZED) + + # Make the repository public. + api_caller.conduct_auth('public', 'password') + api_caller.change_repo_visibility('public', 'newrepo', 'public') + + with FeatureFlagValue('ANONYMOUS_ACCESS', False, registry_server_executor.on(liveserver)): + # Attempt again to pull the (now public) repo anonymously, which should fail since + # the feature flag for anonymous access is turned off. + puller.pull(liveserver_session, 'public', 'newrepo', 'latest', basic_images, + expected_failure=Failures.ANONYMOUS_NOT_ALLOWED) + + # Using a non-public user should now succeed. + puller.pull(liveserver_session, 'public', 'newrepo', 'latest', basic_images, + credentials=('devtable', 'password')) + + +def test_basic_organization_flow(pusher, puller, basic_images, liveserver_session, app_reloader): + """ Test: Basic push and pull of an image to a new repository by members of an organization. """ + # Add a new repository under the organization via the creator user. + pusher.push(liveserver_session, 'buynlarge', 'newrepo', 'latest', basic_images, + credentials=('creator', 'password')) + + # Ensure that the creator can pull it. + puller.pull(liveserver_session, 'buynlarge', 'newrepo', 'latest', basic_images, + credentials=('creator', 'password')) + + # Ensure that the admin can pull it. + puller.pull(liveserver_session, 'buynlarge', 'newrepo', 'latest', basic_images, + credentials=('devtable', 'password')) + + # Ensure that the reader *cannot* pull it. + puller.pull(liveserver_session, 'buynlarge', 'newrepo', 'latest', basic_images, + credentials=('reader', 'password'), + expected_failure=Failures.UNAUTHORIZED) + + +def test_library_support(pusher, puller, basic_images, liveserver_session, app_reloader): + """ Test: Pushing and pulling from the implicit library namespace. """ + credentials = ('devtable', 'password') + + # Push a new repository. + pusher.push(liveserver_session, '', 'newrepo', 'latest', basic_images, + credentials=credentials) + + # Pull the repository to verify. + puller.pull(liveserver_session, '', 'newrepo', 'latest', basic_images, + credentials=credentials) + + # Pull the repository from the library namespace to verify. + puller.pull(liveserver_session, 'library', 'newrepo', 'latest', basic_images, + credentials=credentials) + + +def test_library_namespace_with_support_disabled(pusher, puller, basic_images, liveserver_session, + app_reloader, liveserver, + registry_server_executor): + """ Test: Pushing and pulling from the explicit library namespace, even when the + implicit one is disabled. + """ + credentials = ('devtable', 'password') + + with FeatureFlagValue('LIBRARY_SUPPORT', False, registry_server_executor.on(liveserver)): + # Push a new repository. + pusher.push(liveserver_session, 'library', 'newrepo', 'latest', basic_images, + credentials=credentials) + + # Pull the repository from the library namespace to verify. + puller.pull(liveserver_session, 'library', 'newrepo', 'latest', basic_images, + credentials=credentials) + + +def test_push_library_with_support_disabled(pusher, basic_images, liveserver_session, + app_reloader, liveserver, + registry_server_executor): + """ Test: Pushing to the implicit library namespace, when disabled, + should fail. + """ + credentials = ('devtable', 'password') + + with FeatureFlagValue('LIBRARY_SUPPORT', False, registry_server_executor.on(liveserver)): + # Attempt to push a new repository. + pusher.push(liveserver_session, '', 'newrepo', 'latest', basic_images, + credentials=credentials, + expected_failure=Failures.DISALLOWED_LIBRARY_NAMESPACE) + + +def test_pull_library_with_support_disabled(puller, basic_images, liveserver_session, + app_reloader, liveserver, + registry_server_executor): + """ Test: Pushing to the implicit library namespace, when disabled, + should fail. + """ + credentials = ('devtable', 'password') + + with FeatureFlagValue('LIBRARY_SUPPORT', False, registry_server_executor.on(liveserver)): + # Attempt to pull the repository from the library namespace. + puller.pull(liveserver_session, '', 'newrepo', 'latest', basic_images, + credentials=credentials, + expected_failure=Failures.DISALLOWED_LIBRARY_NAMESPACE) + + +def test_image_replication(pusher, basic_images, liveserver_session, app_reloader, liveserver, + registry_server_executor): + """ Test: Ensure that entries are created for replication of the images pushed. """ + credentials = ('devtable', 'password') + + with FeatureFlagValue('STORAGE_REPLICATION', True, registry_server_executor.on(liveserver)): + pusher.push(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images, + credentials=credentials) + + # Ensure that entries were created for each image. + for image in basic_images: + r = registry_server_executor.on(liveserver).get_storage_replication_entry(image.id) + assert r.text == 'OK' + + +def test_push_reponame_with_slashes(pusher, basic_images, liveserver_session, app_reloader): + """ Test: Attempt to add a repository name with slashes. This should fail as we do not + support it. + """ + credentials = ('devtable', 'password') + + pusher.push(liveserver_session, 'devtable', 'some/slash', 'latest', basic_images, + credentials=credentials, + expected_failure=Failures.INVALID_REPOSITORY) + + +@pytest.mark.parametrize('tag_name, expected_failure', [ + ('l', None), + ('1', None), + ('x' * 128, None), + + ('', Failures.MISSING_TAG), + ('x' * 129, Failures.INVALID_TAG), + ('.fail', Failures.INVALID_TAG), + ('-fail', Failures.INVALID_TAG), +]) +def test_tag_validaton(tag_name, expected_failure, pusher, basic_images, liveserver_session, + app_reloader): + """ Test: Various forms of tags and whether they succeed or fail as expected. """ + credentials = ('devtable', 'password') + + pusher.push(liveserver_session, 'devtable', 'newrepo', tag_name, basic_images, + credentials=credentials, + expected_failure=expected_failure) + + +def test_invalid_parent(pusher, liveserver_session, app_reloader): + """ Test: Attempt to push an image with an invalid/missing parent. """ + images = [ + Image(id='childimage', parent_id='parentimage', size=None, + bytes=layer_bytes_for_contents('child')), + ] + + credentials = ('devtable', 'password') + + pusher.push(liveserver_session, 'devtable', 'newrepo', 'latest', images, + credentials=credentials, + expected_failure=Failures.INVALID_IMAGES) + + +def test_wrong_image_order(pusher, liveserver_session, app_reloader): + """ Test: Attempt to push an image with its layers in the wrong order. """ + images = [ + Image(id='childimage', parent_id='parentimage', size=None, + bytes=layer_bytes_for_contents('child')), + Image(id='parentimage', parent_id=None, size=None, + bytes=layer_bytes_for_contents('parent')), + ] + + credentials = ('devtable', 'password') + + pusher.push(liveserver_session, 'devtable', 'newrepo', 'latest', images, + credentials=credentials, + expected_failure=Failures.INVALID_IMAGES) + + +@pytest.mark.parametrize('labels', [ + # Basic labels. + [('foo', 'bar', 'text/plain'), ('baz', 'meh', 'text/plain')], + + # Theoretically invalid, but allowed when pushed via registry protocol. + [('theoretically-invalid--label', 'foo', 'text/plain')], + + # JSON label. + [('somejson', '{"some": "json"}', 'application/json'), ('plain', '', 'text/plain')], + + # JSON-esque (but not valid JSON) labels. + [('foo', '[hello world]', 'text/plain'), ('bar', '{wassup?!}', 'text/plain')], +]) +def test_labels(labels, manifest_protocol, liveserver_session, api_caller, app_reloader): + """ Test: Image pushed with labels has those labels found in the database after the + push succeeds. + """ + images = [ + Image(id='theimage', parent_id=None, bytes=layer_bytes_for_contents('image'), + config={'Labels': {key: value for (key, value, _) in labels}}), + ] + + credentials = ('devtable', 'password') + result = manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', images, + credentials=credentials) + + digest = result.manifests['latest'].digest + api_caller.conduct_auth('devtable', 'password') + + data = api_caller.get('/api/v1/repository/devtable/newrepo/manifest/%s/labels' % digest).json() + labels_found = data['labels'] + assert len(labels_found) == len(labels) + + labels_found_map = {l['key']: l for l in labels_found} + assert set(images[0].config['Labels'].keys()) == set(labels_found_map.keys()) + + for key, _, media_type in labels: + assert labels_found_map[key]['source_type'] == 'manifest' + assert labels_found_map[key]['media_type'] == media_type + + +@pytest.mark.parametrize('label_value, expected_expiration', [ + ('1d', True), + ('1h', True), + ('2w', True), + + ('1g', False), + ('something', False), +]) +def test_expiration_label(label_value, expected_expiration, manifest_protocol, liveserver_session, + api_caller, app_reloader): + """ Test: Tag pushed with a valid `quay.expires-after` will have its expiration set to its + start time plus the duration specified. If the duration is invalid, no expiration will + be set. + """ + images = [ + Image(id='theimage', parent_id=None, bytes=layer_bytes_for_contents('image'), + config={'Labels': {'quay.expires-after': label_value}}), + ] + + credentials = ('devtable', 'password') + manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', images, + credentials=credentials) + + api_caller.conduct_auth('devtable', 'password') + + tag_data = api_caller.get('/api/v1/repository/devtable/newrepo/tag').json()['tags'][0] + if expected_expiration: + diff = convert_to_timedelta(label_value).total_seconds() + assert tag_data['end_ts'] == tag_data['start_ts'] + diff + else: + assert tag_data.get('end_ts') is None + + +@pytest.mark.parametrize('content_type', [ + 'application/vnd.oci.image.manifest.v1+json', + 'application/vnd.docker.distribution.manifest.v2+json', + 'application/vnd.foo.bar', +]) +def test_unsupported_manifest_content_type(content_type, manifest_protocol, basic_images, + liveserver_session, app_reloader): + """ Test: Attempt to push a manifest with an unsupported media type. """ + credentials = ('devtable', 'password') + + options = ProtocolOptions() + options.manifest_content_type = content_type + + # Attempt to push a new repository. + manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images, + credentials=credentials, + options=options, + expected_failure=Failures.UNSUPPORTED_CONTENT_TYPE) + + +def test_invalid_blob_reference(manifest_protocol, basic_images, liveserver_session, app_reloader): + """ Test: Attempt to push a manifest with an invalid blob reference. """ + credentials = ('devtable', 'password') + + options = ProtocolOptions() + options.manifest_invalid_blob_references = True + + # Attempt to push a new repository. + manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images, + credentials=credentials, + options=options, + expected_failure=Failures.INVALID_BLOB) + + +def test_delete_tag(manifest_protocol, puller, basic_images, liveserver_session, + app_reloader): + """ Test: Push a repository, delete a tag, and attempt to pull. """ + credentials = ('devtable', 'password') + + # Push the tags. + result = manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', ['one', 'two'], + basic_images, credentials=credentials) + + # Delete tag `one` by digest. + manifest_protocol.delete(liveserver_session, 'devtable', 'newrepo', + result.manifests['one'].digest, + credentials=credentials) + + # Attempt to pull tag `one` and ensure it doesn't work. + puller.pull(liveserver_session, 'devtable', 'newrepo', 'one', basic_images, + credentials=credentials, + expected_failure=Failures.UNKNOWN_TAG) + + # Pull tag `two` to verify it works. + puller.pull(liveserver_session, 'devtable', 'newrepo', 'two', basic_images, + credentials=credentials) + + +def test_cancel_upload(manifest_protocol, basic_images, liveserver_session, app_reloader): + """ Test: Cancelation of blob uploads. """ + credentials = ('devtable', 'password') + + options = ProtocolOptions() + options.cancel_blob_upload = True + + manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', + basic_images, credentials=credentials) + + +def test_blob_caching(manifest_protocol, basic_images, liveserver_session, app_reloader, + liveserver, registry_server_executor): + """ Test: Pulling of blobs after initially pulled will result in the blobs being cached. """ + credentials = ('devtable', 'password') + + # Push a tag. + result = manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', + basic_images, credentials=credentials) + + + # Conduct the initial pull to prime the cache. + manifest_protocol.pull(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images, + credentials=credentials) + + # Disconnect the server from the database. + registry_server_executor.on(liveserver).break_database() + + # Pull each blob, which should succeed due to caching. If caching is broken, this will + # fail when it attempts to hit the database. + for layer in result.manifests['latest'].layers: + blob_id = str(layer.digest) + r = liveserver_session.get('/v2/devtable/newrepo/blobs/%s' % blob_id, headers=result.headers) + assert r.status_code == 200 + + +@pytest.mark.parametrize('chunks', [ + # Two chunks. + [(0, 100), (100, None)], + + # Multiple chunks. + [(0, 10), (10, 20), (20, None)], + [(0, 10), (10, 20), (20, 30), (30, 40), (40, 50), (50, None)], + + # Overlapping chunks. + [(0, 1024), (10, None)], +]) +def test_chunked_blob_uploading(chunks, random_layer_data, manifest_protocol, puller, + liveserver_session, app_reloader): + """ Test: Uploading of blobs as chunks. """ + credentials = ('devtable', 'password') + + adjusted_chunks = [] + for start_byte, end_byte in chunks: + adjusted_chunks.append((start_byte, end_byte if end_byte else len(random_layer_data))) + + images = [ + Image(id='theimage', parent_id=None, bytes=random_layer_data), + ] + + options = ProtocolOptions() + options.chunks_for_upload = adjusted_chunks + + # Push the image, using the specified chunking. + manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', + images, credentials=credentials, options=options) + + # Pull to verify the image was created. + puller.pull(liveserver_session, 'devtable', 'newrepo', 'latest', images, + credentials=credentials) + + +def test_chunked_uploading_mismatched_chunks(manifest_protocol, random_layer_data, + liveserver_session, app_reloader): + """ Test: Attempt to upload chunks with data missing. """ + credentials = ('devtable', 'password') + + images = [ + Image(id='theimage', parent_id=None, bytes=random_layer_data), + ] + + # Note: Byte #100 is missing. + options = ProtocolOptions() + options.chunks_for_upload = [(0, 100), (101, len(random_layer_data), 416)] + + # Attempt to push, with the chunked upload failing. + manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', + images, credentials=credentials, options=options) + + +@pytest.mark.parametrize('public_catalog, credentials, expected_repos', [ + # No public access and no credentials => No results. + (False, None, None), + + # Public access and no credentials => public repositories. + (True, None, ['public/publicrepo']), + + # Private creds => private repositories. + (False, ('devtable', 'password'), ['devtable/simple', 'devtable/complex', 'devtable/gargantuan']), + (True, ('devtable', 'password'), ['devtable/simple', 'devtable/complex', 'devtable/gargantuan']), +]) +@pytest.mark.parametrize('page_size', [ + 1, + 2, + 10, + 50, + 100, +]) +def test_catalog(public_catalog, credentials, expected_repos, page_size, v2_protocol, + liveserver_session, app_reloader, liveserver, registry_server_executor): + """ Test: Retrieving results from the V2 catalog. """ + with FeatureFlagValue('PUBLIC_CATALOG', public_catalog, registry_server_executor.on(liveserver)): + results = v2_protocol.catalog(liveserver_session, page_size=page_size, credentials=credentials, + namespace='devtable', repo_name='simple') + + if expected_repos is None: + assert len(results) == 0 + else: + assert set(expected_repos).issubset(set(results)) + + +def test_pull_torrent(pusher, basic_images, liveserver_session, liveserver, + registry_server_executor, app_reloader): + """ Test: Retrieve a torrent for pulling the image via the Quay CLI. """ + credentials = ('devtable', 'password') + + # Push an image to download. + pusher.push(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images, + credentials=credentials) + + # Required for torrent. + registry_server_executor.on(liveserver).set_supports_direct_download(True) + + # For each layer, retrieve a torrent for the blob. + for image in basic_images: + digest = 'sha256:' + hashlib.sha256(image.bytes).hexdigest() + response = liveserver_session.get('/c1/torrent/devtable/newrepo/blobs/%s' % digest, + auth=credentials) + torrent_info = bencode.bdecode(response.content) + + # Check the announce URL. + assert torrent_info['url-list'] == 'http://somefakeurl?goes=here' + + # Check the metadata. + assert torrent_info.get('info', {}).get('pieces') is not None + assert torrent_info.get('announce') is not None + + # Check the pieces. + sha = resumablehashlib.sha1() + sha.update(image.bytes) + + expected = binascii.hexlify(sha.digest()) + found = binascii.hexlify(torrent_info['info']['pieces']) + assert expected == found + + +@pytest.mark.parametrize('use_estimates', [ + False, + True, +]) +def test_squashed_images(use_estimates, pusher, sized_images, liveserver_session, + liveserver, registry_server_executor, app_reloader): + """ Test: Pulling of squashed images. """ + credentials = ('devtable', 'password') + + # Push an image to download. + pusher.push(liveserver_session, 'devtable', 'newrepo', 'latest', sized_images, + credentials=credentials) + + if use_estimates: + # Clear the uncompressed size stored for the images, to ensure that we estimate instead. + for image in sized_images: + registry_server_executor.on(liveserver).clear_uncompressed_size(image.id) + + # Pull the squashed version. + response = liveserver_session.get('/c1/squash/devtable/newrepo/latest', auth=credentials) + tar = tarfile.open(fileobj=StringIO(response.content)) + + # Verify the squashed image. + expected_image_id = '13a2d9711a3e242fcd50a7627c02d86901ac801b78dcea3147e8ff640078de52' + expected_names = ['repositories', + expected_image_id, + '%s/json' % expected_image_id, + '%s/VERSION' % expected_image_id, + '%s/layer.tar' % expected_image_id] + + assert tar.getnames() == expected_names + + # Verify the JSON image data. + json_data = (tar.extractfile(tar.getmember('%s/json' % expected_image_id)).read()) + + # Ensure the JSON loads and parses. + result = json.loads(json_data) + assert result['id'] == expected_image_id + + # Ensure that squashed layer tar can be opened. + tarfile.open(fileobj=tar.extractfile(tar.getmember('%s/layer.tar' % expected_image_id))) + + +def get_robot_password(api_caller): + api_caller.conduct_auth('devtable', 'password') + resp = api_caller.get('/api/v1/organization/buynlarge/robots/ownerbot') + return resp.json()['token'] + + +def get_encrypted_password(api_caller): + api_caller.conduct_auth('devtable', 'password') + resp = api_caller.post('/api/v1/user/clientkey', data=json.dumps(dict(password='password')), + headers={'Content-Type': 'application/json'}) + return resp.json()['key'] + + +@pytest.mark.parametrize('username, password, expect_success', [ + # Invalid username. + ('invaliduser', 'somepassword', False), + + # Invalid password. + ('devtable', 'invalidpassword', False), + + # Invalid OAuth token. + ('$oauthtoken', 'unknown', False), + + # Invalid CLI token. + ('$app', 'invalid', False), + + # Valid. + ('devtable', 'password', True), + + # Robot. + ('buynlarge+ownerbot', get_robot_password, True), + + # Encrypted password. + ('devtable', get_encrypted_password, True), + + # OAuth. + ('$oauthtoken', 'test', True), + + # CLI Token. + ('$app', 'test', True), +]) +def test_login(username, password, expect_success, loginer, liveserver_session, + api_caller, app_reloader): + """ Test: Login flow. """ + if not isinstance(password, str): + password = password(api_caller) + + loginer.login(liveserver_session, username, password, [], expect_success) + + +@pytest.mark.parametrize('username, password, scopes, expected_access, expect_success', [ + # No scopes. + ('devtable', 'password', [], [], True), + + # Basic pull. + ('devtable', 'password', ['repository:devtable/simple:pull'], [ + {'type': 'repository', 'name': 'devtable/simple', 'actions': ['pull']}, + ], True), + + # Basic push. + ('devtable', 'password', ['repository:devtable/simple:push'], [ + {'type': 'repository', 'name': 'devtable/simple', 'actions': ['push']}, + ], True), + + # Basic push/pull. + ('devtable', 'password', ['repository:devtable/simple:push,pull'], [ + {'type': 'repository', 'name': 'devtable/simple', 'actions': ['push', 'pull']}, + ], True), + + # Admin. + ('devtable', 'password', ['repository:devtable/simple:push,pull,*'], [ + {'type': 'repository', 'name': 'devtable/simple', 'actions': ['push', 'pull', '*']}, + ], True), + + # Basic pull with endpoint. + ('devtable', 'password', ['repository:localhost:5000/devtable/simple:pull'], [ + {'type': 'repository', 'name': 'localhost:5000/devtable/simple', 'actions': ['pull']}, + ], True), + + # Basic pull with invalid endpoint. + ('devtable', 'password', ['repository:someinvalid/devtable/simple:pull'], [], False), + + # Pull with no access. + ('public', 'password', ['repository:devtable/simple:pull'], [ + {'type': 'repository', 'name': 'devtable/simple', 'actions': []}, + ], True), + + # Anonymous push and pull on a private repository. + ('', '', ['repository:devtable/simple:pull,push'], [ + {'type': 'repository', 'name': 'devtable/simple', 'actions': []}, + ], True), + + # Pull and push with no push access. + ('reader', 'password', ['repository:buynlarge/orgrepo:pull,push'], [ + {'type': 'repository', 'name': 'buynlarge/orgrepo', 'actions': ['pull']}, + ], True), + + # OAuth. + ('$oauthtoken', 'test', ['repository:public/publicrepo:pull,push'], [ + {'type': 'repository', 'name': 'public/publicrepo', 'actions': ['pull']}, + ], True), + + # Anonymous public repo. + ('', '', ['repository:public/publicrepo:pull,push'], [ + {'type': 'repository', 'name': 'public/publicrepo', 'actions': ['pull']}, + ], True), + + # Multiple scopes. + ('devtable', 'password', + ['repository:devtable/simple:push,pull,*', 'repository:buynlarge/orgrepo:pull'], [ + {'type': 'repository', 'name': 'devtable/simple', 'actions': ['push', 'pull', '*']}, + {'type': 'repository', 'name': 'buynlarge/orgrepo', 'actions': ['pull']}, + ], True), + + # Multiple scopes. + ('devtable', 'password', + ['repository:devtable/simple:push,pull,*', 'repository:public/publicrepo:push,pull'], [ + {'type': 'repository', 'name': 'devtable/simple', 'actions': ['push', 'pull', '*']}, + {'type': 'repository', 'name': 'public/publicrepo', 'actions': ['pull']}, + ], True), +]) +def test_login_scopes(username, password, scopes, expected_access, expect_success, v2_protocol, + liveserver_session, api_caller, app_reloader): + """ Test: Login via the V2 auth protocol reacts correctly to requested scopes. """ + if not isinstance(password, str): + password = password(api_caller) + + response = v2_protocol.login(liveserver_session, username, password, scopes, expect_success) + if not expect_success: + return + + # Validate the returned token. + encoded = response.json()['token'] + payload = decode_bearer_header('Bearer ' + encoded, instance_keys, + {'SERVER_HOSTNAME': 'localhost:5000'}) + assert payload is not None + assert payload['access'] == expected_access diff --git a/tox.ini b/tox.ini index 648cb2ccc..9a359fd53 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27}-{unittest,registry} +envlist = {py27}-{unittest,registry,registry_old} skipsdist = True [testenv] @@ -11,7 +11,8 @@ deps = setenv = PYTHONPATH = {toxinidir}:{toxinidir} TEST=true - registry: FILE="test/registry_tests.py" + registry: FILE="test/registry/registry_tests.py" + registry_old: FILE="test/registry_tests.py" unittest: FILE="" commands = py.test --verbose {env:FILE} -vv {posargs} diff --git a/util/expiresdict.py b/util/expiresdict.py index 952cad26b..62c87b9d9 100644 --- a/util/expiresdict.py +++ b/util/expiresdict.py @@ -17,9 +17,9 @@ class ExpiresEntry(object): class ExpiresDict(object): """ ExpiresDict defines a dictionary-like class whose keys have expiration. The rebuilder is a function that returns the full contents of the cached dictionary as a dict of the keys - and whose values are TTLEntry's. + and whose values are TTLEntry's. If the rebuilder is None, then no rebuilding is performed. """ - def __init__(self, rebuilder): + def __init__(self, rebuilder=None): self._rebuilder = rebuilder self._items = {} @@ -49,6 +49,9 @@ class ExpiresDict(object): return self.get(key) is not None def _rebuild(self): + if self._rebuilder is None: + return self._items + items = self._rebuilder() self._items = items return items diff --git a/util/names.py b/util/names.py index 9fb6f3064..e8eb72645 100644 --- a/util/names.py +++ b/util/names.py @@ -14,6 +14,13 @@ TAG_REGEX = re.compile(FULL_TAG_PATTERN) TAG_ERROR = ('Invalid tag: must match [A-Za-z0-9_.-], NOT start with "." or "-", ' 'and can contain 1-128 characters') + +class ImplicitLibraryNamespaceNotAllowed(Exception): + """ Exception raised if the implicit library namespace was specified but is + not allowed. """ + pass + + def escape_tag(tag, default='latest'): """ Escapes a Docker tag, ensuring it matches the tag regular expression. """ if not tag: @@ -24,13 +31,16 @@ def escape_tag(tag, default='latest'): return tag[0:127] -def parse_namespace_repository(repository, library_namespace, include_tag=False): +def parse_namespace_repository(repository, library_namespace, include_tag=False, + allow_library=True): repository = repository.encode('unidecode', 'ignore') parts = repository.rstrip('/').split('/', 1) if len(parts) < 2: namespace = library_namespace repository = parts[0] + if not allow_library: + raise ImplicitLibraryNamespaceNotAllowed() else: (namespace, repository) = parts