diff --git a/.travis.yml b/.travis.yml index f0950275f..173cbe963 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,6 +48,9 @@ jobs: - stage: test script: scripts/ci registry_old + - stage: test + script: scripts/ci certs_test + - stage: database script: scripts/ci mysql diff --git a/Makefile b/Makefile index 50f8109c7..bfe578094 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,9 @@ registry-test-old: --timeout=3600 --verbose --show-count -x \ ./test/registry_tests.py +certs-test: + ./test/test_certs_install.sh + full-db-test: ensure-test-db TEST=true PYTHONPATH=. alembic upgrade head TEST=true PYTHONPATH=. SKIP_DB_SCHEMA=true py.test --timeout=7200 \ diff --git a/buildman/component/buildcomponent.py b/buildman/component/buildcomponent.py index 4b25c4636..d0e9edfae 100644 --- a/buildman/component/buildcomponent.py +++ b/buildman/component/buildcomponent.py @@ -17,9 +17,10 @@ from buildman.jobutil.buildstatus import StatusHandler from buildman.jobutil.workererror import WorkerError from app import app -from data import model from data.database import BUILD_PHASE, UseThenDisconnect from data.model import InvalidRepositoryBuildException +from data.registry_model import registry_model +from data.registry_model.datatypes import RepositoryReference from util import slash_join HEARTBEAT_DELTA = datetime.timedelta(seconds=60) @@ -29,6 +30,9 @@ INITIAL_TIMEOUT = 25 SUPPORTED_WORKER_VERSIONS = ['0.3'] +# Label which marks a manifest with its source build ID. +INTERNAL_LABEL_BUILD_UUID = 'quay.build.uuid' + logger = logging.getLogger(__name__) class ComponentStatus(object): @@ -357,18 +361,17 @@ class BuildComponent(BaseComponent): # Label the pushed manifests with the build metadata. manifest_digests = kwargs.get('digests') or [] - for digest in manifest_digests: - with UseThenDisconnect(app.config): - try: - manifest = model.tag.load_manifest_by_digest(self._current_job.namespace, - self._current_job.repo_name, digest) - model.label.create_manifest_label(manifest, model.label.INTERNAL_LABEL_BUILD_UUID, - build_id, 'internal', 'text/plain') - except model.InvalidManifestException: - logger.debug('Could not find built manifest with digest %s under repo %s/%s for build %s', - digest, self._current_job.namespace, self._current_job.repo_name, - build_id) - continue + repository = registry_model.lookup_repository(self._current_job.namespace, + self._current_job.repo_name) + if repository is not None: + for digest in manifest_digests: + with UseThenDisconnect(app.config): + manifest = registry_model.lookup_manifest_by_digest(repository, digest) + if manifest is None: + continue + + registry_model.create_manifest_label(manifest, INTERNAL_LABEL_BUILD_UUID, + build_id, 'internal', 'text/plain') # Send the notification that the build has completed successfully. self._current_job.send_notification('build_success', diff --git a/buildman/jobutil/buildjob.py b/buildman/jobutil/buildjob.py index 253d85ac9..99c786fca 100644 --- a/buildman/jobutil/buildjob.py +++ b/buildman/jobutil/buildjob.py @@ -8,7 +8,6 @@ from data import model from data.registry_model import registry_model from data.registry_model.datatypes import RepositoryReference from data.database import UseThenDisconnect -from util.imagetree import ImageTree from util.morecollections import AttrDict logger = logging.getLogger(__name__) diff --git a/conf/init/certs_install.sh b/conf/init/certs_install.sh index e58d282bb..8643faa8c 100755 --- a/conf/init/certs_install.sh +++ b/conf/init/certs_install.sh @@ -1,9 +1,9 @@ #! /bin/bash set -e QUAYPATH=${QUAYPATH:-"."} -QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf/stack"} +QUAYCONF=${QUAYCONF:-"$QUAYPATH/conf"} QUAYCONFIG=${QUAYCONFIG:-"$QUAYCONF/stack"} -CERTDIR=${QUAYCONFIG/extra_ca_certs} +CERTDIR=${CERTDIR:-"$QUAYCONFIG/extra_ca_certs"} # If we're running under kube, the previous script (02_get_kube_certs.sh) will put the certs in a different location if [[ "$KUBERNETES_SERVICE_HOST" != "" ]];then @@ -37,7 +37,7 @@ if [ -f $CERTDIR ]; then fi # Add extra trusted certificates (prefixed) -for f in $(find $CERTDIR/ -maxdepth 1 -type f -name "extra_ca*") +for f in $(find $QUAYCONFIG/ -maxdepth 1 -type f -name "extra_ca*") do echo "Installing extra cert $f" cp "$f" /usr/local/share/ca-certificates/ diff --git a/data/model/image.py b/data/model/image.py index d7b6149a1..527f1ccb7 100644 --- a/data/model/image.py +++ b/data/model/image.py @@ -357,7 +357,11 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created def get_image(repo, docker_image_id): try: - return Image.get(Image.docker_image_id == docker_image_id, Image.repository == repo) + return (Image + .select(Image, ImageStorage) + .join(ImageStorage) + .where(Image.docker_image_id == docker_image_id, Image.repository == repo) + .get()) except Image.DoesNotExist: return None diff --git a/data/model/label.py b/data/model/label.py index 1592efaa6..3e461e2d7 100644 --- a/data/model/label.py +++ b/data/model/label.py @@ -11,9 +11,6 @@ from util.validation import is_json logger = logging.getLogger(__name__) -# Label which marks a manifest with its source build ID. -INTERNAL_LABEL_BUILD_UUID = 'quay.build.uuid' - @lru_cache(maxsize=1) def get_label_source_types(): diff --git a/data/model/tag.py b/data/model/tag.py index eec39adcc..dbff7ff73 100644 --- a/data/model/tag.py +++ b/data/model/tag.py @@ -208,8 +208,9 @@ def list_active_repo_tags(repo): and (if present), their manifest. """ query = _tag_alive(RepositoryTag - .select(RepositoryTag, Image, TagManifest.digest) + .select(RepositoryTag, Image, ImageStorage, TagManifest.digest) .join(Image) + .join(ImageStorage) .where(RepositoryTag.repository == repo, RepositoryTag.hidden == False) .switch(RepositoryTag) .join(TagManifest, JOIN.LEFT_OUTER)) @@ -470,8 +471,9 @@ def get_tag_image(namespace_name, repository_name, tag_name, include_storage=Fal def list_repository_tag_history(repo_obj, page=1, size=100, specific_tag=None): query = (RepositoryTag - .select(RepositoryTag, Image) + .select(RepositoryTag, Image, ImageStorage) .join(Image) + .join(ImageStorage) .switch(RepositoryTag) .where(RepositoryTag.repository == repo_obj) .where(RepositoryTag.hidden == False) @@ -515,7 +517,7 @@ def restore_tag_to_manifest(repo_obj, tag_name, manifest_digest): # Change the tag manifest to point to the updated image. docker_image_id = tag_manifest.tag.image.docker_image_id - updated_tag = create_or_update_tag_for_repo(repo_obj.id, tag_name, docker_image_id, + updated_tag = create_or_update_tag_for_repo(repo_obj, tag_name, docker_image_id, reversion=True) tag_manifest.tag = updated_tag tag_manifest.save() @@ -544,8 +546,7 @@ def restore_tag_to_image(repo_obj, tag_name, docker_image_id): except DataModelException: existing_image = None - create_or_update_tag(repo_obj.namespace_user.username, repo_obj.name, tag_name, - docker_image_id, reversion=True) + create_or_update_tag_for_repo(repo_obj, tag_name, docker_image_id, reversion=True) return existing_image @@ -589,6 +590,16 @@ def get_active_tag(namespace, repo_name, tag_name): .where(RepositoryTag.name == tag_name, Repository.name == repo_name, Namespace.username == namespace)).get() +def get_active_tag_for_repo(repo, tag_name): + try: + return _tag_alive(RepositoryTag + .select(RepositoryTag, Image, ImageStorage) + .join(Image) + .join(ImageStorage) + .where(RepositoryTag.name == tag_name, + RepositoryTag.repository == repo)).get() + except RepositoryTag.DoesNotExist: + return None def get_possibly_expired_tag(namespace, repo_name, tag_name): return (RepositoryTag @@ -641,6 +652,13 @@ def populate_manifest(repository, manifest, legacy_image, storage_ids): return manifest_row +def get_tag_manifest(tag): + try: + return TagManifest.get(tag=tag) + except TagManifest.DoesNotExist: + return None + + def load_tag_manifest(namespace, repo_name, tag_name): try: return (_load_repo_manifests(namespace, repo_name) diff --git a/data/registry_model/datatype.py b/data/registry_model/datatype.py new file mode 100644 index 000000000..f5ea0b5ae --- /dev/null +++ b/data/registry_model/datatype.py @@ -0,0 +1,49 @@ +# pylint: disable=protected-access + +from functools import wraps, total_ordering + +def datatype(name, static_fields): + """ Defines a base class for a datatype that will represent a row from the database, + in an abstracted form. + """ + @total_ordering + class DataType(object): + __name__ = name + + def __init__(self, **kwargs): + self._db_id = kwargs.pop('db_id', None) + self._inputs = kwargs.pop('inputs', None) + self._fields = kwargs + + for name in static_fields: + assert name in self._fields, 'Missing field %s' % name + + def __eq__(self, other): + return self._db_id == other._db_id + + def __lt__(self, other): + return self._db_id < other._db_id + + def __getattr__(self, name): + if name in static_fields: + return self._fields[name] + + raise AttributeError('Unknown field `%s`' % name) + + return DataType + + +def requiresinput(input_name): + """ Marks a property on the data type as requiring an input to be invoked. """ + def inner(func): + @wraps(func) + def wrapper(self, *args, **kwargs): + if self._inputs.get(input_name) is None: + raise Exception('Cannot invoke function with missing input `%s`' % input_name) + + kwargs[input_name] = self._inputs[input_name] + result = func(self, *args, **kwargs) + return result + + return wrapper + return inner diff --git a/data/registry_model/datatypes.py b/data/registry_model/datatypes.py index 3f2cae187..9b6674120 100644 --- a/data/registry_model/datatypes.py +++ b/data/registry_model/datatypes.py @@ -1,20 +1,120 @@ -from collections import namedtuple +from enum import Enum, unique -class RepositoryReference(object): +from data.registry_model.datatype import datatype, requiresinput + +class RepositoryReference(datatype('Repository', [])): """ RepositoryReference is a reference to a repository, passed to registry interface methods. """ - def __init__(self, repo_id): - self.repo_id = repo_id - @classmethod def for_repo_obj(cls, repo_obj): - return RepositoryReference(repo_obj.id) + if repo_obj is None: + return None + + return RepositoryReference(db_id=repo_obj.id) -class Tag(namedtuple('Tag', ['id', 'name'])): +class Label(datatype('Label', ['key', 'value', 'uuid', 'source_type_name', 'media_type_name'])): + """ Label represents a label on a manifest. """ + @classmethod + def for_label(cls, label): + if label is None: + return None + + return Label(db_id=label.id, key=label.key, value=label.value, + uuid=label.uuid, media_type_name=label.media_type.name, + source_type_name=label.source_type.name) + + +class Tag(datatype('Tag', ['name', 'reversion', 'manifest_digest', 'lifetime_start_ts', + 'lifetime_end_ts'])): """ Tag represents a tag in a repository, which points to a manifest or image. """ @classmethod - def for_repository_tag(cls, repository_tag): + def for_repository_tag(cls, repository_tag, manifest_digest=None, legacy_image=None): if repository_tag is None: return None - return Tag(id=repository_tag.id, name=repository_tag.name) + return Tag(db_id=repository_tag.id, + name=repository_tag.name, + reversion=repository_tag.reversion, + lifetime_start_ts=repository_tag.lifetime_start_ts, + lifetime_end_ts=repository_tag.lifetime_end_ts, + manifest_digest=manifest_digest, + inputs=dict(legacy_image=legacy_image)) + + @property + @requiresinput('legacy_image') + def legacy_image(self, legacy_image): + """ Returns the legacy Docker V1-style image for this tag. Note that this + will be None for tags whose manifests point to other manifests instead of images. + """ + return legacy_image + + +class Manifest(datatype('Manifest', ['digest', 'manifest_bytes'])): + """ Manifest represents a manifest in a repository. """ + @classmethod + def for_tag_manifest(cls, tag_manifest, legacy_image=None): + if tag_manifest is None: + return None + + return Manifest(db_id=tag_manifest.id, digest=tag_manifest.digest, + manifest_bytes=tag_manifest.json_data, + inputs=dict(legacy_image=legacy_image)) + + @property + @requiresinput('legacy_image') + def legacy_image(self, legacy_image): + """ Returns the legacy Docker V1-style image for this manifest. Note that this + will be None for manifests that point to other manifests instead of images. + """ + return legacy_image + + +class LegacyImage(datatype('LegacyImage', ['docker_image_id', 'created', 'comment', 'command', + 'image_size', 'aggregate_size', 'uploading'])): + """ LegacyImage represents a Docker V1-style image found in a repository. """ + @classmethod + def for_image(cls, image, images_map=None, tags_map=None): + if image is None: + return None + + return LegacyImage(db_id=image.id, + inputs=dict(images_map=images_map, tags_map=tags_map, + ancestor_id_list=image.ancestor_id_list()), + docker_image_id=image.docker_image_id, + created=image.created, + comment=image.comment, + command=image.command, + image_size=image.storage.image_size, + aggregate_size=image.aggregate_size, + uploading=image.storage.uploading) + + @property + @requiresinput('images_map') + @requiresinput('ancestor_id_list') + def parents(self, images_map, ancestor_id_list): + """ Returns the parent images for this image. Raises an exception if the parents have + not been loaded before this property is invoked. + """ + return [LegacyImage.for_image(images_map[ancestor_id], images_map=images_map) + for ancestor_id in reversed(ancestor_id_list) + if images_map.get(ancestor_id)] + + @property + @requiresinput('tags_map') + def tags(self, tags_map): + """ Returns the tags pointing to this image. Raises an exception if the tags have + not been loaded before this property is invoked. + """ + tags = tags_map.get(self._db_id) + if not tags: + return [] + + return [Tag.for_repository_tag(tag) for tag in tags] + + +@unique +class SecurityScanStatus(Enum): + """ Security scan status enum """ + SCANNED = 'scanned' + FAILED = 'failed' + QUEUED = 'queued' diff --git a/data/registry_model/interface.py b/data/registry_model/interface.py index e67366733..323d83f85 100644 --- a/data/registry_model/interface.py +++ b/data/registry_model/interface.py @@ -19,3 +19,106 @@ class RegistryDataInterface(object): """ Returns the most recently pushed alive tag in the repository, if any. If none, returns None. """ + + @abstractmethod + def lookup_repository(self, namespace_name, repo_name, kind_filter=None): + """ Looks up and returns a reference to the repository with the given namespace and name, + or None if none. """ + + @abstractmethod + def get_manifest_for_tag(self, tag): + """ Returns the manifest associated with the given tag. """ + + @abstractmethod + def lookup_manifest_by_digest(self, repository_ref, manifest_digest, allow_dead=False): + """ Looks up the manifest with the given digest under the given repository and returns it + or None if none. """ + + @abstractmethod + def get_legacy_images(self, repository_ref): + """ + Returns an iterator of all the LegacyImage's defined in the matching repository. + """ + + @abstractmethod + def get_legacy_image(self, repository_ref, docker_image_id, include_parents=False): + """ + Returns the matching LegacyImages under the matching repository, if any. If none, + returns None. + """ + + @abstractmethod + def create_manifest_label(self, manifest, key, value, source_type_name, media_type_name=None): + """ Creates a label on the manifest with the given key and value. + + Can raise InvalidLabelKeyException or InvalidMediaTypeException depending + on the validation errors. + """ + + @abstractmethod + def list_manifest_labels(self, manifest, key_prefix=None): + """ Returns all labels found on the manifest. If specified, the key_prefix will filter the + labels returned to those keys that start with the given prefix. + """ + + @abstractmethod + def get_manifest_label(self, manifest, label_uuid): + """ Returns the label with the specified UUID on the manifest or None if none. """ + + @abstractmethod + def delete_manifest_label(self, manifest, label_uuid): + """ Delete the label with the specified UUID on the manifest. Returns the label deleted + or None if none. + """ + + @abstractmethod + def list_repository_tags(self, repository_ref, include_legacy_images=False): + """ + Returns a list of all the active tags in the repository. Note that this can be a *heavy* + operation on repositories with a lot of tags, and should be avoided for more targetted + operations wherever possible. + """ + + @abstractmethod + def list_repository_tag_history(self, repository_ref, page=1, size=100, specific_tag_name=None): + """ + Returns the history of all tags in the repository (unless filtered). This includes tags that + have been made in-active due to newer versions of those tags coming into service. + """ + + @abstractmethod + def get_repo_tag(self, repository_ref, tag_name, include_legacy_image=False): + """ + Returns the latest, *active* tag found in the repository, with the matching name + or None if none. + """ + + @abstractmethod + def retarget_tag(self, repository_ref, tag_name, manifest_or_legacy_image, + is_reversion=False): + """ + Creates, updates or moves a tag to a new entry in history, pointing to the manifest or + legacy image specified. If is_reversion is set to True, this operation is considered a + reversion over a previous tag move operation. Returns the updated Tag or None on error. + """ + + @abstractmethod + def delete_tag(self, repository_ref, tag_name): + """ + Deletes the latest, *active* tag with the given name in the repository. + """ + + @abstractmethod + def change_repository_tag_expiration(self, tag, expiration_date): + """ Sets the expiration date of the tag under the matching repository to that given. If the + expiration date is None, then the tag will not expire. Returns a tuple of the previous + expiration timestamp in seconds (if any), and whether the operation succeeded. + """ + + @abstractmethod + def get_legacy_images_owned_by_tag(self, tag): + """ Returns all legacy images *solely owned and used* by the given tag. """ + + @abstractmethod + def get_security_status(self, manifest_or_legacy_image): + """ Returns the security status for the given manifest or legacy image or None if none. """ diff --git a/data/registry_model/registry_pre_oci_model.py b/data/registry_model/registry_pre_oci_model.py index 5cd6ed631..a7f5eabfe 100644 --- a/data/registry_model/registry_pre_oci_model.py +++ b/data/registry_model/registry_pre_oci_model.py @@ -1,6 +1,12 @@ +# pylint: disable=protected-access + +from collections import defaultdict + +from data import database from data import model from data.registry_model.interface import RegistryDataInterface -from data.registry_model.datatypes import Tag +from data.registry_model.datatypes import (Tag, RepositoryReference, Manifest, LegacyImage, Label, + SecurityScanStatus) class PreOCIModel(RegistryDataInterface): @@ -13,15 +19,275 @@ class PreOCIModel(RegistryDataInterface): """ Finds an alive tag in the repository matching one of the given tag names and returns it or None if none. """ - found_tag = model.tag.find_matching_tag(repository_ref.repo_id, tag_names) + found_tag = model.tag.find_matching_tag(repository_ref._db_id, tag_names) return Tag.for_repository_tag(found_tag) def get_most_recent_tag(self, repository_ref): """ Returns the most recently pushed alive tag in the repository, if any. If none, returns None. """ - found_tag = model.tag.get_most_recent_tag(repository_ref.repo_id) + found_tag = model.tag.get_most_recent_tag(repository_ref._db_id) return Tag.for_repository_tag(found_tag) + def lookup_repository(self, namespace_name, repo_name, kind_filter=None): + """ Looks up and returns a reference to the repository with the given namespace and name, + or None if none. """ + repo = model.repository.get_repository(namespace_name, repo_name, kind_filter=kind_filter) + return RepositoryReference.for_repo_obj(repo) + + def get_manifest_for_tag(self, tag): + """ Returns the manifest associated with the given tag. """ + try: + tag_manifest = database.TagManifest.get(tag_id=tag._db_id) + except database.TagManifest.DoesNotExist: + return + + return Manifest.for_tag_manifest(tag_manifest) + + def lookup_manifest_by_digest(self, repository_ref, manifest_digest, allow_dead=False, + include_legacy_image=False): + """ Looks up the manifest with the given digest under the given repository and returns it + or None if none. """ + repo = model.repository.lookup_repository(repository_ref._db_id) + if repo is None: + return None + + try: + tag_manifest = model.tag.load_manifest_by_digest(repo.namespace_user.username, + repo.name, + manifest_digest, + allow_dead=allow_dead) + except model.tag.InvalidManifestException: + return None + + legacy_image = None + if include_legacy_image: + legacy_image = self.get_legacy_image(repository_ref, tag_manifest.tag.image.docker_image_id, + include_parents=True) + + return Manifest.for_tag_manifest(tag_manifest, legacy_image) + + def get_legacy_images(self, repository_ref): + """ + Returns an iterator of all the LegacyImage's defined in the matching repository. + """ + repo = model.repository.lookup_repository(repository_ref._db_id) + if repo is None: + return None + + all_images = model.image.get_repository_images_without_placements(repo) + all_images_map = {image.id: image for image in all_images} + + all_tags = model.tag.list_repository_tags(repo.namespace_user.username, repo.name) + tags_by_image_id = defaultdict(list) + for tag in all_tags: + tags_by_image_id[tag.image_id].append(tag) + + return [LegacyImage.for_image(image, images_map=all_images_map, tags_map=tags_by_image_id) + for image in all_images] + + def get_legacy_image(self, repository_ref, docker_image_id, include_parents=False): + """ + Returns the matching LegacyImages under the matching repository, if any. If none, + returns None. + """ + repo = model.repository.lookup_repository(repository_ref._db_id) + if repo is None: + return None + + image = model.image.get_image(repository_ref._db_id, docker_image_id) + if image is None: + return None + + parent_images_map = None + if include_parents: + parent_images = model.image.get_parent_images(repo.namespace_user.username, repo.name, image) + parent_images_map = {image.id: image for image in parent_images} + + return LegacyImage.for_image(image, images_map=parent_images_map) + + def create_manifest_label(self, manifest, key, value, source_type_name, media_type_name=None): + """ Creates a label on the manifest with the given key and value. """ + try: + tag_manifest = database.TagManifest.get(id=manifest._db_id) + except database.TagManifest.DoesNotExist: + return None + + label = model.label.create_manifest_label(tag_manifest, key, value, source_type_name, + media_type_name) + return Label.for_label(label) + + def list_manifest_labels(self, manifest, key_prefix=None): + """ Returns all labels found on the manifest. If specified, the key_prefix will filter the + labels returned to those keys that start with the given prefix. + """ + labels = model.label.list_manifest_labels(manifest._db_id, prefix_filter=key_prefix) + return [Label.for_label(l) for l in labels] + + def get_manifest_label(self, manifest, label_uuid): + """ Returns the label with the specified UUID on the manifest or None if none. """ + return Label.for_label(model.label.get_manifest_label(label_uuid, manifest._db_id)) + + def delete_manifest_label(self, manifest, label_uuid): + """ Delete the label with the specified UUID on the manifest. Returns the label deleted + or None if none. + """ + return Label.for_label(model.label.delete_manifest_label(label_uuid, manifest._db_id)) + + def list_repository_tags(self, repository_ref, include_legacy_images=False): + """ + Returns a list of all the active tags in the repository. Note that this can be a *heavy* + operation on repositories with a lot of tags, and should be avoided for more targetted + operations wherever possible. + """ + # NOTE: include_legacy_images isn't used here because `list_active_repo_tags` includes the + # information already, so we might as well just use it. However, the new model classes will + # *not* include it by default, so we make it a parameter now. + tags = model.tag.list_active_repo_tags(repository_ref._db_id) + return [Tag.for_repository_tag(tag, + legacy_image=LegacyImage.for_image(tag.image), + manifest_digest=(tag.tagmanifest.digest + if hasattr(tag, 'tagmanifest') + else None)) + for tag in tags] + + def list_repository_tag_history(self, repository_ref, page=1, size=100, specific_tag_name=None): + """ + Returns the history of all tags in the repository (unless filtered). This includes tags that + have been made in-active due to newer versions of those tags coming into service. + """ + tags, manifest_map, has_more = model.tag.list_repository_tag_history(repository_ref._db_id, + page, size, + specific_tag_name) + return [Tag.for_repository_tag(tag, manifest_map.get(tag.id), + legacy_image=LegacyImage.for_image(tag.image)) + for tag in tags], has_more + + def get_repo_tag(self, repository_ref, tag_name, include_legacy_image=False): + """ + Returns the latest, *active* tag found in the repository, with the matching name + or None if none. + """ + tag = model.tag.get_active_tag_for_repo(repository_ref._db_id, tag_name) + if tag is None: + return None + + legacy_image = LegacyImage.for_image(tag.image) if include_legacy_image else None + tag_manifest = model.tag.get_tag_manifest(tag) + manifest_digest = tag_manifest.digest if tag_manifest else None + return Tag.for_repository_tag(tag, legacy_image=legacy_image, manifest_digest=manifest_digest) + + def retarget_tag(self, repository_ref, tag_name, manifest_or_legacy_image, + is_reversion=False): + """ + Creates, updates or moves a tag to a new entry in history, pointing to the manifest or + legacy image specified. If is_reversion is set to True, this operation is considered a + reversion over a previous tag move operation. Returns the updated Tag or None on error. + """ + # TODO: unify this. + if not is_reversion: + if isinstance(manifest_or_legacy_image, Manifest): + raise NotImplementedError('Not yet implemented') + else: + model.tag.create_or_update_tag_for_repo(repository_ref._db_id, tag_name, + manifest_or_legacy_image.docker_image_id) + else: + if isinstance(manifest_or_legacy_image, Manifest): + image = model.tag.restore_tag_to_manifest(repository_ref._db_id, tag_name, + manifest_or_legacy_image.digest) + if image is None: + return None + else: + image = model.tag.restore_tag_to_image(repository_ref._db_id, tag_name, + manifest_or_legacy_image.docker_image_id) + if image is None: + return None + + return self.get_repo_tag(repository_ref, tag_name, include_legacy_image=True) + + def delete_tag(self, repository_ref, tag_name): + """ + Deletes the latest, *active* tag with the given name in the repository. + """ + repo = model.repository.lookup_repository(repository_ref._db_id) + if repo is None: + return None + + deleted_tag = model.tag.delete_tag(repo.namespace_user.username, repo.name, tag_name) + return Tag.for_repository_tag(deleted_tag) + + def change_repository_tag_expiration(self, tag, expiration_date): + """ Sets the expiration date of the tag under the matching repository to that given. If the + expiration date is None, then the tag will not expire. Returns a tuple of the previous + expiration timestamp in seconds (if any), and whether the operation succeeded. + """ + try: + tag_obj = database.RepositoryTag.get(id=tag._db_id) + except database.RepositoryTag.DoesNotExist: + return (None, False) + + return model.tag.change_tag_expiration(tag_obj, expiration_date) + + def get_legacy_images_owned_by_tag(self, tag): + """ Returns all legacy images *solely owned and used* by the given tag. """ + try: + tag_obj = database.RepositoryTag.get(id=tag._db_id) + except database.RepositoryTag.DoesNotExist: + return None + + # Collect the IDs of all images that the tag uses. + tag_image_ids = set() + tag_image_ids.add(tag_obj.image.id) + tag_image_ids.update(tag_obj.image.ancestor_id_list()) + + # Remove any images shared by other tags. + for current_tag in model.tag.list_active_repo_tags(tag_obj.repository_id): + if current_tag == tag_obj: + continue + + tag_image_ids.discard(current_tag.image.id) + tag_image_ids = tag_image_ids.difference(current_tag.image.ancestor_id_list()) + if not tag_image_ids: + return [] + + if not tag_image_ids: + return [] + + # Load the images we need to return. + images = database.Image.select().where(database.Image.id << list(tag_image_ids)) + all_image_ids = set() + for image in images: + all_image_ids.add(image.id) + all_image_ids.update(image.ancestor_id_list()) + + # Build a map of all the images and their parents. + images_map = {} + all_images = database.Image.select().where(database.Image.id << list(all_image_ids)) + for image in all_images: + images_map[image.id] = image + + return [LegacyImage.for_image(image, images_map=images_map) for image in images] + + def get_security_status(self, manifest_or_legacy_image): + """ Returns the security status for the given manifest or legacy image or None if none. """ + image = None + + if isinstance(manifest_or_legacy_image, Manifest): + try: + tag_manifest = database.TagManifest.get(id=manifest_or_legacy_image._db_id) + image = tag_manifest.tag.image + except database.TagManifest.DoesNotExist: + return None + else: + try: + image = database.Image.get(id=manifest_or_legacy_image._db_id) + except database.Image.DoesNotExist: + return None + + if image.security_indexed_engine is not None and image.security_indexed_engine >= 0: + return SecurityScanStatus.SCANNED if image.security_indexed else SecurityScanStatus.FAILED + + return SecurityScanStatus.QUEUED + pre_oci_model = PreOCIModel() diff --git a/data/registry_model/test/test_pre_oci_model.py b/data/registry_model/test/test_pre_oci_model.py index 6e81c3fff..dfff01bbf 100644 --- a/data/registry_model/test/test_pre_oci_model.py +++ b/data/registry_model/test/test_pre_oci_model.py @@ -1,8 +1,13 @@ +from datetime import datetime, timedelta + import pytest +from playhouse.test_utils import assert_query_count + from data import model from data.registry_model.registry_pre_oci_model import PreOCIModel from data.registry_model.datatypes import RepositoryReference + from test.fixtures import * @pytest.fixture() @@ -28,7 +33,7 @@ def test_find_matching_tag(names, expected, pre_oci_model): @pytest.mark.parametrize('repo_namespace, repo_name, expected', [ - ('devtable', 'simple', {'latest'}), + ('devtable', 'simple', {'latest', 'prod'}), ('buynlarge', 'orgrepo', {'latest', 'prod'}), ]) def test_get_most_recent_tag(repo_namespace, repo_name, expected, pre_oci_model): @@ -39,3 +44,274 @@ def test_get_most_recent_tag(repo_namespace, repo_name, expected, pre_oci_model) assert found is None else: assert found.name in expected + + +@pytest.mark.parametrize('repo_namespace, repo_name, expected', [ + ('devtable', 'simple', True), + ('buynlarge', 'orgrepo', True), + ('buynlarge', 'unknownrepo', False), +]) +def test_lookup_repository(repo_namespace, repo_name, expected, pre_oci_model): + repo_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) + if expected: + assert repo_ref + else: + assert repo_ref is None + + +@pytest.mark.parametrize('repo_namespace, repo_name', [ + ('devtable', 'simple'), + ('buynlarge', 'orgrepo'), +]) +def test_lookup_manifests(repo_namespace, repo_name, pre_oci_model): + repo = model.repository.get_repository(repo_namespace, repo_name) + repository_ref = RepositoryReference.for_repo_obj(repo) + found_tag = pre_oci_model.find_matching_tag(repository_ref, ['latest']) + found_manifest = pre_oci_model.get_manifest_for_tag(found_tag) + found = pre_oci_model.lookup_manifest_by_digest(repository_ref, found_manifest.digest, + include_legacy_image=True) + assert found._db_id == found_manifest._db_id + assert found.digest == found_manifest.digest + assert found.legacy_image + + +def test_lookup_unknown_manifest(pre_oci_model): + repo = model.repository.get_repository('devtable', 'simple') + repository_ref = RepositoryReference.for_repo_obj(repo) + found = pre_oci_model.lookup_manifest_by_digest(repository_ref, 'sha256:deadbeef') + assert found is None + + +@pytest.mark.parametrize('repo_namespace, repo_name', [ + ('devtable', 'simple'), + ('devtable', 'complex'), + ('devtable', 'history'), + ('buynlarge', 'orgrepo'), +]) +def test_legacy_images(repo_namespace, repo_name, pre_oci_model): + repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) + legacy_images = pre_oci_model.get_legacy_images(repository_ref) + assert len(legacy_images) + + found_tags = set() + for image in legacy_images: + found_image = pre_oci_model.get_legacy_image(repository_ref, image.docker_image_id, + include_parents=True) + + with assert_query_count(4 if found_image.parents else 3): + found_image = pre_oci_model.get_legacy_image(repository_ref, image.docker_image_id, + include_parents=True) + assert found_image.docker_image_id == image.docker_image_id + assert found_image.parents == image.parents + + # Check that the tags list can be retrieved. + assert image.tags is not None + found_tags.update({tag.name for tag in image.tags}) + + # Check against the actual DB row. + model_image = model.image.get_image(repository_ref._db_id, found_image.docker_image_id) + assert model_image.id == found_image._db_id + assert ([pid for pid in reversed(model_image.ancestor_id_list())] == + [p._db_id for p in found_image.parents]) + + # Try without parents and ensure it raises an exception. + found_image = pre_oci_model.get_legacy_image(repository_ref, image.docker_image_id, + include_parents=False) + with pytest.raises(Exception): + assert not found_image.parents + + assert found_tags + + unknown = pre_oci_model.get_legacy_image(repository_ref, 'unknown', include_parents=True) + assert unknown is None + + +def test_manifest_labels(pre_oci_model): + repo = model.repository.get_repository('devtable', 'simple') + repository_ref = RepositoryReference.for_repo_obj(repo) + found_tag = pre_oci_model.find_matching_tag(repository_ref, ['latest']) + found_manifest = pre_oci_model.get_manifest_for_tag(found_tag) + + # Create a new label. + created = pre_oci_model.create_manifest_label(found_manifest, 'foo', 'bar', 'api') + assert created.key == 'foo' + assert created.value == 'bar' + assert created.source_type_name == 'api' + assert created.media_type_name == 'text/plain' + + # Ensure we can look it up. + assert pre_oci_model.get_manifest_label(found_manifest, created.uuid) == created + + # Ensure it is in our list of labels. + assert created in pre_oci_model.list_manifest_labels(found_manifest) + assert created in pre_oci_model.list_manifest_labels(found_manifest, key_prefix='fo') + + # Ensure it is *not* in our filtered list. + assert created not in pre_oci_model.list_manifest_labels(found_manifest, key_prefix='ba') + + # Delete the label and ensure it is gone. + assert pre_oci_model.delete_manifest_label(found_manifest, created.uuid) + assert pre_oci_model.get_manifest_label(found_manifest, created.uuid) is None + assert created not in pre_oci_model.list_manifest_labels(found_manifest) + + +@pytest.mark.parametrize('repo_namespace, repo_name', [ + ('devtable', 'simple'), + ('devtable', 'complex'), + ('devtable', 'history'), + ('buynlarge', 'orgrepo'), +]) +def test_repository_tags(repo_namespace, repo_name, pre_oci_model): + repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) + + with assert_query_count(1): + tags = pre_oci_model.list_repository_tags(repository_ref, include_legacy_images=True) + assert len(tags) + + for tag in tags: + with assert_query_count(2): + found_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True) + assert found_tag == tag + + if found_tag.legacy_image is None: + continue + + with assert_query_count(2): + found_image = pre_oci_model.get_legacy_image(repository_ref, + found_tag.legacy_image.docker_image_id) + assert found_image == found_tag.legacy_image + + +def test_repository_tag_history(pre_oci_model): + repository_ref = pre_oci_model.lookup_repository('devtable', 'history') + + with assert_query_count(2): + history, has_more = pre_oci_model.list_repository_tag_history(repository_ref) + assert not has_more + assert len(history) == 2 + + +@pytest.mark.parametrize('repo_namespace, repo_name', [ + ('devtable', 'simple'), + ('devtable', 'complex'), + ('devtable', 'history'), + ('buynlarge', 'orgrepo'), +]) +def test_delete_tags(repo_namespace, repo_name, pre_oci_model): + repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) + tags = pre_oci_model.list_repository_tags(repository_ref) + assert len(tags) + + # Save history before the deletions. + previous_history, _ = pre_oci_model.list_repository_tag_history(repository_ref, size=1000) + assert len(previous_history) >= len(tags) + + # Delete every tag in the repository. + for tag in tags: + assert pre_oci_model.delete_tag(repository_ref, tag.name) + + # Make sure the tag is no longer found. + with assert_query_count(1): + found_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True) + assert found_tag is None + + # Ensure all tags have been deleted. + tags = pre_oci_model.list_repository_tags(repository_ref) + assert not len(tags) + + # Ensure that the tags all live in history. + history, _ = pre_oci_model.list_repository_tag_history(repository_ref, size=1000) + assert len(history) == len(previous_history) + + +@pytest.mark.parametrize('use_manifest', [ + True, + False, +]) +def test_retarget_tag_history(use_manifest, pre_oci_model): + repository_ref = pre_oci_model.lookup_repository('devtable', 'history') + history, _ = pre_oci_model.list_repository_tag_history(repository_ref) + + if use_manifest: + manifest_or_legacy_image = pre_oci_model.lookup_manifest_by_digest(repository_ref, + history[1].manifest_digest, + allow_dead=True) + else: + manifest_or_legacy_image = history[1].legacy_image + + # Retarget the tag. + assert manifest_or_legacy_image + updated_tag = pre_oci_model.retarget_tag(repository_ref, 'latest', manifest_or_legacy_image, + is_reversion=True) + + # Ensure the tag has changed targets. + if use_manifest: + assert updated_tag.manifest_digest == manifest_or_legacy_image.digest + else: + assert updated_tag.legacy_image == manifest_or_legacy_image + + # Ensure history has been updated. + new_history, _ = pre_oci_model.list_repository_tag_history(repository_ref) + assert len(new_history) == len(history) + 1 + + +def test_retarget_tag(pre_oci_model): + repository_ref = pre_oci_model.lookup_repository('devtable', 'complex') + history, _ = pre_oci_model.list_repository_tag_history(repository_ref) + + prod_tag = pre_oci_model.get_repo_tag(repository_ref, 'prod', include_legacy_image=True) + + # Retarget the tag. + updated_tag = pre_oci_model.retarget_tag(repository_ref, 'latest', prod_tag.legacy_image) + + # Ensure the tag has changed targets. + assert updated_tag.legacy_image == prod_tag.legacy_image + + # Ensure history has been updated. + new_history, _ = pre_oci_model.list_repository_tag_history(repository_ref) + assert len(new_history) == len(history) + 1 + + +def test_change_repository_tag_expiration(pre_oci_model): + repository_ref = pre_oci_model.lookup_repository('devtable', 'simple') + tag = pre_oci_model.get_repo_tag(repository_ref, 'latest') + assert tag.lifetime_end_ts is None + + new_datetime = datetime.utcnow() + timedelta(days=2) + previous, okay = pre_oci_model.change_repository_tag_expiration(tag, new_datetime) + + assert okay + assert previous is None + + tag = pre_oci_model.get_repo_tag(repository_ref, 'latest') + assert tag.lifetime_end_ts is not None + + +@pytest.mark.parametrize('repo_namespace, repo_name, expected_non_empty', [ + ('devtable', 'simple', []), + ('devtable', 'complex', ['prod', 'v2.0']), + ('devtable', 'history', ['latest']), + ('buynlarge', 'orgrepo', []), + ('devtable', 'gargantuan', ['v2.0', 'v3.0', 'v4.0', 'v5.0', 'v6.0']), +]) +def test_get_legacy_images_owned_by_tag(repo_namespace, repo_name, expected_non_empty, + pre_oci_model): + repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) + tags = pre_oci_model.list_repository_tags(repository_ref) + assert len(tags) + + non_empty = set() + for tag in tags: + if pre_oci_model.get_legacy_images_owned_by_tag(tag): + non_empty.add(tag.name) + + assert non_empty == set(expected_non_empty) + + +def test_get_security_status(pre_oci_model): + repository_ref = pre_oci_model.lookup_repository('devtable', 'simple') + tags = pre_oci_model.list_repository_tags(repository_ref, include_legacy_images=True) + assert len(tags) + + for tag in tags: + assert pre_oci_model.get_security_status(tag.legacy_image) diff --git a/endpoints/api/image.py b/endpoints/api/image.py index 522fdf951..9fb3c5c92 100644 --- a/endpoints/api/image.py +++ b/endpoints/api/image.py @@ -1,11 +1,35 @@ """ List and lookup repository images. """ +import json +from data.registry_model import registry_model from endpoints.api import (resource, nickname, require_repo_read, RepositoryParamResource, - path_param, disallow_for_app_repositories) -from endpoints.api.image_models_pre_oci import pre_oci_model as model + path_param, disallow_for_app_repositories, format_date) from endpoints.exception import NotFound +def image_dict(image, with_history=False, with_tags=False): + image_data = { + 'id': image.docker_image_id, + 'created': format_date(image.created), + 'comment': image.comment, + 'command': json.loads(image.command) if image.command else None, + 'size': image.image_size, + 'uploading': image.uploading, + 'sort_index': len(image.parents), + } + + if with_tags: + image_data['tags'] = [tag.name for tag in image.tags] + + if with_history: + image_data['history'] = [image_dict(parent) for parent in image.parents] + + # Calculate the ancestors string, with the DBID's replaced with the docker IDs. + parent_docker_ids = [parent_image.docker_image_id for parent_image in image.parents] + image_data['ancestors'] = '/{0}/'.format('/'.join(parent_docker_ids)) + return image_data + + @resource('/v1/repository//image/') @path_param('repository', 'The full path of the repository. e.g. namespace/name') class RepositoryImageList(RepositoryParamResource): @@ -16,11 +40,12 @@ class RepositoryImageList(RepositoryParamResource): @disallow_for_app_repositories def get(self, namespace, repository): """ List the images for the specified repository. """ - images = model.get_repository_images(namespace, repository) - if images is None: + repo_ref = registry_model.lookup_repository(namespace, repository) + if repo_ref is None: raise NotFound() - return {'images': [image.to_dict() for image in images]} + images = registry_model.get_legacy_images(repo_ref) + return {'images': [image_dict(image, with_tags=True) for image in images]} @resource('/v1/repository//image/') @@ -34,8 +59,12 @@ class RepositoryImage(RepositoryParamResource): @disallow_for_app_repositories def get(self, namespace, repository, image_id): """ Get the information available for the specified image. """ - image = model.get_repository_image(namespace, repository, image_id) + repo_ref = registry_model.lookup_repository(namespace, repository) + if repo_ref is None: + raise NotFound() + + image = registry_model.get_legacy_image(repo_ref, image_id, include_parents=True) if image is None: raise NotFound() - return image.to_dict() + return image_dict(image, with_history=True) diff --git a/endpoints/api/image_models_interface.py b/endpoints/api/image_models_interface.py deleted file mode 100644 index 0cd5c0a8f..000000000 --- a/endpoints/api/image_models_interface.py +++ /dev/null @@ -1,77 +0,0 @@ -import json - -from endpoints.api import format_date - -from abc import ABCMeta, abstractmethod -from collections import namedtuple -from six import add_metaclass - -class Image(namedtuple('Image', ['docker_image_id', 'created', 'comment', 'command', 'image_size', - 'uploading', 'parents'])): - """ - Image represents an image. - :type name: string - """ - - def to_dict(self): - image_data = { - 'id': self.docker_image_id, - 'created': format_date(self.created), - 'comment': self.comment, - 'command': json.loads(self.command) if self.command else None, - 'size': self.image_size, - 'uploading': self.uploading, - 'sort_index': len(self.parents), - } - - # Calculate the ancestors string, with the DBID's replaced with the docker IDs. - parent_docker_ids = [parent_image.docker_image_id for parent_image in self.parents] - image_data['ancestors'] = '/{0}/'.format('/'.join(parent_docker_ids)) - - return image_data - -class ImageWithTags(namedtuple('ImageWithTags', ['image', 'tag_names'])): - """ - ImageWithTags represents an image, along with the tags that point to it. - :type image: Image - :type tag_names: list of string - """ - - def to_dict(self): - image_dict = self.image.to_dict() - image_dict['tags'] = self.tag_names - return image_dict - - -class ImageWithHistory(namedtuple('ImageWithHistory', ['image'])): - """ - ImageWithHistory represents an image, along with its full parent image dictionaries. - :type image: Image - :type history: list of Image parents (name is old and must be kept for compat) - """ - - def to_dict(self): - image_dict = self.image.to_dict() - image_dict['history'] = [parent_image.to_dict() for parent_image in self.image.parents] - return image_dict - - -@add_metaclass(ABCMeta) -class ImageInterface(object): - """ - Interface that represents all data store interactions required by the image API endpoint. - """ - - @abstractmethod - def get_repository_images(self, namespace_name, repo_name): - """ - Returns an iterator of all the ImageWithTag's defined in the matching repository. If the - repository doesn't exist, returns None. - """ - - @abstractmethod - def get_repository_image(self, namespace_name, repo_name, docker_image_id): - """ - Returns the matching ImageWithHistory under the matching repository, if any. If none, - returns None. - """ diff --git a/endpoints/api/image_models_pre_oci.py b/endpoints/api/image_models_pre_oci.py deleted file mode 100644 index e29c294b0..000000000 --- a/endpoints/api/image_models_pre_oci.py +++ /dev/null @@ -1,56 +0,0 @@ -from collections import defaultdict -from data import model -from endpoints.api.image_models_interface import (ImageInterface, ImageWithHistory, ImageWithTags, - Image) - -def _image(namespace_name, repo_name, image, all_images, include_parents=True): - parent_image_tuples = [] - if include_parents: - parent_images = [all_images[ancestor_id] for ancestor_id in image.ancestor_id_list() - if all_images.get(ancestor_id)] - parent_image_tuples = [_image(namespace_name, repo_name, parent_image, all_images, False) - for parent_image in parent_images] - - return Image(image.docker_image_id, image.created, image.comment, image.command, - image.storage.image_size, image.storage.uploading, parent_image_tuples) - -def _tag_names(image, tags_by_docker_id): - return [tag.name for tag in tags_by_docker_id.get(image.docker_image_id, [])] - - -class PreOCIModel(ImageInterface): - """ - PreOCIModel implements the data model for the Image API using a database schema - before it was changed to support the OCI specification. - """ - def get_repository_images(self, namespace_name, repo_name): - repo = model.repository.get_repository(namespace_name, repo_name) - if not repo: - return None - - all_images = model.image.get_repository_images_without_placements(repo) - all_images_map = {image.id: image for image in all_images} - - all_tags = list(model.tag.list_repository_tags(namespace_name, repo_name)) - - tags_by_docker_id = defaultdict(list) - for tag in all_tags: - tags_by_docker_id[tag.image.docker_image_id].append(tag) - - def _build_image(image): - image_itself = _image(namespace_name, repo_name, image, all_images_map) - return ImageWithTags(image_itself, tag_names=_tag_names(image, tags_by_docker_id)) - - return [_build_image(image) for image in all_images] - - def get_repository_image(self, namespace_name, repo_name, docker_image_id): - image = model.image.get_repo_image_and_storage(namespace_name, repo_name, docker_image_id) - if not image: - return None - - parent_images = model.image.get_parent_images(namespace_name, repo_name, image) - all_images_map = {image.id: image for image in parent_images} - return ImageWithHistory(_image(namespace_name, repo_name, image, all_images_map)) - - -pre_oci_model = PreOCIModel() diff --git a/endpoints/api/manifest.py b/endpoints/api/manifest.py index b1546c8c5..b8fe7d61f 100644 --- a/endpoints/api/manifest.py +++ b/endpoints/api/manifest.py @@ -1,23 +1,43 @@ """ Manage the manifests of a repository. """ -import json +from flask import request from app import label_validator -from flask import request +from data.model import InvalidLabelKeyException, InvalidMediaTypeException +from data.registry_model import registry_model +from digest import digest_tools from endpoints.api import (resource, nickname, require_repo_read, require_repo_write, RepositoryParamResource, log_action, validate_json_request, path_param, parse_args, query_param, abort, api, disallow_for_app_repositories) +from endpoints.api.image import image_dict from endpoints.exception import NotFound -from manifest_models_pre_oci import pre_oci_model as model -from data.model import InvalidLabelKeyException, InvalidMediaTypeException - -from digest import digest_tools from util.validation import VALID_LABEL_KEY_REGEX + BASE_MANIFEST_ROUTE = '/v1/repository//manifest/' MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN) ALLOWED_LABEL_MEDIA_TYPES = ['text/plain', 'application/json'] +def _label_dict(label): + return { + 'id': label.uuid, + 'key': label.key, + 'value': label.value, + 'source_type': label.source_type_name, + 'media_type': label.media_type_name, + } + +def _manifest_dict(manifest): + image = None + if manifest.legacy_image is not None: + image = image_dict(manifest.legacy_image, with_history=True) + + return { + 'digest': manifest.digest, + 'manifest_data': manifest.manifest_bytes, + 'image': image, + } + @resource(MANIFEST_DIGEST_ROUTE) @path_param('repository', 'The full path of the repository. e.g. namespace/name') @@ -28,11 +48,16 @@ class RepositoryManifest(RepositoryParamResource): @nickname('getRepoManifest') @disallow_for_app_repositories def get(self, namespace_name, repository_name, manifestref): - manifest = model.get_repository_manifest(namespace_name, repository_name, manifestref) + repo_ref = registry_model.lookup_repository(namespace_name, repository_name) + if repo_ref is None: + raise NotFound() + + manifest = registry_model.lookup_manifest_by_digest(repo_ref, manifestref, + include_legacy_image=True) if manifest is None: raise NotFound() - return manifest.to_dict() + return _manifest_dict(manifest) @resource(MANIFEST_DIGEST_ROUTE + '/labels') @@ -74,11 +99,20 @@ class RepositoryManifestLabels(RepositoryParamResource): @query_param('filter', 'If specified, only labels matching the given prefix will be returned', type=str, default=None) def get(self, namespace_name, repository_name, manifestref, parsed_args): - labels = model.get_manifest_labels(namespace_name, repository_name, manifestref, filter=parsed_args['filter']) + repo_ref = registry_model.lookup_repository(namespace_name, repository_name) + if repo_ref is None: + raise NotFound() + + manifest = registry_model.lookup_manifest_by_digest(repo_ref, manifestref) + if manifest is None: + raise NotFound() + + labels = registry_model.list_manifest_labels(manifest, parsed_args['filter']) if labels is None: raise NotFound() + return { - 'labels': [label.to_dict() for label in labels] + 'labels': [_label_dict(label) for label in labels] } @require_repo_write @@ -93,24 +127,32 @@ class RepositoryManifestLabels(RepositoryParamResource): if label_validator.has_reserved_prefix(label_data['key']): abort(400, message='Label has a reserved prefix') + repo_ref = registry_model.lookup_repository(namespace_name, repository_name) + if repo_ref is None: + raise NotFound() + + manifest = registry_model.lookup_manifest_by_digest(repo_ref, manifestref) + if manifest is None: + raise NotFound() + label = None try: - label = model.create_manifest_label(namespace_name, - repository_name, - manifestref, - label_data['key'], - label_data['value'], - 'api', - label_data['media_type']) + label = registry_model.create_manifest_label(manifest, + label_data['key'], + label_data['value'], + 'api', + label_data['media_type']) except InvalidLabelKeyException: - abort(400, message='Label is of an invalid format or missing please use %s format for labels'.format( - VALID_LABEL_KEY_REGEX)) + message = ('Label is of an invalid format or missing please ' + + 'use %s format for labels' % VALID_LABEL_KEY_REGEX) + abort(400, message=message) except InvalidMediaTypeException: - abort(400, message='Media type is invalid please use a valid media type of text/plain or application/json') + message = 'Media type is invalid please use a valid media type: text/plain, application/json' + abort(400, message=message) if label is None: raise NotFound() - + metadata = { 'id': label.uuid, 'key': label.key, @@ -123,7 +165,7 @@ class RepositoryManifestLabels(RepositoryParamResource): log_action('manifest_label_add', namespace_name, metadata, repo_name=repository_name) - resp = {'label': label.to_dict()} + resp = {'label': _label_dict(label)} repo_string = '%s/%s' % (namespace_name, repository_name) headers = { 'Location': api.url_for(ManageRepositoryManifestLabel, repository=repo_string, @@ -143,11 +185,19 @@ class ManageRepositoryManifestLabel(RepositoryParamResource): @disallow_for_app_repositories def get(self, namespace_name, repository_name, manifestref, labelid): """ Retrieves the label with the specific ID under the manifest. """ - label = model.get_manifest_label(namespace_name, repository_name, manifestref, labelid) + repo_ref = registry_model.lookup_repository(namespace_name, repository_name) + if repo_ref is None: + raise NotFound() + + manifest = registry_model.lookup_manifest_by_digest(repo_ref, manifestref) + if manifest is None: + raise NotFound() + + label = registry_model.get_manifest_label(manifest, labelid) if label is None: raise NotFound() - return label.to_dict() + return _label_dict(label) @require_repo_write @@ -155,7 +205,15 @@ class ManageRepositoryManifestLabel(RepositoryParamResource): @disallow_for_app_repositories def delete(self, namespace_name, repository_name, manifestref, labelid): """ Deletes an existing label from a manifest. """ - deleted = model.delete_manifest_label(namespace_name, repository_name, manifestref, labelid) + repo_ref = registry_model.lookup_repository(namespace_name, repository_name) + if repo_ref is None: + raise NotFound() + + manifest = registry_model.lookup_manifest_by_digest(repo_ref, manifestref) + if manifest is None: + raise NotFound() + + deleted = registry_model.delete_manifest_label(manifest, labelid) if deleted is None: raise NotFound() @@ -170,4 +228,3 @@ class ManageRepositoryManifestLabel(RepositoryParamResource): log_action('manifest_label_delete', namespace_name, metadata, repo_name=repository_name) return '', 204 - diff --git a/endpoints/api/manifest_models_interface.py b/endpoints/api/manifest_models_interface.py deleted file mode 100644 index 03615b208..000000000 --- a/endpoints/api/manifest_models_interface.py +++ /dev/null @@ -1,117 +0,0 @@ -from abc import ABCMeta, abstractmethod -from collections import namedtuple - -from six import add_metaclass - - -class ManifestLabel( - namedtuple('ManifestLabel', [ - 'uuid', - 'key', - 'value', - 'source_type_name', - 'media_type_name', - ])): - """ - ManifestLabel represents a label on a manifest - :type uuid: string - :type key: string - :type value: string - :type source_type_name: string - :type media_type_name: string - """ - def to_dict(self): - return { - 'id': self.uuid, - 'key': self.key, - 'value': self.value, - 'source_type': self.source_type_name, - 'media_type': self.media_type_name, - } - - -class ManifestAndImage( - namedtuple('ManifestAndImage', [ - 'digest', - 'manifest_data', - 'image', - ])): - def to_dict(self): - return { - 'digest': self.digest, - 'manifest_data': self.manifest_data, - 'image': self.image.to_dict(), - } - - -@add_metaclass(ABCMeta) -class ManifestLabelInterface(object): - """ - Data interface that the manifest labels API uses - """ - - @abstractmethod - def get_manifest_labels(self, namespace_name, repository_name, manifestref, filter=None): - """ - - Args: - namespace_name: string - repository_name: string - manifestref: string - filter: string - - Returns: - list(ManifestLabel) or None - - """ - - @abstractmethod - def create_manifest_label(self, namespace_name, repository_name, manifestref, key, value, source_type_name, media_type_name): - """ - - Args: - namespace_name: string - repository_name: string - manifestref: string - key: string - value: string - source_type_name: string - media_type_name: string - - Returns: - ManifestLabel or None - """ - - @abstractmethod - def get_manifest_label(self, namespace_name, repository_name, manifestref, label_uuid): - """ - - Args: - namespace_name: string - repository_name: string - manifestref: string - label_uuid: string - - Returns: - ManifestLabel or None - """ - - @abstractmethod - def delete_manifest_label(self, namespace_name, repository_name, manifestref, label_uuid): - """ - - Args: - namespace_name: string - repository_name: string - manifestref: string - label_uuid: string - - Returns: - ManifestLabel or None - """ - - @abstractmethod - def get_repository_manifest(self, namespace_name, repository_name, digest): - """ - Returns the manifest and image for the manifest with the specified digest, if any. - """ diff --git a/endpoints/api/manifest_models_pre_oci.py b/endpoints/api/manifest_models_pre_oci.py deleted file mode 100644 index a5d3d78af..000000000 --- a/endpoints/api/manifest_models_pre_oci.py +++ /dev/null @@ -1,68 +0,0 @@ -import json - -from manifest_models_interface import ManifestLabel, ManifestLabelInterface, ManifestAndImage -from data import model -from image_models_pre_oci import pre_oci_model as image_models - - -class ManifestLabelPreOCI(ManifestLabelInterface): - def get_manifest_labels(self, namespace_name, repository_name, manifestref, filter=None): - try: - tag_manifest = model.tag.load_manifest_by_digest(namespace_name, repository_name, manifestref) - except model.DataModelException: - return None - - labels = model.label.list_manifest_labels(tag_manifest, prefix_filter=filter) - return [self._label(l) for l in labels] - - def create_manifest_label(self, namespace_name, repository_name, manifestref, key, value, source_type_name, media_type_name): - try: - tag_manifest = model.tag.load_manifest_by_digest(namespace_name, repository_name, manifestref) - except model.DataModelException: - return None - - return self._label(model.label.create_manifest_label(tag_manifest, key, value, source_type_name, media_type_name)) - - def get_manifest_label(self, namespace_name, repository_name, manifestref, label_uuid): - try: - tag_manifest = model.tag.load_manifest_by_digest(namespace_name, repository_name, manifestref) - except model.DataModelException: - return None - - return self._label(model.label.get_manifest_label(label_uuid, tag_manifest)) - - def delete_manifest_label(self, namespace_name, repository_name, manifestref, label_uuid): - try: - tag_manifest = model.tag.load_manifest_by_digest(namespace_name, repository_name, manifestref) - except model.DataModelException: - return None - - return self._label(model.label.delete_manifest_label(label_uuid, tag_manifest)) - - def get_repository_manifest(self, namespace_name, repository_name, digest): - try: - tag_manifest = model.tag.load_manifest_by_digest(namespace_name, repository_name, digest, - allow_dead=True) - except model.DataModelException: - return None - - # TODO: remove this dependency on image once we've moved to the new data model. - image = image_models.get_repository_image(namespace_name, repository_name, - tag_manifest.tag.image.docker_image_id) - - manifest_data = json.loads(tag_manifest.json_data) - return ManifestAndImage(digest=digest, manifest_data=manifest_data, image=image) - - - def _label(self, label_obj): - if not label_obj: - return None - return ManifestLabel( - uuid=label_obj.uuid, - key=label_obj.key, - value=label_obj.value, - source_type_name=label_obj.source_type.name, - media_type_name=label_obj.media_type.name, - ) - -pre_oci_model = ManifestLabelPreOCI() diff --git a/endpoints/api/repository_models_pre_oci.py b/endpoints/api/repository_models_pre_oci.py index 7c60138f3..0ef62773c 100644 --- a/endpoints/api/repository_models_pre_oci.py +++ b/endpoints/api/repository_models_pre_oci.py @@ -5,6 +5,8 @@ from datetime import datetime, timedelta from auth.permissions import ReadRepositoryPermission from data import model from data.appr_model import channel as channel_model, release as release_model +from data.registry_model import registry_model +from data.registry_model.datatypes import RepositoryReference from endpoints.appr.models_cnr import model as appr_model from endpoints.api.repository_models_interface import RepositoryDataInterface, RepositoryBaseElement, Repository, \ ApplicationRepository, ImageRepositoryRepository, Tag, Channel, Release, Count @@ -154,13 +156,16 @@ class PreOCIModel(RepositoryDataInterface): for release in releases ]) - tags = model.tag.list_active_repo_tags(repo) + repo_ref = RepositoryReference.for_repo_obj(repo) + tags = registry_model.list_repository_tags(repo_ref, include_legacy_images=True) + start_date = datetime.now() - timedelta(days=MAX_DAYS_IN_3_MONTHS) counts = model.log.get_repository_action_counts(repo, start_date) return ImageRepositoryRepository(base, [ - Tag(tag.name, tag.image.docker_image_id, tag.image.aggregate_size, tag.lifetime_start_ts, - tag.tagmanifest.digest if hasattr(tag, 'tagmanifest') else None, + Tag(tag.name, tag.legacy_image.docker_image_id, tag.legacy_image.aggregate_size, + tag.lifetime_start_ts, + tag.manifest_digest, tag.lifetime_end_ts) for tag in tags ], [Count(count.date, count.count) for count in counts], repo.badge_token, repo.trust_enabled) diff --git a/endpoints/api/secscan.py b/endpoints/api/secscan.py index 3fd04a66d..a5ae88043 100644 --- a/endpoints/api/secscan.py +++ b/endpoints/api/secscan.py @@ -4,7 +4,8 @@ import logging import features from app import secscan_api -from data import model +from data.registry_model import registry_model +from data.registry_model.datatypes import SecurityScanStatus from endpoints.api import (require_repo_read, path_param, RepositoryParamResource, resource, nickname, show_if, parse_args, query_param, truthy_bool, disallow_for_app_repositories) @@ -15,37 +16,24 @@ from util.secscan.api import APIRequestFailure logger = logging.getLogger(__name__) - -class SCAN_STATUS(object): - """ Security scan status enum """ - SCANNED = 'scanned' - FAILED = 'failed' - QUEUED = 'queued' - - -def _get_status(repo_image): - """ Returns the SCAN_STATUS value for the given image. """ - if repo_image.security_indexed_engine is not None and repo_image.security_indexed_engine >= 0: - return SCAN_STATUS.SCANNED if repo_image.security_indexed else SCAN_STATUS.FAILED - - return SCAN_STATUS.QUEUED - -def _security_status_for_image(namespace, repository, repo_image, include_vulnerabilities=True): +def _security_info(manifest_or_legacy_image, include_vulnerabilities=True): """ Returns a dict representing the result of a call to the security status API for the given - image. + manifest or image. """ - if not repo_image.security_indexed: - logger.debug('Image %s under repository %s/%s not security indexed', - repo_image.docker_image_id, namespace, repository) + status = registry_model.get_security_status(manifest_or_legacy_image) + if status is None: + raise NotFound() + + if status != SecurityScanStatus.SCANNED: return { - 'status': _get_status(repo_image), + 'status': status.value, } try: if include_vulnerabilities: - data = secscan_api.get_layer_data(repo_image, include_vulnerabilities=True) + data = secscan_api.get_layer_data(manifest_or_legacy_image, include_vulnerabilities=True) else: - data = secscan_api.get_layer_data(repo_image, include_features=True) + data = secscan_api.get_layer_data(manifest_or_legacy_image, include_features=True) except APIRequestFailure as arf: raise DownstreamIssue(arf.message) @@ -53,7 +41,7 @@ def _security_status_for_image(namespace, repository, repo_image, include_vulner raise NotFound() return { - 'status': _get_status(repo_image), + 'status': status.value, 'data': data, } @@ -73,12 +61,16 @@ class RepositoryImageSecurity(RepositoryParamResource): default=False) def get(self, namespace, repository, imageid, parsed_args): """ Fetches the features and vulnerabilities (if any) for a repository image. """ - repo_image = model.image.get_repo_image(namespace, repository, imageid) - if repo_image is None: + repo_ref = registry_model.lookup_repository(namespace, repository) + if repo_ref is None: raise NotFound() - return _security_status_for_image(namespace, repository, repo_image, - parsed_args.vulnerabilities) + legacy_image = registry_model.get_legacy_image(repo_ref, imageid) + if legacy_image is None: + raise NotFound() + + return _security_info(legacy_image, parsed_args.vulnerabilities) + @resource(MANIFEST_DIGEST_ROUTE + '/security') @show_if(features.SECURITY_SCANNER) @@ -94,12 +86,12 @@ class RepositoryManifestSecurity(RepositoryParamResource): @query_param('vulnerabilities', 'Include vulnerabilities informations', type=truthy_bool, default=False) def get(self, namespace, repository, manifestref, parsed_args): - try: - tag_manifest = model.tag.load_manifest_by_digest(namespace, repository, manifestref) - except model.DataModelException: + repo_ref = registry_model.lookup_repository(namespace, repository) + if repo_ref is None: raise NotFound() - repo_image = tag_manifest.tag.image + manifest = registry_model.lookup_manifest_by_digest(repo_ref, manifestref, allow_dead=True) + if manifest is None: + raise NotFound() - return _security_status_for_image(namespace, repository, repo_image, - parsed_args.vulnerabilities) + return _security_info(manifest, parsed_args.vulnerabilities) diff --git a/endpoints/api/tag.py b/endpoints/api/tag.py index df5861096..b96210965 100644 --- a/endpoints/api/tag.py +++ b/endpoints/api/tag.py @@ -1,20 +1,40 @@ """ Manage the tags of a repository. """ -from datetime import datetime, timedelta +from datetime import datetime from flask import request, abort from auth.auth_context import get_authenticated_user -from data.model import DataModelException +from data.registry_model import registry_model from endpoints.api import (resource, nickname, require_repo_read, require_repo_write, RepositoryParamResource, log_action, validate_json_request, path_param, parse_args, query_param, truthy_bool, disallow_for_app_repositories) -from endpoints.api.tag_models_interface import Repository -from endpoints.api.tag_models_pre_oci import pre_oci_model as model +from endpoints.api.image import image_dict from endpoints.exception import NotFound, InvalidRequest from endpoints.v2.manifest import _generate_and_store_manifest from util.names import TAG_ERROR, TAG_REGEX +def _tag_dict(tag): + tag_info = { + 'name': tag.name, + 'reversion': tag.reversion, + } + + if tag.lifetime_start_ts > 0: + tag_info['start_ts'] = tag.lifetime_start_ts + + if tag.lifetime_end_ts > 0: + tag_info['end_ts'] = tag.lifetime_end_ts + + if tag.manifest_digest: + tag_info['manifest_digest'] = tag.manifest_digest + + if tag.legacy_image: + tag_info['docker_image_id'] = tag.legacy_image.docker_image_id + + return tag_info + + @resource('/v1/repository//tag/') @path_param('repository', 'The full path of the repository. e.g. namespace/name') class ListRepositoryTags(RepositoryParamResource): @@ -33,17 +53,17 @@ class ListRepositoryTags(RepositoryParamResource): page = max(1, parsed_args.get('page', 1)) limit = min(100, max(1, parsed_args.get('limit', 50))) - tag_history = model.list_repository_tag_history(namespace_name=namespace, - repository_name=repository, page=page, - size=limit, specific_tag=specific_tag) - - if not tag_history: + repo_ref = registry_model.lookup_repository(namespace, repository) + if repo_ref is None: raise NotFound() + history, has_more = registry_model.list_repository_tag_history(repo_ref, page=page, + size=limit, + specific_tag_name=specific_tag) return { - 'tags': [tag.to_dict() for tag in tag_history.tags], + 'tags': [_tag_dict(tag) for tag in history], 'page': page, - 'has_additional': tag_history.more, + 'has_additional': has_more, } @@ -75,59 +95,67 @@ class RepositoryTag(RepositoryParamResource): @validate_json_request('ChangeTag') def put(self, namespace, repository, tag): """ Change which image a tag points to or create a new tag.""" - if not TAG_REGEX.match(tag): abort(400, TAG_ERROR) - repo = model.get_repo(namespace, repository) - if not repo: + repo_ref = registry_model.lookup_repository(namespace, repository) + if repo_ref is None: raise NotFound() if 'expiration' in request.get_json(): + tag_ref = registry_model.get_repo_tag(repo_ref, tag) + if tag_ref is None: + raise NotFound() + expiration = request.get_json().get('expiration') expiration_date = None if expiration is not None: - try: - expiration_date = datetime.utcfromtimestamp(float(expiration)) - except ValueError: - abort(400) + try: + expiration_date = datetime.utcfromtimestamp(float(expiration)) + except ValueError: + abort(400) - if expiration_date <= datetime.now(): - abort(400) + if expiration_date <= datetime.now(): + abort(400) - existing_end_ts, ok = model.change_repository_tag_expiration(namespace, repository, tag, - expiration_date) + existing_end_ts, ok = registry_model.change_repository_tag_expiration(tag_ref, + expiration_date) if ok: - if not (existing_end_ts is None and expiration_date is None): - log_action('change_tag_expiration', namespace, { - 'username': get_authenticated_user().username, - 'repo': repository, - 'tag': tag, - 'namespace': namespace, - 'expiration_date': expiration_date, - 'old_expiration_date': existing_end_ts - }, repo_name=repository) + if not (existing_end_ts is None and expiration_date is None): + log_action('change_tag_expiration', namespace, { + 'username': get_authenticated_user().username, + 'repo': repository, + 'tag': tag, + 'namespace': namespace, + 'expiration_date': expiration_date, + 'old_expiration_date': existing_end_ts + }, repo_name=repository) else: raise InvalidRequest('Could not update tag expiration; Tag has probably changed') if 'image' in request.get_json(): + existing_tag = registry_model.get_repo_tag(repo_ref, tag, include_legacy_image=True) + image_id = request.get_json()['image'] - image = model.get_repository_image(namespace, repository, image_id) + image = registry_model.get_legacy_image(repo_ref, image_id) if image is None: raise NotFound() - original_image_id = model.get_repo_tag_image(repo, tag) - model.create_or_update_tag(namespace, repository, tag, image_id) + if not registry_model.retarget_tag(repo_ref, tag, image): + raise InvalidRequest('Could not move tag') username = get_authenticated_user().username - log_action('move_tag' if original_image_id else 'create_tag', namespace, { + + log_action('move_tag' if existing_tag else 'create_tag', namespace, { 'username': username, 'repo': repository, 'tag': tag, 'namespace': namespace, 'image': image_id, - 'original_image': original_image_id + 'original_image': existing_tag.legacy_image.docker_image_id if existing_tag else None, }, repo_name=repository) + + # TODO(jschorr): Move this into the retarget_tag call _generate_and_store_manifest(namespace, repository, tag) return 'Updated', 201 @@ -137,7 +165,11 @@ class RepositoryTag(RepositoryParamResource): @nickname('deleteFullTag') def delete(self, namespace, repository, tag): """ Delete the specified repository tag. """ - model.delete_tag(namespace, repository, tag) + repo_ref = registry_model.lookup_repository(namespace, repository) + if repo_ref is None: + raise NotFound() + + registry_model.delete_tag(repo_ref, tag) username = get_authenticated_user().username log_action('delete_tag', namespace, @@ -163,37 +195,28 @@ class RepositoryTagImages(RepositoryParamResource): type=truthy_bool, default=False) def get(self, namespace, repository, tag, parsed_args): """ List the images for the specified repository tag. """ - try: - tag_image = model.get_repo_tag_image( - Repository(namespace_name=namespace, repository_name=repository), tag) - except DataModelException: + repo_ref = registry_model.lookup_repository(namespace, repository) + if repo_ref is None: raise NotFound() - if tag_image is None: + tag_ref = registry_model.get_repo_tag(repo_ref, tag, include_legacy_image=True) + if tag_ref is None: raise NotFound() - # Find all the parent images for the tag. - parent_images = model.get_parent_images(namespace, repository, tag_image.docker_image_id) - all_images = [tag_image] + list(parent_images) - image_map = {image.docker_image_id: image for image in all_images} - skip_set = set() + image_id = tag_ref.legacy_image.docker_image_id - # Filter the images returned to those not found in the ancestry of any of the other tags in - # the repository. + all_images = None if parsed_args['owned']: - all_tags = model.list_repository_tags(namespace, repository) - for current_tag in all_tags: - if current_tag.name == tag: - continue + all_images = registry_model.get_legacy_images_owned_by_tag(tag_ref) + else: + image_with_parents = registry_model.get_legacy_image(repo_ref, image_id, include_parents=True) + if image_with_parents is None: + raise NotFound() - skip_set.add(current_tag.image.ancestor_id) - skip_set = skip_set | set(current_tag.image.ancestor_id_list) + all_images = [image_with_parents] + image_with_parents.parents return { - 'images': [ - image.to_dict(image_map) for image in all_images - if not parsed_args['owned'] or (image.ancestor_id not in skip_set) - ] + 'images': [image_dict(image) for image in all_images], } @@ -226,6 +249,9 @@ class RestoreTag(RepositoryParamResource): @validate_json_request('RestoreTag') def post(self, namespace, repository, tag): """ Restores a repository tag back to a previous image in the repository. """ + repo_ref = registry_model.lookup_repository(namespace, repository) + if repo_ref is None: + raise NotFound() # Restore the tag back to the previous image. image_id = request.get_json()['image'] @@ -239,19 +265,26 @@ class RestoreTag(RepositoryParamResource): 'tag': tag, 'image': image_id, } - repo = Repository(namespace, repository) - if manifest_digest is not None: - existing_image = model.restore_tag_to_manifest(repo, tag, manifest_digest) - else: - existing_image = model.restore_tag_to_image(repo, tag, image_id) - _generate_and_store_manifest(namespace, repository, tag) - if existing_image is not None: - log_data['original_image'] = existing_image.docker_image_id + manifest_or_legacy_image = None + if manifest_digest is not None: + manifest_or_legacy_image = registry_model.lookup_manifest_by_digest(repo_ref, manifest_digest, + allow_dead=True) + else: + manifest_or_legacy_image = registry_model.get_legacy_image(repo_ref, image_id) + + if manifest_or_legacy_image is None: + raise NotFound() + + if not registry_model.retarget_tag(repo_ref, tag, manifest_or_legacy_image, is_reversion=True): + raise InvalidRequest('Could not restore tag') + + if manifest_digest is None: + # TODO(jschorr): Move this into the retarget_tag call + _generate_and_store_manifest(namespace, repository, tag) log_action('revert_tag', namespace, log_data, repo_name=repository) return { 'image_id': image_id, - 'original_image_id': existing_image.docker_image_id if existing_image else None, } diff --git a/endpoints/api/tag_models_interface.py b/endpoints/api/tag_models_interface.py deleted file mode 100644 index 82e5c4ca2..000000000 --- a/endpoints/api/tag_models_interface.py +++ /dev/null @@ -1,187 +0,0 @@ -import json -from abc import ABCMeta, abstractmethod -from collections import namedtuple - -from six import add_metaclass - -from endpoints.api import format_date - - -class Tag( - namedtuple('Tag', [ - 'name', 'image', 'reversion', 'lifetime_start_ts', 'lifetime_end_ts', 'manifest_list', - 'docker_image_id' - ])): - """ - Tag represents a name to an image. - :type name: string - :type image: Image - :type reversion: boolean - :type lifetime_start_ts: int - :type lifetime_end_ts: int - :type manifest_list: [manifest_digest] - :type docker_image_id: string - """ - - def to_dict(self): - tag_info = { - 'name': self.name, - 'docker_image_id': self.docker_image_id, - 'reversion': self.reversion, - } - - if self.lifetime_start_ts > 0: - tag_info['start_ts'] = self.lifetime_start_ts - - if self.lifetime_end_ts > 0: - tag_info['end_ts'] = self.lifetime_end_ts - - if self.manifest_list: - tag_info['manifest_digest'] = self.manifest_list - - return tag_info - - -class RepositoryTagHistory(namedtuple('RepositoryTagHistory', ['tags', 'more'])): - """ - Tag represents a name to an image. - :type tags: [Tag] - :type more: boolean - """ - - -class Repository(namedtuple('Repository', ['namespace_name', 'repository_name'])): - """ - Repository a single quay repository - :type namespace_name: string - :type repository_name: string - """ - - -class Image( - namedtuple('Image', [ - 'docker_image_id', 'created', 'comment', 'command', 'storage_image_size', - 'storage_uploading', 'ancestor_id_list', 'ancestor_id' - ])): - """ - Image - :type docker_image_id: string - :type created: datetime - :type comment: string - :type command: string - :type storage_image_size: int - :type storage_uploading: boolean - :type ancestor_id_list: [int] - :type ancestor_id: int - """ - - def to_dict(self, image_map, include_ancestors=True): - command = self.command - - def docker_id(aid): - if aid not in image_map: - return '' - - return image_map[aid].docker_image_id - - image_data = { - 'id': self.docker_image_id, - 'created': format_date(self.created), - 'comment': self.comment, - 'command': json.loads(command) if command else None, - 'size': self.storage_image_size, - 'uploading': self.storage_uploading, - 'sort_index': len(self.ancestor_id_list), - } - - if include_ancestors: - # Calculate the ancestors string, with the DBID's replaced with the docker IDs. - ancestors = [docker_id(a) for a in self.ancestor_id_list] - image_data['ancestors'] = '/{0}/'.format('/'.join(ancestors)) - - return image_data - - -@add_metaclass(ABCMeta) -class TagDataInterface(object): - """ - Interface that represents all data store interactions required by a Tag. - """ - - @abstractmethod - def list_repository_tag_history(self, namespace_name, repository_name, page=1, size=100, - specific_tag=None): - """ - Returns a RepositoryTagHistory with a list of historic tags and whether there are more tags then returned. - """ - - @abstractmethod - def get_repo(self, namespace_name, repository_name): - """ - Returns a repository associated with the given namespace and repository name. - """ - - @abstractmethod - def get_repo_tag_image(self, repository, tag_name): - """ - Returns an image associated with the repository and tag_name - """ - - @abstractmethod - def create_or_update_tag(self, namespace_name, repository_name, tag_name, docker_image_id): - """ - Returns the repository tag if it is created. - """ - - @abstractmethod - def delete_tag(self, namespace_name, repository_name, tag_name): - """ - Returns the tag for the given namespace and repository if it was created - """ - - @abstractmethod - def get_parent_images(self, namespace, repository, tag_name): - """ - Returns a list of the parent images for the namespace, repository and tag specified. - """ - - @abstractmethod - def list_repository_tags(self, namespace_name, repository_name): - """ - Returns a list of all tags associated with namespace_nam and repository_name - """ - - @abstractmethod - def get_repository(self, namespace_name, repository_name): - """ - Returns the repository associated with the namespace_name and repository_name - """ - - @abstractmethod - def get_repository_image(self, namespace_name, repository_name, docker_image_id): - """ - Returns the repository image associated with the namespace_name, repository_name, and docker - image ID. - """ - - @abstractmethod - def restore_tag_to_manifest(self, repository_name, tag_name, manifest_digest): - """ - Returns the existing repo tag image if it exists or else returns None. - Side effects include adding the tag with associated name to the manifest_digest in the named repo. - """ - - @abstractmethod - def restore_tag_to_image(self, repository_name, tag_name, image_id): - """ - Returns the existing repo tag image if it exists or else returns None - Side effects include adding the tag with associated name to the image with the associated id in the named repo. - """ - - @abstractmethod - def change_repository_tag_expiration(self, namespace_name, repository_name, tag_name, - expiration_date): - """ Sets the expiration date of the tag under the matching repository to that given. If the - expiration date is None, then the tag will not expire. Returns a tuple of the previous - expiration timestamp in seconds (if any), and whether the operation succeeded. - """ diff --git a/endpoints/api/tag_models_pre_oci.py b/endpoints/api/tag_models_pre_oci.py deleted file mode 100644 index 72297d593..000000000 --- a/endpoints/api/tag_models_pre_oci.py +++ /dev/null @@ -1,133 +0,0 @@ -from data import model -from data.model import DataModelException, InvalidImageException -from endpoints.api.tag_models_interface import TagDataInterface, Tag, RepositoryTagHistory, Repository, Image - - -class PreOCIModel(TagDataInterface): - """ - PreOCIModel implements the data model for the Tags using a database schema - before it was changed to support the OCI specification. - """ - - def list_repository_tag_history(self, namespace_name, repository_name, page=1, size=100, - specific_tag=None): - repository = model.repository.get_repository(namespace_name, repository_name) - if repository is None: - return None - - tags, manifest_map, more = model.tag.list_repository_tag_history(repository, page, size, - specific_tag) - repository_tag_history = [] - for tag in tags: - manifest_list = None - if tag.id in manifest_map: - manifest_list = manifest_map[tag.id] - - repository_tag_history.append(convert_tag(tag, manifest_list)) - - return RepositoryTagHistory(tags=repository_tag_history, more=more) - - def get_repo(self, namespace_name, repository_name): - repo = model.repository.get_repository(namespace_name, repository_name) - if repo is None: - return None - - return Repository(repo.namespace_user, repo.name) - - def get_repo_tag_image(self, repository, tag_name): - repo = model.repository.get_repository(str(repository.namespace_name), str(repository.repository_name)) - if repo is None: - return None - - try: - image = model.tag.get_repo_tag_image(repo, tag_name) - except DataModelException: - return None - - return convert_image(image) - - def create_or_update_tag(self, namespace_name, repository_name, tag_name, docker_image_id): - return model.tag.create_or_update_tag(namespace_name, repository_name, tag_name, - docker_image_id) - - def delete_tag(self, namespace_name, repository_name, tag_name): - return model.tag.delete_tag(namespace_name, repository_name, tag_name) - - def get_parent_images(self, namespace_name, repository_name, docker_image_id): - try: - image = model.image.get_image_by_id(namespace_name, repository_name, docker_image_id) - except InvalidImageException: - return [] - - parent_tags = model.image.get_parent_images(namespace_name, repository_name, image) - return_tags = [] - for image in parent_tags: - return_tags.append(convert_image(image)) - - return return_tags - - def list_repository_tags(self, namespace_name, repository_name): - tags = model.tag.list_repository_tags(namespace_name, repository_name) - new_tags = [] - for tag in tags: - new_tags.append(convert_tag(tag)) - return new_tags - - def get_repository_image(self, namespace_name, repository_name, docker_image_id): - image = model.image.get_repo_image(namespace_name, repository_name, docker_image_id) - if image is None: - return None - - return convert_image(image) - - def get_repository(self, namespace_name, repository_name): - repo = model.repository.get_repository(namespace_name, repository_name) - if repo is None: - return None - - return Repository(namespace_name=namespace_name, repository_name=repository_name) - - def restore_tag_to_manifest(self, repository, tag_name, manifest_digest): - repo = model.repository.get_repository(repository.namespace_name, repository.repository_name) - if repo is None: - return None - - image = model.tag.restore_tag_to_manifest(repo, tag_name, manifest_digest) - if image is None: - return None - - return convert_image(image) - - def restore_tag_to_image(self, repository, tag_name, image_id): - repo = model.repository.get_repository(repository.namespace_name, repository.repository_name) - if repo is None: - return None - - image = model.tag.restore_tag_to_image(repo, tag_name, image_id) - if image is None: - return None - - return convert_image(image) - - def change_repository_tag_expiration(self, namespace_name, repository_name, tag_name, - expiration_date): - return model.tag.change_repository_tag_expiration(namespace_name, repository_name, tag_name, - expiration_date) - - -def convert_image(database_image): - return Image(docker_image_id=database_image.docker_image_id, created=database_image.created, - comment=database_image.comment, command=database_image.command, - storage_image_size=database_image.storage.image_size, - storage_uploading=database_image.storage.uploading, - ancestor_id_list=database_image.ancestor_id_list(), - ancestor_id=database_image.id) - - -def convert_tag(tag, manifest_list=None): - return Tag(name=tag.name, image=convert_image(tag.image), reversion=tag.reversion, - lifetime_start_ts=tag.lifetime_start_ts, lifetime_end_ts=tag.lifetime_end_ts, - manifest_list=manifest_list, docker_image_id=tag.image.docker_image_id) - - -pre_oci_model = PreOCIModel() diff --git a/endpoints/api/test/test_manifest.py b/endpoints/api/test/test_manifest.py index 906ee07d2..7237ac020 100644 --- a/endpoints/api/test/test_manifest.py +++ b/endpoints/api/test/test_manifest.py @@ -1,22 +1,24 @@ -import pytest - -from data import model +from data.registry_model import registry_model from endpoints.api.manifest import RepositoryManifest from endpoints.api.test.shared import conduct_api_call from endpoints.test.shared import client_with_identity + from test.fixtures import * def test_repository_manifest(client): with client_with_identity('devtable', client) as cl: - tags = model.tag.list_repository_tags('devtable', 'simple') - digests = model.tag.get_tag_manifest_digests(tags) + repo_ref = registry_model.lookup_repository('devtable', 'simple') + tags = registry_model.list_repository_tags(repo_ref) for tag in tags: - manifest = digests[tag.id] + manifest_digest = tag.manifest_digest + if manifest_digest is None: + continue + params = { 'repository': 'devtable/simple', - 'manifestref': manifest, + 'manifestref': manifest_digest, } result = conduct_api_call(cl, RepositoryManifest, 'GET', params, None, 200).json - assert result['digest'] == manifest + assert result['digest'] == manifest_digest assert result['manifest_data'] assert result['image'] diff --git a/endpoints/api/test/test_security.py b/endpoints/api/test/test_security.py index 9cf3ee1a9..2c95c184d 100644 --- a/endpoints/api/test/test_security.py +++ b/endpoints/api/test/test_security.py @@ -788,11 +788,11 @@ SECURITY_TESTS = [ (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'public/publicrepo'}, {u'image': 'WXNG'}, 'freshuser', 403), (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'public/publicrepo'}, {u'image': 'WXNG'}, 'reader', 403), (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'devtable/shared'}, {u'image': 'WXNG'}, None, 401), - (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'devtable/shared'}, {u'image': 'WXNG'}, 'devtable', 400), + (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'devtable/shared'}, {u'image': 'WXNG'}, 'devtable', 404), (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'devtable/shared'}, {u'image': 'WXNG'}, 'freshuser', 403), (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'devtable/shared'}, {u'image': 'WXNG'}, 'reader', 403), (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'buynlarge/orgrepo'}, {u'image': 'WXNG'}, None, 401), - (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'buynlarge/orgrepo'}, {u'image': 'WXNG'}, 'devtable', 400), + (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'buynlarge/orgrepo'}, {u'image': 'WXNG'}, 'devtable', 404), (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'buynlarge/orgrepo'}, {u'image': 'WXNG'}, 'freshuser', 403), (RestoreTag, 'POST', {'tag': 'HP8R', 'repository': 'buynlarge/orgrepo'}, {u'image': 'WXNG'}, 'reader', 403), diff --git a/endpoints/api/test/test_tag.py b/endpoints/api/test/test_tag.py index 316b9fbd3..f3da18972 100644 --- a/endpoints/api/test/test_tag.py +++ b/endpoints/api/test/test_tag.py @@ -1,129 +1,13 @@ -import json - import pytest -from mock import patch, Mock, MagicMock, call +from data.registry_model import registry_model -from data.model import DataModelException -from endpoints.api.tag_models_interface import RepositoryTagHistory, Tag from endpoints.api.test.shared import conduct_api_call from endpoints.test.shared import client_with_identity from endpoints.api.tag import RepositoryTag, RestoreTag, ListRepositoryTags, RepositoryTagImages -from features import FeatureNameValue - from test.fixtures import * - -@pytest.fixture() -def get_repo_image(): - def mock_callable(namespace, repository, image_id): - mock = Mock(namespace_user='devtable') - mock.name = 'simple' - img = Mock(repository=mock, docker_image_id=12) if image_id == 'image1' else None - return img - - with patch('endpoints.api.tag_models_pre_oci.model.image.get_repo_image', - side_effect=mock_callable) as mk: - yield mk - - -@pytest.fixture() -def get_repository(): - with patch('endpoints.api.tag_models_pre_oci.model.image.get_repo_image', - return_value='mock_repo') as mk: - yield mk - - -@pytest.fixture() -def get_repo_tag_image(): - def mock_get_repo_tag_image(repository, tag): - storage_mock = Mock(image_size=1234, uploading='uploading') - - def fake_ancestor_id_list(): - return [] - - if tag == 'existing-tag': - return Mock(docker_image_id='mock_docker_image_id', created=12345, comment='comment', - command='command', storage=storage_mock, ancestors=[], - ancestor_id_list=fake_ancestor_id_list) - else: - raise DataModelException('Unable to find image for tag.') - - with patch('endpoints.api.tag_models_pre_oci.model.tag.get_repo_tag_image', - side_effect=mock_get_repo_tag_image): - yield - - -@pytest.fixture() -def restore_tag_to_manifest(): - def mock_restore_tag_to_manifest(repository, tag, manifest_digest): - tag_img = Mock(docker_image_id='mock_docker_image_id') if tag == 'existing-tag' else None - return tag_img - - with patch('endpoints.api.tag_models_pre_oci.model.tag.restore_tag_to_manifest', - side_effect=mock_restore_tag_to_manifest): - yield - - -@pytest.fixture() -def restore_tag_to_image(): - def mock_restore_tag_to_image(repository, tag, image_id): - tag_img = Mock(docker_image_id='mock_docker_image_id') if tag == 'existing-tag' else None - return tag_img - - with patch('endpoints.api.tag_models_pre_oci.model.tag.restore_tag_to_image', - side_effect=mock_restore_tag_to_image): - yield - - -@pytest.fixture() -def create_or_update_tag(): - with patch('endpoints.api.tag_models_pre_oci.model.tag.create_or_update_tag') as mk: - yield mk - - -@pytest.fixture() -def generate_manifest(): - def mock_callable(namespace, repository, tag): - if tag == 'generatemanifestfail': - raise Exception('test_failure') - - with patch('endpoints.api.tag._generate_and_store_manifest', side_effect=mock_callable) as mk: - yield mk - - -@pytest.fixture() -def authd_client(client): - with client_with_identity('devtable', client) as cl: - yield cl - - -@pytest.fixture() -def list_repository_tag_history(): - def list_repository_tag_history(namespace_name, repository_name, page, size, specific_tag): - return RepositoryTagHistory(tags=[ - Tag(name='First Tag', image='image', reversion=False, lifetime_start_ts=0, lifetime_end_ts=0, - manifest_list=[], docker_image_id='first docker image id'), - Tag(name='Second Tag', image='second image', reversion=True, lifetime_start_ts=10, - lifetime_end_ts=100, manifest_list=[], docker_image_id='second docker image id') - ], more=False) - - with patch('endpoints.api.tag.model.list_repository_tag_history', - side_effect=list_repository_tag_history): - yield - - -@pytest.fixture() -def find_no_repo_tag_history(): - def list_repository_tag_history(namespace_name, repository_name, page, size, specific_tag): - return None - - with patch('endpoints.api.tag.model.list_repository_tag_history', - side_effect=list_repository_tag_history): - yield - - @pytest.mark.parametrize('expiration_time, expected_status', [ (None, 201), ('aksdjhasd', 400), @@ -161,126 +45,50 @@ def test_change_tag_expiration(client, app): assert tag.lifetime_end_ts == updated_expiration -@pytest.mark.parametrize('test_image,test_tag,expected_status', [ - ('image1', '-INVALID-TAG-NAME', 400), - ('image1', '.INVALID-TAG-NAME', 400), - ('image1', +@pytest.mark.parametrize('image_exists,test_tag,expected_status', [ + (True, '-INVALID-TAG-NAME', 400), + (True, '.INVALID-TAG-NAME', 400), + (True, 'INVALID-TAG_NAME-BECAUSE-THIS-IS-WAY-WAY-TOO-LOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOONG', 400), - ('nonexistantimage', 'newtag', 404), - ('image1', 'generatemanifestfail', None), - ('image1', 'existing-tag', 201), - ('image1', 'newtag', 201), + (False, 'newtag', 404), + (True, 'generatemanifestfail', None), + (True, 'latest', 201), + (True, 'newtag', 201), ]) -def test_move_tag(test_image, test_tag, expected_status, get_repo_image, get_repo_tag_image, - create_or_update_tag, generate_manifest, authd_client): - params = {'repository': 'devtable/simple', 'tag': test_tag} - request_body = {'image': test_image} - if expected_status is None: - with pytest.raises(Exception): - conduct_api_call(authd_client, RepositoryTag, 'put', params, request_body, expected_status) - else: - conduct_api_call(authd_client, RepositoryTag, 'put', params, request_body, expected_status) +def test_move_tag(image_exists, test_tag, expected_status, client, app): + with client_with_identity('devtable', client) as cl: + test_image = 'unknown' + if image_exists: + repo_ref = registry_model.lookup_repository('devtable', 'simple') + tag_ref = registry_model.get_repo_tag(repo_ref, 'latest', include_legacy_image=True) + assert tag_ref + + test_image = tag_ref.legacy_image.docker_image_id + + params = {'repository': 'devtable/simple', 'tag': test_tag} + request_body = {'image': test_image} + if expected_status is None: + with pytest.raises(Exception): + conduct_api_call(cl, RepositoryTag, 'put', params, request_body, expected_status) + else: + conduct_api_call(cl, RepositoryTag, 'put', params, request_body, expected_status) -@pytest.mark.parametrize( - 'namespace, repository, specific_tag, page, limit, expected_response_code, expected', [ - ('devtable', 'simple', None, 1, 10, 200, { - 'has_additional': False - }), - ('devtable', 'simple', None, 1, 10, 200, { - 'page': 1 - }), - ('devtable', 'simple', None, 1, 10, 200, { - 'tags': [{ - 'docker_image_id': 'first docker image id', - 'name': 'First Tag', - 'reversion': False - }, { - 'docker_image_id': 'second docker image id', - 'end_ts': 100, - 'name': 'Second Tag', - 'reversion': True, - 'start_ts': 10 - }] - }), - ]) -def test_list_repository_tags_view_is_correct(namespace, repository, specific_tag, page, limit, - list_repository_tag_history, expected_response_code, - expected, authd_client): - params = { - 'repository': namespace + '/' + repository, - 'specificTag': specific_tag, - 'page': page, - 'limit': limit - } - response = conduct_api_call(authd_client, ListRepositoryTags, 'get', params, - expected_code=expected_response_code) - compare_list_history_tags_response(expected, response.json) - - -def compare_list_history_tags_response(expected, actual): - if 'has_additional' in expected: - assert expected['has_additional'] == actual['has_additional'] - - if 'page' in expected: - assert expected['page'] == actual['page'] - - if 'tags' in expected: - assert expected['tags'] == actual['tags'] - - -def test_no_repo_tag_history(find_no_repo_tag_history, authd_client): - params = {'repository': 'devtable/simple', 'specificTag': None, 'page': 1, 'limit': 10} - conduct_api_call(authd_client, ListRepositoryTags, 'get', params, expected_code=404) - - -@pytest.mark.parametrize( - 'specific_tag, page, limit, expected_specific_tag, expected_page, expected_limit', [ - (None, None, None, None, 1, 50), - ('specific_tag', 12, 13, 'specific_tag', 12, 13), - ('specific_tag', -1, 101, 'specific_tag', 1, 100), - ('specific_tag', 0, 0, 'specific_tag', 1, 1), - ]) -def test_repo_tag_history_param_parse(specific_tag, page, limit, expected_specific_tag, - expected_page, expected_limit, authd_client): - mock = MagicMock() - mock.return_value = RepositoryTagHistory(tags=[], more=False) - - with patch('endpoints.api.tag.model.list_repository_tag_history', side_effect=mock): - params = { - 'repository': 'devtable/simple', - 'specificTag': specific_tag, - 'page': page, - 'limit': limit - } - conduct_api_call(authd_client, ListRepositoryTags, 'get', params) - - assert mock.call_args == call(namespace_name='devtable', repository_name='simple', - page=expected_page, size=expected_limit, - specific_tag=expected_specific_tag) - - -@pytest.mark.parametrize('test_manifest,test_tag,manifest_generated,expected_status', [ - (None, 'newtag', True, 200), - (None, 'generatemanifestfail', True, None), - ('manifest1', 'newtag', False, 200), +@pytest.mark.parametrize('repo_namespace, repo_name', [ + ('devtable', 'simple'), + ('devtable', 'history'), + ('devtable', 'complex'), + ('buynlarge', 'orgrepo'), ]) -def test_restore_tag(test_manifest, test_tag, manifest_generated, expected_status, get_repository, - restore_tag_to_manifest, restore_tag_to_image, generate_manifest, - authd_client): - params = {'repository': 'devtable/simple', 'tag': test_tag} - request_body = {'image': 'image1'} - if test_manifest is not None: - request_body['manifest_digest'] = test_manifest - if expected_status is None: - with pytest.raises(Exception): - conduct_api_call(authd_client, RestoreTag, 'post', params, request_body, expected_status) - else: - conduct_api_call(authd_client, RestoreTag, 'post', params, request_body, expected_status) +def test_list_repo_tags(repo_namespace, repo_name, client, app): + params = {'repository': repo_namespace + '/' + repo_name} + with client_with_identity('devtable', client) as cl: + tags = conduct_api_call(cl, ListRepositoryTags, 'get', params).json['tags'] + repo_ref = registry_model.lookup_repository(repo_namespace, repo_name) + history, _ = registry_model.list_repository_tag_history(repo_ref) + assert len(tags) == len(history) - if manifest_generated: - generate_manifest.assert_called_with('devtable', 'simple', test_tag) @pytest.mark.parametrize('repository, tag, owned, expect_images', [ ('devtable/simple', 'prod', False, True), @@ -291,7 +99,8 @@ def test_restore_tag(test_manifest, test_tag, manifest_generated, expected_statu ('devtable/complex', 'prod', False, True), ('devtable/complex', 'prod', True, True), ]) -def test_list_tag_images(repository, tag, owned, expect_images, authd_client): - params = {'repository': repository, 'tag': tag, 'owned': owned} - result = conduct_api_call(authd_client, RepositoryTagImages, 'get', params, None, 200).json - assert bool(result['images']) == expect_images +def test_list_tag_images(repository, tag, owned, expect_images, client, app): + with client_with_identity('devtable', client) as cl: + params = {'repository': repository, 'tag': tag, 'owned': owned} + result = conduct_api_call(cl, RepositoryTagImages, 'get', params, None, 200).json + assert bool(result['images']) == expect_images diff --git a/endpoints/api/test/test_tag_models_pre_oci.py b/endpoints/api/test/test_tag_models_pre_oci.py deleted file mode 100644 index 88303c17e..000000000 --- a/endpoints/api/test/test_tag_models_pre_oci.py +++ /dev/null @@ -1,215 +0,0 @@ -import pytest - -from data.model import DataModelException, InvalidImageException -from endpoints.api.tag_models_interface import RepositoryTagHistory, Tag, Repository -from mock import Mock, call - -from data import model -from endpoints.api.tag_models_pre_oci import pre_oci_model -from util.morecollections import AttrDict - -EMPTY_REPOSITORY = 'empty_repository' -EMPTY_NAMESPACE = 'empty_namespace' -BAD_REPOSITORY_NAME = 'bad_repository_name' -BAD_NAMESPACE_NAME = 'bad_namespace_name' - - -@pytest.fixture -def get_monkeypatch(monkeypatch): - return monkeypatch - - -def mock_out_get_repository(monkeypatch, namespace_name, repository_name): - def return_none(namespace_name, repository_name): - return None - - def return_repository(namespace_name, repository_name): - return 'repository' - - if namespace_name == BAD_NAMESPACE_NAME or repository_name == BAD_REPOSITORY_NAME: - return_function = return_none - else: - return_function = return_repository - - monkeypatch.setattr(model.repository, 'get_repository', return_function) - - -def get_repo_mock(monkeypatch, return_value): - def return_return_value(namespace_name, repository_name): - return return_value - - monkeypatch.setattr(model.repository, 'get_repository', return_return_value) - - -def test_get_repo_not_exists(get_monkeypatch): - namespace_name = 'namespace_name' - repository_name = 'repository_name' - get_repo_mock(get_monkeypatch, None) - repo = pre_oci_model.get_repo(namespace_name, repository_name) - - assert repo is None - - -def test_get_repo_exists(get_monkeypatch): - namespace_name = 'namespace_name' - repository_name = 'repository_name' - mock = Mock() - mock.namespace_user = namespace_name - mock.name = repository_name - mock.repository = mock - get_repo_mock(get_monkeypatch, mock) - - repo = pre_oci_model.get_repo(namespace_name, repository_name) - - assert repo is not None - assert repo.repository_name == repository_name - assert repo.namespace_name == namespace_name - - -def get_repository_mock(monkeypatch, return_value): - def return_return_value(namespace_name, repository_name, kind_filter=None): - return return_value - - monkeypatch.setattr(model.repository, 'get_repository', return_return_value) - - -def get_repo_tag_image_mock(monkeypatch, return_value): - def return_return_value(repo, tag_name, include_storage=False): - return return_value - - monkeypatch.setattr(model.tag, 'get_repo_tag_image', return_return_value) - - -def test_get_repo_tag_image_with_repo_and_repo_tag(get_monkeypatch): - mock_storage = Mock() - mock_image = Mock() - mock_image.docker_image_id = 'some docker image id' - mock_image.created = 1235 - mock_image.comment = 'some comment' - mock_image.command = 'some command' - mock_image.storage = mock_storage - mock_image.ancestors = [] - - get_repository_mock(get_monkeypatch, mock_image) - get_repo_tag_image_mock(get_monkeypatch, mock_image) - - image = pre_oci_model.get_repo_tag_image( - Repository('namespace_name', 'repository_name'), 'tag_name') - - assert image is not None - assert image.docker_image_id == 'some docker image id' - - -def test_get_repo_tag_image_without_repo(get_monkeypatch): - get_repository_mock(get_monkeypatch, None) - - image = pre_oci_model.get_repo_tag_image( - Repository('namespace_name', 'repository_name'), 'tag_name') - - assert image is None - - -def test_get_repo_tag_image_without_repo_tag_image(get_monkeypatch): - mock = Mock() - mock.docker_image_id = 'some docker image id' - get_repository_mock(get_monkeypatch, mock) - - def raise_exception(repo, tag_name, include_storage=False): - raise DataModelException() - - get_monkeypatch.setattr(model.tag, 'get_repo_tag_image', raise_exception) - - image = pre_oci_model.get_repo_tag_image( - Repository('namespace_name', 'repository_name'), 'tag_name') - - assert image is None - - -def test_create_or_update_tag(get_monkeypatch): - mock = Mock() - get_monkeypatch.setattr(model.tag, 'create_or_update_tag', mock) - - pre_oci_model.create_or_update_tag('namespace_name', 'repository_name', 'tag_name', - 'docker_image_id') - - assert mock.call_count == 1 - assert mock.call_args == call('namespace_name', 'repository_name', 'tag_name', 'docker_image_id') - - -def test_delete_tag(get_monkeypatch): - mock = Mock() - get_monkeypatch.setattr(model.tag, 'delete_tag', mock) - - pre_oci_model.delete_tag('namespace_name', 'repository_name', 'tag_name') - - assert mock.call_count == 1 - assert mock.call_args == call('namespace_name', 'repository_name', 'tag_name') - - -def test_get_parent_images_with_exception(get_monkeypatch): - mock = Mock(side_effect=InvalidImageException) - - get_monkeypatch.setattr(model.image, 'get_image_by_id', mock) - - images = pre_oci_model.get_parent_images('namespace_name', 'repository_name', 'tag_name') - assert images == [] - - -def test_get_parent_images_empty_parent_images(get_monkeypatch): - get_image_by_id_mock = Mock() - get_monkeypatch.setattr(model.image, 'get_image_by_id', get_image_by_id_mock) - - get_parent_images_mock = Mock(return_value=[]) - get_monkeypatch.setattr(model.image, 'get_parent_images', get_parent_images_mock) - - images = pre_oci_model.get_parent_images('namespace_name', 'repository_name', 'tag_name') - assert images == [] - - -def test_list_repository_tags(get_monkeypatch): - mock = Mock(return_value=[]) - - get_monkeypatch.setattr(model.tag, 'list_repository_tags', mock) - - pre_oci_model.list_repository_tags('namespace_name', 'repository_name') - - mock.assert_called_once_with('namespace_name', 'repository_name') - - -def test_get_repository(get_monkeypatch): - mock = Mock() - - get_monkeypatch.setattr(model.repository, 'get_repository', mock) - - pre_oci_model.get_repository('namespace_name', 'repository_name') - - mock.assert_called_once_with('namespace_name', 'repository_name') - - -def test_tag_to_manifest(get_monkeypatch): - repo_mock = Mock() - restore_tag_mock = Mock(return_value=None) - get_repository_mock = Mock(return_value=repo_mock) - - get_monkeypatch.setattr(model.tag, 'restore_tag_to_manifest', restore_tag_mock) - get_monkeypatch.setattr(model.repository, 'get_repository', get_repository_mock) - - pre_oci_model.restore_tag_to_manifest( - Repository('namespace', 'repository'), 'tag_name', 'manifest_digest') - - get_repository_mock.assert_called_once_with('namespace', 'repository') - restore_tag_mock.assert_called_once_with(repo_mock, 'tag_name', 'manifest_digest') - - -def test__tag_to_image(get_monkeypatch): - repo_mock = Mock() - restore_tag_mock = Mock(return_value=None) - get_repository_mock = Mock(return_value=repo_mock) - - get_monkeypatch.setattr(model.tag, 'restore_tag_to_image', restore_tag_mock) - get_monkeypatch.setattr(model.repository, 'get_repository', get_repository_mock) - - pre_oci_model.restore_tag_to_image(Repository('namespace', 'repository'), 'tag_name', 'image_id') - - get_repository_mock.assert_called_once_with('namespace', 'repository') - restore_tag_mock.assert_called_once_with(repo_mock, 'tag_name', 'image_id') diff --git a/scripts/ci b/scripts/ci index 192c48bf4..1d21607dc 100755 --- a/scripts/ci +++ b/scripts/ci @@ -69,6 +69,10 @@ registry_old() { load_image && quay_run make registry-test-old } +certs_test() { + load_image && quay_run make certs-test +} + mysql_ping() { mysqladmin --connect-timeout=2 --wait=60 --host=127.0.0.1 \ @@ -146,6 +150,10 @@ case "$1" in registry_old ;; + certs_test) + certs_test + ;; + mysql) mysql ;; diff --git a/test/test_api_usage.py b/test/test_api_usage.py index 62382f0f1..7337748f9 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -25,6 +25,7 @@ from app import app, config_provider, all_queues, dockerfile_build_queue, notifi from buildtrigger.basehandler import BuildTriggerHandler from initdb import setup_database_for_testing, finished_database_for_testing from data import database, model, appr_model +from data.registry_model import registry_model from data.appr_model.models import NEW_MODELS from data.database import RepositoryActionCount, Repository as RepositoryTable from test.helpers import assert_action_logged @@ -2142,8 +2143,9 @@ class TestDeleteRepository(ApiTestCase): self.getResponse(Repository, params=dict(repository=self.COMPLEX_REPO)) # Make sure the repository has some images and tags. - self.assertTrue(len(list(model.image.get_repository_images(ADMIN_ACCESS_USER, 'complex'))) > 0) - self.assertTrue(len(list(model.tag.list_repository_tags(ADMIN_ACCESS_USER, 'complex'))) > 0) + repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, 'complex') + self.assertTrue(len(list(registry_model.get_legacy_images(repo_ref))) > 0) + self.assertTrue(len(list(registry_model.list_repository_tags(repo_ref))) > 0) # Add some data for the repository, in addition to is already existing images and tags. repository = model.repository.get_repository(ADMIN_ACCESS_USER, 'complex') @@ -2190,16 +2192,17 @@ class TestDeleteRepository(ApiTestCase): RepositoryActionCount.create( repository=repository, date=datetime.datetime.now() - datetime.timedelta(days=5), count=6) + repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, 'complex') + tag = registry_model.get_repo_tag(repo_ref, 'prod') + manifest = registry_model.get_manifest_for_tag(tag) + # Create some labels. - pre_delete_label_count = database.Label.select().count() + registry_model.create_manifest_label(manifest, 'foo', 'bar', 'manifest') + registry_model.create_manifest_label(manifest, 'foo', 'baz', 'manifest') + registry_model.create_manifest_label(manifest, 'something', '{}', 'api', + media_type_name='application/json') - tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod') - model.label.create_manifest_label(tag_manifest, 'foo', 'bar', 'manifest') - model.label.create_manifest_label(tag_manifest, 'foo', 'baz', 'manifest') - model.label.create_manifest_label(tag_manifest, 'something', '{}', 'api', - media_type_name='application/json') - - model.label.create_manifest_label(tag_manifest, 'something', '{"some": "json"}', 'manifest') + registry_model.create_manifest_label(manifest, 'something', '{"some": "json"}', 'manifest') # Delete the repository. with check_transitive_modifications(): @@ -2208,10 +2211,6 @@ class TestDeleteRepository(ApiTestCase): # Verify the repo was deleted. self.getResponse(Repository, params=dict(repository=self.COMPLEX_REPO), expected_code=404) - # Verify the labels are gone. - post_delete_label_count = database.Label.select().count() - self.assertEquals(post_delete_label_count, pre_delete_label_count) - class TestGetRepository(ApiTestCase): PUBLIC_REPO = PUBLIC_USER + '/publicrepo' @@ -2732,14 +2731,14 @@ class TestRestoreTag(ApiTestCase): self.postResponse(RestoreTag, params=dict(repository=ADMIN_ACCESS_USER + '/history', tag='invalidtag'), - data=dict(image='invalid_image'), expected_code=400) + data=dict(image='invalid_image'), expected_code=404) def test_restoretag_invalidimage(self): self.login(ADMIN_ACCESS_USER) self.postResponse(RestoreTag, params=dict(repository=ADMIN_ACCESS_USER + '/history', tag='latest'), - data=dict(image='invalid_image'), expected_code=400) + data=dict(image='invalid_image'), expected_code=404) def test_restoretag_invalidmanifest(self): self.login(ADMIN_ACCESS_USER) @@ -2902,23 +2901,19 @@ class TestListAndDeleteTag(ApiTestCase): def test_listtagpagination(self): self.login(ADMIN_ACCESS_USER) - latest_image = model.tag.get_tag_image(ADMIN_ACCESS_USER, "complex", "prod") + repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, "simple") + latest_tag = registry_model.get_repo_tag(repo_ref, 'latest', include_legacy_image=True) - # Create 10 tags in an empty repo. - user = model.user.get_user_or_org(ADMIN_ACCESS_USER) - repo = model.repository.create_repository(ADMIN_ACCESS_USER, "empty", user) - - image = model.image.find_create_or_link_image(latest_image.docker_image_id, repo, - ADMIN_ACCESS_USER, {}, ['local_us']) - remaining_tags = set() - for i in xrange(1, 11): + # Create 8 tags in the simple repo. + remaining_tags = {'latest', 'prod'} + for i in xrange(1, 9): tag_name = "tag" + str(i) remaining_tags.add(tag_name) - model.tag.create_or_update_tag(ADMIN_ACCESS_USER, "empty", tag_name, image.docker_image_id) + registry_model.retarget_tag(repo_ref, tag_name, latest_tag.legacy_image) # Make sure we can iterate over all of them. json = self.getJsonResponse(ListRepositoryTags, params=dict( - repository=ADMIN_ACCESS_USER + '/empty', page=1, limit=5)) + repository=ADMIN_ACCESS_USER + '/simple', page=1, limit=5)) self.assertEquals(1, json['page']) self.assertEquals(5, len(json['tags'])) self.assertTrue(json['has_additional']) @@ -2928,7 +2923,7 @@ class TestListAndDeleteTag(ApiTestCase): self.assertEquals(5, len(remaining_tags)) json = self.getJsonResponse(ListRepositoryTags, params=dict( - repository=ADMIN_ACCESS_USER + '/empty', page=2, limit=5)) + repository=ADMIN_ACCESS_USER + '/simple', page=2, limit=5)) self.assertEquals(2, json['page']) self.assertEquals(5, len(json['tags'])) @@ -2939,7 +2934,7 @@ class TestListAndDeleteTag(ApiTestCase): self.assertEquals(0, len(remaining_tags)) json = self.getJsonResponse(ListRepositoryTags, params=dict( - repository=ADMIN_ACCESS_USER + '/empty', page=3, limit=5)) + repository=ADMIN_ACCESS_USER + '/simple', page=3, limit=5)) self.assertEquals(3, json['page']) self.assertEquals(0, len(json['tags'])) @@ -3984,80 +3979,6 @@ class TestSuperUserLogs(ApiTestCase): assert len(json['logs']) > 0 - - -class TestRepositoryImageSecurity(ApiTestCase): - def test_get_vulnerabilities(self): - self.login(ADMIN_ACCESS_USER) - - tag = model.tag.get_active_tag(ADMIN_ACCESS_USER, 'simple', 'latest') - layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, 'simple', 'latest') - - tag_manifest = database.TagManifest.get(tag=tag) - - # Grab the security info for the tag. It should be queued. - manifest_response = self.getJsonResponse(RepositoryManifestSecurity, params=dict( - repository=ADMIN_ACCESS_USER + '/simple', manifestref=tag_manifest.digest, - vulnerabilities='true')) - - image_response = self.getJsonResponse( - RepositoryImageSecurity, params=dict(repository=ADMIN_ACCESS_USER + '/simple', - imageid=layer.docker_image_id, vulnerabilities='true')) - - self.assertEquals(manifest_response, image_response) - self.assertEquals('queued', image_response['status']) - - # Mark the layer as indexed. - layer.security_indexed = True - layer.security_indexed_engine = app.config['SECURITY_SCANNER_ENGINE_VERSION_TARGET'] - layer.save() - - # Grab the security info again. - with fake_security_scanner() as security_scanner: - security_scanner.add_layer(security_scanner.layer_id(layer)) - - manifest_response = self.getJsonResponse(RepositoryManifestSecurity, params=dict( - repository=ADMIN_ACCESS_USER + '/simple', manifestref=tag_manifest.digest, - vulnerabilities='true')) - - image_response = self.getJsonResponse(RepositoryImageSecurity, params=dict( - repository=ADMIN_ACCESS_USER + '/simple', imageid=layer.docker_image_id, - vulnerabilities='true')) - - self.assertEquals(manifest_response, image_response) - self.assertEquals('scanned', image_response['status']) - self.assertEquals(1, image_response['data']['Layer']['IndexedByVersion']) - - def test_get_vulnerabilities_read_failover(self): - self.login(ADMIN_ACCESS_USER) - - # Get a layer and mark it as indexed. - layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, 'simple', 'latest') - layer.security_indexed = True - layer.security_indexed_engine = app.config['SECURITY_SCANNER_ENGINE_VERSION_TARGET'] - layer.save() - - with fake_security_scanner(hostname='failoverscanner') as security_scanner: - # Query the wrong security scanner URL without failover. - self.getResponse(RepositoryImageSecurity, params=dict( - repository=ADMIN_ACCESS_USER + '/simple', imageid=layer.docker_image_id, - vulnerabilities='true'), expected_code=520) - - # Set the failover URL in the global config. - with AppConfigChange({ - 'SECURITY_SCANNER_READONLY_FAILOVER_ENDPOINTS': ['https://failoverscanner'] - }): - # Configure the API to return 200 for this layer. - layer_id = security_scanner.layer_id(layer) - security_scanner.set_ok_layer_id(layer_id) - - # Call the API and succeed on failover. - self.getResponse(RepositoryImageSecurity, params=dict( - repository=ADMIN_ACCESS_USER + '/simple', imageid=layer.docker_image_id, - vulnerabilities='true'), expected_code=200) - - - class TestSuperUserTakeOwnership(ApiTestCase): def test_take_ownership_superuser(self): self.login(ADMIN_ACCESS_USER) @@ -4242,46 +4163,46 @@ class TestRepositoryManifestLabels(ApiTestCase): def test_basic_labels(self): self.login(ADMIN_ACCESS_USER) - # Find the manifest digest for the prod tag in the complex repo. - tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod') + repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, 'complex') + tag = registry_model.get_repo_tag(repo_ref, 'prod') repository = ADMIN_ACCESS_USER + '/complex' # Check the existing labels on the complex repo, which should be empty json = self.getJsonResponse( RepositoryManifestLabels, - params=dict(repository=repository, manifestref=tag_manifest.digest)) + params=dict(repository=repository, manifestref=tag.manifest_digest)) self.assertEquals(0, len(json['labels'])) self.postJsonResponse(RepositoryManifestLabels, params=dict(repository=repository, - manifestref=tag_manifest.digest), + manifestref=tag.manifest_digest), data=dict(key='bad_label', value='world', media_type='text/plain'), expected_code=400) self.postJsonResponse(RepositoryManifestLabels, params=dict(repository=repository, - manifestref=tag_manifest.digest), + manifestref=tag.manifest_digest), data=dict(key='hello', value='world', media_type='bad_media_type'), expected_code=400) # Add some labels to the manifest. with assert_action_logged('manifest_label_add'): label1 = self.postJsonResponse(RepositoryManifestLabels, params=dict( - repository=repository, manifestref=tag_manifest.digest), data=dict( + repository=repository, manifestref=tag.manifest_digest), data=dict( key='hello', value='world', media_type='text/plain'), expected_code=201) with assert_action_logged('manifest_label_add'): label2 = self.postJsonResponse(RepositoryManifestLabels, params=dict( - repository=repository, manifestref=tag_manifest.digest), data=dict( + repository=repository, manifestref=tag.manifest_digest), data=dict( key='hi', value='there', media_type='text/plain'), expected_code=201) with assert_action_logged('manifest_label_add'): label3 = self.postJsonResponse(RepositoryManifestLabels, params=dict( - repository=repository, manifestref=tag_manifest.digest), data=dict( + repository=repository, manifestref=tag.manifest_digest), data=dict( key='hello', value='someone', media_type='application/json'), expected_code=201) # Ensure we have *3* labels json = self.getJsonResponse(RepositoryManifestLabels, params=dict( - repository=repository, manifestref=tag_manifest.digest)) + repository=repository, manifestref=tag.manifest_digest)) self.assertEquals(3, len(json['labels'])) @@ -4296,73 +4217,75 @@ class TestRepositoryManifestLabels(ApiTestCase): # Ensure we can retrieve each of the labels. for label in json['labels']: label_json = self.getJsonResponse(ManageRepositoryManifestLabel, params=dict( - repository=repository, manifestref=tag_manifest.digest, labelid=label['id'])) + repository=repository, manifestref=tag.manifest_digest, labelid=label['id'])) self.assertEquals(label['id'], label_json['id']) # Delete a label. with assert_action_logged('manifest_label_delete'): self.deleteEmptyResponse(ManageRepositoryManifestLabel, params=dict( - repository=repository, manifestref=tag_manifest.digest, labelid=label1['label']['id'])) + repository=repository, manifestref=tag.manifest_digest, labelid=label1['label']['id'])) # Ensure the label is gone. json = self.getJsonResponse(RepositoryManifestLabels, params=dict( - repository=repository, manifestref=tag_manifest.digest)) + repository=repository, manifestref=tag.manifest_digest)) self.assertEquals(2, len(json['labels'])) # Check filtering. json = self.getJsonResponse(RepositoryManifestLabels, params=dict( - repository=repository, manifestref=tag_manifest.digest, filter='hello')) + repository=repository, manifestref=tag.manifest_digest, filter='hello')) self.assertEquals(1, len(json['labels'])) def test_prefixed_labels(self): self.login(ADMIN_ACCESS_USER) - # Find the manifest digest for the prod tag in the complex repo. - tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod') + repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, 'complex') + tag = registry_model.get_repo_tag(repo_ref, 'prod') repository = ADMIN_ACCESS_USER + '/complex' self.postJsonResponse(RepositoryManifestLabels, params=dict(repository=repository, - manifestref=tag_manifest.digest), + manifestref=tag.manifest_digest), data=dict(key='com.dockers.whatever', value='pants', media_type='text/plain'), expected_code=201) self.postJsonResponse(RepositoryManifestLabels, params=dict(repository=repository, - manifestref=tag_manifest.digest), + manifestref=tag.manifest_digest), data=dict(key='my.cool.prefix.for.my.label', value='value', media_type='text/plain'), expected_code=201) def test_add_invalid_media_type(self): self.login(ADMIN_ACCESS_USER) - tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod') + repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, 'complex') + tag = registry_model.get_repo_tag(repo_ref, 'prod') repository = ADMIN_ACCESS_USER + '/complex' self.postResponse(RepositoryManifestLabels, params=dict(repository=repository, - manifestref=tag_manifest.digest), + manifestref=tag.manifest_digest), data=dict(key='hello', value='world', media_type='some/invalid'), expected_code=400) def test_add_invalid_key(self): self.login(ADMIN_ACCESS_USER) - tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod') + repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, 'complex') + tag = registry_model.get_repo_tag(repo_ref, 'prod') repository = ADMIN_ACCESS_USER + '/complex' # Try to add an empty label key. self.postResponse(RepositoryManifestLabels, params=dict(repository=repository, - manifestref=tag_manifest.digest), + manifestref=tag.manifest_digest), data=dict(key='', value='world'), expected_code=400) # Try to add an invalid label key. self.postResponse(RepositoryManifestLabels, params=dict(repository=repository, - manifestref=tag_manifest.digest), + manifestref=tag.manifest_digest), data=dict(key='invalid___key', value='world'), expected_code=400) # Try to add a label key in a reserved namespace. self.postResponse(RepositoryManifestLabels, params=dict(repository=repository, - manifestref=tag_manifest.digest), + manifestref=tag.manifest_digest), data=dict(key='io.docker.whatever', value='world'), expected_code=400) diff --git a/test/test_certs_install.sh b/test/test_certs_install.sh new file mode 100755 index 000000000..e0fc80251 --- /dev/null +++ b/test/test_certs_install.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -e + +echo "> Starting certs install test" + +# Set up all locations needed for the test +QUAYPATH=${QUAYPATH:-"."} +SCRIPT_LOCATION=${SCRIPT_LOCATION:-"/quay-registry/conf/init"} + +# Parameters: (quay config dir, certifcate dir, number of certs expected). +function call_script_and_check_num_certs { + QUAYCONFIG=$1 CERTDIR=$2 ${SCRIPT_LOCATION}/certs_install.sh + if [ $? -ne 0 ]; then + echo "Failed to install $3 certs" + exit 1; + fi + + certs_found=$(ls /usr/local/share/ca-certificates | wc -l) + if [ ${certs_found} -ne "$3" ]; then + echo "Expected there to be $3 in ca-certificates, found $certs_found" + exit 1 + fi +} + +# Create a dummy cert we can test to install +echo '{"CN":"CA","key":{"algo":"rsa","size":2048}}' | cfssl gencert -initca - | cfssljson -bare test + +# Create temp dirs we can test with +WORK_DIR=`mktemp -d` +CERTS_WORKDIR=`mktemp -d` + +# deletes the temp directory +function cleanup { + rm -rf "$WORK_DIR" + rm -rf "$CERTS_WORKDIR" + rm test.pem + rm test-key.pem +} + +# register the cleanup function to be called on the EXIT signal +trap cleanup EXIT + +# Test calling with empty directory to not fail +call_script_and_check_num_certs ${WORK_DIR} ${CERTS_WORKDIR} 0 +if [ "$?" -ne 0 ]; then + echo "Failed to install certs with no files in the directory" + exit 1 +fi + +# Move an ldap cert into the temp directory and test that installation +cp test.pem ${WORK_DIR}/ldap.crt +call_script_and_check_num_certs ${WORK_DIR} ${CERTS_WORKDIR} 1 + +# Move 1 cert to extra cert dir and test +cp test.pem ${CERTS_WORKDIR}/cert1.crt +call_script_and_check_num_certs ${WORK_DIR} ${CERTS_WORKDIR} 2 + + +# Move another cert to extra cer dir and test all three exist +cp test.pem ${CERTS_WORKDIR}/cert2.crt +call_script_and_check_num_certs ${WORK_DIR} ${CERTS_WORKDIR} 3 + + +echo "> Certs install script test succeeded" +exit 0 diff --git a/util/imagetree.py b/util/imagetree.py deleted file mode 100644 index 9df68b3d1..000000000 --- a/util/imagetree.py +++ /dev/null @@ -1,99 +0,0 @@ -from collections import defaultdict - - -class ImageTreeNode(object): - """ A node in the image tree. """ - def __init__(self, image, child_map): - self.image = image - self.parent = None - self.tags = [] - - self._child_map = child_map - - @property - def children(self): - return self._child_map[self.image.id] - - def add_tag(self, tag): - self.tags.append(tag) - - -class ImageTree(object): - """ In-memory tree for easy traversal and lookup of images in a repository. """ - - def __init__(self, all_images, all_tags, base_filter=None): - self._image_map = {} - self._child_map = defaultdict(list) - - self._build(all_images, all_tags, base_filter) - - def _build(self, all_images, all_tags, base_filter=None): - # Build nodes for each of the images. - for image in all_images: - ancestors = image.ancestor_id_list() - - # Filter any unneeded images. - if base_filter is not None: - if image.id != base_filter and not base_filter in ancestors: - continue - - # Create the node for the image. - image_node = ImageTreeNode(image, self._child_map) - self._image_map[image.id] = image_node - - # Add the node to the child map for its parent image (if any). - parent_image_id = image.parent_id - if parent_image_id is not None: - self._child_map[parent_image_id].append(image_node) - - # Build the tag map. - for tag in all_tags: - image_node = self._image_map.get(tag.image.id) - if not image_node: - continue - - image_node.add_tag(tag.name) - - def find_longest_path(self, image_id, checker): - """ Returns a list of images representing the longest path that matches the given - checker function, starting from the given image_id *exclusive*. - """ - start_node = self._image_map.get(image_id) - if not start_node: - return [] - - return self._find_longest_path(start_node, checker, -1)[1:] - - def _find_longest_path(self, image_node, checker, index): - found_path = [] - - for child_node in image_node.children: - if not checker(index + 1, child_node.image): - continue - - found = self._find_longest_path(child_node, checker, index + 1) - if found and len(found) > len(found_path): - found_path = found - - return [image_node.image] + found_path - - def tag_containing_image(self, image): - """ Returns the name of the closest tag containing the given image. """ - if not image: - return None - - # Check the current image for a tag. - image_node = self._image_map.get(image.id) - if image_node is None: - return None - - if image_node.tags: - return image_node.tags[0] - - # Check any deriving images for a tag. - for child_node in image_node.children: - found = self.tag_containing_image(child_node.image) - if found is not None: - return found - - return None diff --git a/util/secscan/api.py b/util/secscan/api.py index 25ef36807..cc6e282f2 100644 --- a/util/secscan/api.py +++ b/util/secscan/api.py @@ -7,9 +7,10 @@ from urlparse import urljoin import requests -from data.database import CloseForLongOperation from data import model +from data.database import CloseForLongOperation, TagManifest, Image from data.model.storage import get_storage_locations +from data.registry_model.datatypes import Manifest, LegacyImage from util.abchelpers import nooper from util.failover import failover, FailoverException from util.secscan.validator import SecurityConfigValidator @@ -61,6 +62,12 @@ _API_METHOD_PING = 'metrics' def compute_layer_id(layer): """ Returns the ID for the layer in the security scanner. """ + # NOTE: this is temporary until we switch to Clair V3. + if isinstance(layer, Manifest): + layer = TagManifest.get(id=layer._db_id).tag.image + elif isinstance(layer, LegacyImage): + layer = Image.get(id=layer._db_id) + return '%s.%s' % (layer.docker_image_id, layer.storage.uuid) diff --git a/util/test/test_imagetree.py b/util/test/test_imagetree.py deleted file mode 100644 index 46e59b09c..000000000 --- a/util/test/test_imagetree.py +++ /dev/null @@ -1,82 +0,0 @@ -from data import model -from util.imagetree import ImageTree - -from test.fixtures import * - -NAMESPACE = 'devtable' -SIMPLE_REPO = 'simple' -COMPLEX_REPO = 'complex' - -def _get_base_image(all_images): - for image in all_images: - if image.ancestors == '/': - return image - - return None - -def test_longest_path_simple_repo(initialized_db): - all_images = list(model.image.get_repository_images(NAMESPACE, SIMPLE_REPO)) - all_tags = list(model.tag.list_repository_tags(NAMESPACE, SIMPLE_REPO)) - tree = ImageTree(all_images, all_tags) - - base_image = _get_base_image(all_images) - tag_image = all_tags[0].image - - def checker(index, image): - return True - - ancestors = tag_image.ancestors.split('/')[2:-1] # Skip the first image. - result = tree.find_longest_path(base_image.id, checker) - assert len(result) == 3 - for index in range(0, 2): - assert result[index].id == int(ancestors[index]) - - assert tree.tag_containing_image(result[-1]) == 'latest' - -def test_longest_path_complex_repo(initialized_db): - all_images = list(model.image.get_repository_images(NAMESPACE, COMPLEX_REPO)) - all_tags = list(model.tag.list_repository_tags(NAMESPACE, COMPLEX_REPO)) - tree = ImageTree(all_images, all_tags) - - base_image = _get_base_image(all_images) - - def checker(index, image): - return True - - result = tree.find_longest_path(base_image.id, checker) - assert len(result) == 5 - assert tree.tag_containing_image(result[-1]) == 'prod' - -def test_filtering(initialized_db): - all_images = list(model.image.get_repository_images(NAMESPACE, COMPLEX_REPO)) - all_tags = list(model.tag.list_repository_tags(NAMESPACE, COMPLEX_REPO)) - tree = ImageTree(all_images, all_tags, base_filter=1245) - - base_image = _get_base_image(all_images) - - def checker(index, image): - return True - - result = tree.find_longest_path(base_image.id, checker) - assert len(result) == 0 - -def test_longest_path_simple_repo_direct_lookup(initialized_db): - repository = model.repository.get_repository(NAMESPACE, SIMPLE_REPO) - all_images = list(model.image.get_repository_images(NAMESPACE, SIMPLE_REPO)) - all_tags = list(model.tag.list_repository_tags(NAMESPACE, SIMPLE_REPO)) - - base_image = _get_base_image(all_images) - - def checker(index, image): - return True - - filtered_images = model.image.get_repository_images_without_placements( - repository, - with_ancestor=base_image) - assert set([a.id for a in all_images]) == set([f.id for f in filtered_images]) - - tree = ImageTree(filtered_images, all_tags) - - result = tree.find_longest_path(base_image.id, checker) - assert len(result) == 3 - assert tree.tag_containing_image(result[-1]) == 'latest'