diff --git a/data/model/tag.py b/data/model/tag.py index eec39adcc..2c24e71d9 100644 --- a/data/model/tag.py +++ b/data/model/tag.py @@ -515,7 +515,7 @@ def restore_tag_to_manifest(repo_obj, tag_name, manifest_digest): # Change the tag manifest to point to the updated image. docker_image_id = tag_manifest.tag.image.docker_image_id - updated_tag = create_or_update_tag_for_repo(repo_obj.id, tag_name, docker_image_id, + updated_tag = create_or_update_tag_for_repo(repo_obj, tag_name, docker_image_id, reversion=True) tag_manifest.tag = updated_tag tag_manifest.save() @@ -544,8 +544,7 @@ def restore_tag_to_image(repo_obj, tag_name, docker_image_id): except DataModelException: existing_image = None - create_or_update_tag(repo_obj.namespace_user.username, repo_obj.name, tag_name, - docker_image_id, reversion=True) + create_or_update_tag_for_repo(repo_obj, tag_name, docker_image_id, reversion=True) return existing_image @@ -589,6 +588,14 @@ def get_active_tag(namespace, repo_name, tag_name): .where(RepositoryTag.name == tag_name, Repository.name == repo_name, Namespace.username == namespace)).get() +def get_active_tag_for_repo(repo, tag_name): + try: + return _tag_alive(RepositoryTag + .select() + .where(RepositoryTag.name == tag_name, + RepositoryTag.repository == repo)).get() + except RepositoryTag.DoesNotExist: + return None def get_possibly_expired_tag(namespace, repo_name, tag_name): return (RepositoryTag @@ -641,6 +648,13 @@ def populate_manifest(repository, manifest, legacy_image, storage_ids): return manifest_row +def get_tag_manifest(tag): + try: + return TagManifest.get(tag=tag) + except TagManifest.DoesNotExist: + return None + + def load_tag_manifest(namespace, repo_name, tag_name): try: return (_load_repo_manifests(namespace, repo_name) diff --git a/data/registry_model/datatype.py b/data/registry_model/datatype.py index 0b72b5fa8..f5ea0b5ae 100644 --- a/data/registry_model/datatype.py +++ b/data/registry_model/datatype.py @@ -39,7 +39,7 @@ def requiresinput(input_name): @wraps(func) def wrapper(self, *args, **kwargs): if self._inputs.get(input_name) is None: - raise Exception('Cannot invoke function with missing input `%s`', input_name) + raise Exception('Cannot invoke function with missing input `%s`' % input_name) kwargs[input_name] = self._inputs[input_name] result = func(self, *args, **kwargs) diff --git a/data/registry_model/datatypes.py b/data/registry_model/datatypes.py index 32df615ee..dd229f6cf 100644 --- a/data/registry_model/datatypes.py +++ b/data/registry_model/datatypes.py @@ -22,14 +22,29 @@ class Label(datatype('Label', ['key', 'value', 'uuid', 'source_type_name', 'medi source_type_name=label.source_type.name) -class Tag(datatype('Tag', ['name'])): +class Tag(datatype('Tag', ['name', 'reversion', 'manifest_digest', 'lifetime_start_ts', + 'lifetime_end_ts'])): """ Tag represents a tag in a repository, which points to a manifest or image. """ @classmethod - def for_repository_tag(cls, repository_tag): + def for_repository_tag(cls, repository_tag, manifest_digest=None, legacy_image=None): if repository_tag is None: return None - return Tag(db_id=repository_tag.id, name=repository_tag.name) + return Tag(db_id=repository_tag.id, + name=repository_tag.name, + reversion=repository_tag.reversion, + lifetime_start_ts=repository_tag.lifetime_start_ts, + lifetime_end_ts=repository_tag.lifetime_end_ts, + manifest_digest=manifest_digest, + inputs=dict(legacy_image=legacy_image)) + + @property + @requiresinput('legacy_image') + def legacy_image(self, legacy_image): + """ Returns the legacy Docker V1-style image for this tag. Note that this + will be None for tags whose manifests point to other manifests instead of images. + """ + return legacy_image class Manifest(datatype('Manifest', ['digest', 'manifest_bytes'])): @@ -78,7 +93,7 @@ class LegacyImage(datatype('LegacyImage', ['docker_image_id', 'created', 'commen not been loaded before this property is invoked. """ return [LegacyImage.for_image(images_map[ancestor_id], images_map=images_map) - for ancestor_id in ancestor_id_list + for ancestor_id in reversed(ancestor_id_list) if images_map.get(ancestor_id)] @property diff --git a/data/registry_model/interface.py b/data/registry_model/interface.py index 2cd3d5c6d..bedbb02ce 100644 --- a/data/registry_model/interface.py +++ b/data/registry_model/interface.py @@ -70,3 +70,51 @@ class RegistryDataInterface(object): """ Delete the label with the specified UUID on the manifest. Returns the label deleted or None if none. """ + + @abstractmethod + def list_repository_tags(self, repository_ref, include_legacy_images=False): + """ + Returns a list of all the active tags in the repository. Note that this can be a *heavy* + operation on repositories with a lot of tags, and should be avoided for more targetted + operations wherever possible. + """ + + @abstractmethod + def list_repository_tag_history(self, repository_ref, page=1, size=100, specific_tag_name=None): + """ + Returns the history of all tags in the repository (unless filtered). This includes tags that + have been made in-active due to newer versions of those tags coming into service. + """ + + @abstractmethod + def get_repo_tag(self, repository_ref, tag_name, include_legacy_image=False): + """ + Returns the latest, *active* tag found in the repository, with the matching name + or None if none. + """ + + @abstractmethod + def retarget_tag(self, repository_ref, tag_name, manifest_or_legacy_image, + is_reversion=False): + """ + Creates, updates or moves a tag to a new entry in history, pointing to the manifest or + legacy image specified. If is_reversion is set to True, this operation is considered a + reversion over a previous tag move operation. Returns the updated Tag or None on error. + """ + + @abstractmethod + def delete_tag(self, repository_ref, tag_name): + """ + Deletes the latest, *active* tag with the given name in the repository. + """ + + @abstractmethod + def change_repository_tag_expiration(self, tag, expiration_date): + """ Sets the expiration date of the tag under the matching repository to that given. If the + expiration date is None, then the tag will not expire. Returns a tuple of the previous + expiration timestamp in seconds (if any), and whether the operation succeeded. + """ + + @abstractmethod + def get_legacy_images_owned_by_tag(self, tag): + """ Returns all legacy images *solely owned and used* by the given tag. """ diff --git a/data/registry_model/registry_pre_oci_model.py b/data/registry_model/registry_pre_oci_model.py index 0a8d7a8a6..beb9c5866 100644 --- a/data/registry_model/registry_pre_oci_model.py +++ b/data/registry_model/registry_pre_oci_model.py @@ -133,4 +133,139 @@ class PreOCIModel(RegistryDataInterface): """ return Label.for_label(model.label.delete_manifest_label(label_uuid, manifest._db_id)) + def list_repository_tags(self, repository_ref, include_legacy_images=False): + """ + Returns a list of all the active tags in the repository. Note that this can be a *heavy* + operation on repositories with a lot of tags, and should be avoided for more targetted + operations wherever possible. + """ + # NOTE: include_legacy_images isn't used here because `list_active_repo_tags` includes the + # information already, so we might as well just use it. However, the new model classes will + # *not* include it by default, so we make it a parameter now. + tags = model.tag.list_active_repo_tags(repository_ref._db_id) + return [Tag.for_repository_tag(tag, + legacy_image=LegacyImage.for_image(tag.image), + manifest_digest=(tag.tagmanifest.digest + if hasattr(tag, 'tagmanifest') + else None)) + for tag in tags] + + def list_repository_tag_history(self, repository_ref, page=1, size=100, specific_tag_name=None): + """ + Returns the history of all tags in the repository (unless filtered). This includes tags that + have been made in-active due to newer versions of those tags coming into service. + """ + tags, manifest_map, has_more = model.tag.list_repository_tag_history(repository_ref._db_id, + page, size, + specific_tag_name) + return [Tag.for_repository_tag(tag, manifest_map.get(tag.id), + legacy_image=LegacyImage.for_image(tag.image)) + for tag in tags], has_more + + def get_repo_tag(self, repository_ref, tag_name, include_legacy_image=False): + """ + Returns the latest, *active* tag found in the repository, with the matching name + or None if none. + """ + tag = model.tag.get_active_tag_for_repo(repository_ref._db_id, tag_name) + if tag is None: + return None + + legacy_image = LegacyImage.for_image(tag.image) if include_legacy_image else None + tag_manifest = model.tag.get_tag_manifest(tag) + manifest_digest = tag_manifest.digest if tag_manifest else None + return Tag.for_repository_tag(tag, legacy_image=legacy_image, manifest_digest=manifest_digest) + + def retarget_tag(self, repository_ref, tag_name, manifest_or_legacy_image, + is_reversion=False): + """ + Creates, updates or moves a tag to a new entry in history, pointing to the manifest or + legacy image specified. If is_reversion is set to True, this operation is considered a + reversion over a previous tag move operation. Returns the updated Tag or None on error. + """ + # TODO: unify this. + if not is_reversion: + if isinstance(manifest_or_legacy_image, Manifest): + raise NotImplementedError('Not yet implemented') + else: + model.tag.create_or_update_tag_for_repo(repository_ref._db_id, tag_name, + manifest_or_legacy_image.docker_image_id) + else: + if isinstance(manifest_or_legacy_image, Manifest): + image = model.tag.restore_tag_to_manifest(repository_ref._db_id, tag_name, + manifest_or_legacy_image.digest) + if image is None: + return None + else: + image = model.tag.restore_tag_to_image(repository_ref._db_id, tag_name, + manifest_or_legacy_image.docker_image_id) + if image is None: + return None + + return self.get_repo_tag(repository_ref, tag_name, include_legacy_image=True) + + def delete_tag(self, repository_ref, tag_name): + """ + Deletes the latest, *active* tag with the given name in the repository. + """ + repo = model.repository.lookup_repository(repository_ref._db_id) + if repo is None: + return None + + deleted_tag = model.tag.delete_tag(repo.namespace_user.username, repo.name, tag_name) + return Tag.for_repository_tag(deleted_tag) + + def change_repository_tag_expiration(self, tag, expiration_date): + """ Sets the expiration date of the tag under the matching repository to that given. If the + expiration date is None, then the tag will not expire. Returns a tuple of the previous + expiration timestamp in seconds (if any), and whether the operation succeeded. + """ + try: + tag_obj = database.RepositoryTag.get(id=tag._db_id) + except database.RepositoryTag.DoesNotExist: + return (None, False) + + return model.tag.change_tag_expiration(tag_obj, expiration_date) + + def get_legacy_images_owned_by_tag(self, tag): + """ Returns all legacy images *solely owned and used* by the given tag. """ + try: + tag_obj = database.RepositoryTag.get(id=tag._db_id) + except database.RepositoryTag.DoesNotExist: + return None + + # Collect the IDs of all images that the tag uses. + tag_image_ids = set() + tag_image_ids.add(tag_obj.image.id) + tag_image_ids.update(tag_obj.image.ancestor_id_list()) + + # Remove any images shared by other tags. + for current_tag in model.tag.list_active_repo_tags(tag_obj.repository_id): + if current_tag == tag_obj: + continue + + tag_image_ids.discard(current_tag.image.id) + tag_image_ids = tag_image_ids.difference(current_tag.image.ancestor_id_list()) + if not tag_image_ids: + return [] + + if not tag_image_ids: + return [] + + # Load the images we need to return. + images = database.Image.select().where(database.Image.id << list(tag_image_ids)) + all_image_ids = set() + for image in images: + all_image_ids.add(image.id) + all_image_ids.update(image.ancestor_id_list()) + + # Build a map of all the images and their parents. + images_map = {} + all_images = database.Image.select().where(database.Image.id << list(all_image_ids)) + for image in all_images: + images_map[image.id] = image + + return [LegacyImage.for_image(image, images_map=images_map) for image in images] + + pre_oci_model = PreOCIModel() diff --git a/data/registry_model/test/test_pre_oci_model.py b/data/registry_model/test/test_pre_oci_model.py index 41a5546ec..50c1a36f9 100644 --- a/data/registry_model/test/test_pre_oci_model.py +++ b/data/registry_model/test/test_pre_oci_model.py @@ -1,3 +1,5 @@ +from datetime import datetime, timedelta + import pytest from data import model @@ -103,7 +105,7 @@ def test_legacy_images(repo_namespace, repo_name, pre_oci_model): # Check against the actual DB row. model_image = model.image.get_image(repository_ref._db_id, found_image.docker_image_id) assert model_image.id == found_image._db_id - assert ([pid for pid in model_image.ancestor_id_list()] == + assert ([pid for pid in reversed(model_image.ancestor_id_list())] == [p._db_id for p in found_image.parents]) # Try without parents and ensure it raises an exception. @@ -145,3 +147,149 @@ def test_manifest_labels(pre_oci_model): assert pre_oci_model.delete_manifest_label(found_manifest, created.uuid) assert pre_oci_model.get_manifest_label(found_manifest, created.uuid) is None assert created not in pre_oci_model.list_manifest_labels(found_manifest) + + +@pytest.mark.parametrize('repo_namespace, repo_name', [ + ('devtable', 'simple'), + ('devtable', 'complex'), + ('devtable', 'history'), + ('buynlarge', 'orgrepo'), +]) +def test_repository_tags(repo_namespace, repo_name, pre_oci_model): + repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) + tags = pre_oci_model.list_repository_tags(repository_ref) + assert len(tags) + + for tag in tags: + found_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True) + assert found_tag == tag + + if found_tag.legacy_image is None: + continue + + found_image = pre_oci_model.get_legacy_image(repository_ref, + found_tag.legacy_image.docker_image_id) + assert found_image == found_tag.legacy_image + + +def test_repository_tag_history(pre_oci_model): + repository_ref = pre_oci_model.lookup_repository('devtable', 'history') + history, has_more = pre_oci_model.list_repository_tag_history(repository_ref) + assert not has_more + assert len(history) == 2 + + +@pytest.mark.parametrize('repo_namespace, repo_name', [ + ('devtable', 'simple'), + ('devtable', 'complex'), + ('devtable', 'history'), + ('buynlarge', 'orgrepo'), +]) +def test_delete_tags(repo_namespace, repo_name, pre_oci_model): + repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) + tags = pre_oci_model.list_repository_tags(repository_ref) + assert len(tags) + + # Save history before the deletions. + previous_history, _ = pre_oci_model.list_repository_tag_history(repository_ref, size=1000) + assert len(previous_history) >= len(tags) + + # Delete every tag in the repository. + for tag in tags: + assert pre_oci_model.delete_tag(repository_ref, tag.name) + + # Make sure the tag is no longer found. + found_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True) + assert found_tag is None + + # Ensure all tags have been deleted. + tags = pre_oci_model.list_repository_tags(repository_ref) + assert not len(tags) + + # Ensure that the tags all live in history. + history, _ = pre_oci_model.list_repository_tag_history(repository_ref, size=1000) + assert len(history) == len(previous_history) + + +@pytest.mark.parametrize('use_manifest', [ + True, + False, +]) +def test_retarget_tag_history(use_manifest, pre_oci_model): + repository_ref = pre_oci_model.lookup_repository('devtable', 'history') + history, _ = pre_oci_model.list_repository_tag_history(repository_ref) + + if use_manifest: + manifest_or_legacy_image = pre_oci_model.lookup_manifest_by_digest(repository_ref, + history[1].manifest_digest, + allow_dead=True) + else: + manifest_or_legacy_image = history[1].legacy_image + + # Retarget the tag. + assert manifest_or_legacy_image + updated_tag = pre_oci_model.retarget_tag(repository_ref, 'latest', manifest_or_legacy_image, + is_reversion=True) + + # Ensure the tag has changed targets. + if use_manifest: + assert updated_tag.manifest_digest == manifest_or_legacy_image.digest + else: + assert updated_tag.legacy_image == manifest_or_legacy_image + + # Ensure history has been updated. + new_history, _ = pre_oci_model.list_repository_tag_history(repository_ref) + assert len(new_history) == len(history) + 1 + + +def test_retarget_tag(pre_oci_model): + repository_ref = pre_oci_model.lookup_repository('devtable', 'complex') + history, _ = pre_oci_model.list_repository_tag_history(repository_ref) + + prod_tag = pre_oci_model.get_repo_tag(repository_ref, 'prod', include_legacy_image=True) + + # Retarget the tag. + updated_tag = pre_oci_model.retarget_tag(repository_ref, 'latest', prod_tag.legacy_image) + + # Ensure the tag has changed targets. + assert updated_tag.legacy_image == prod_tag.legacy_image + + # Ensure history has been updated. + new_history, _ = pre_oci_model.list_repository_tag_history(repository_ref) + assert len(new_history) == len(history) + 1 + + +def test_change_repository_tag_expiration(pre_oci_model): + repository_ref = pre_oci_model.lookup_repository('devtable', 'simple') + tag = pre_oci_model.get_repo_tag(repository_ref, 'latest') + assert tag.lifetime_end_ts is None + + new_datetime = datetime.utcnow() + timedelta(days=2) + previous, okay = pre_oci_model.change_repository_tag_expiration(tag, new_datetime) + + assert okay + assert previous is None + + tag = pre_oci_model.get_repo_tag(repository_ref, 'latest') + assert tag.lifetime_end_ts is not None + + +@pytest.mark.parametrize('repo_namespace, repo_name, expected_non_empty', [ + ('devtable', 'simple', []), + ('devtable', 'complex', ['prod', 'v2.0']), + ('devtable', 'history', ['latest']), + ('buynlarge', 'orgrepo', []), + ('devtable', 'gargantuan', ['v2.0', 'v3.0', 'v4.0', 'v5.0', 'v6.0']), +]) +def test_get_legacy_images_owned_by_tag(repo_namespace, repo_name, expected_non_empty, + pre_oci_model): + repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) + tags = pre_oci_model.list_repository_tags(repository_ref) + assert len(tags) + + non_empty = set() + for tag in tags: + if pre_oci_model.get_legacy_images_owned_by_tag(tag): + non_empty.add(tag.name) + + assert non_empty == set(expected_non_empty)