Implement tag functions in new registry model interface

data/model/tag.py

@@ -515,7 +515,7 @@ def restore_tag_to_manifest(repo_obj, tag_name, manifest_digest):
 
     # Change the tag manifest to point to the updated image.
     docker_image_id = tag_manifest.tag.image.docker_image_id
-    updated_tag = create_or_update_tag_for_repo(repo_obj.id, tag_name, docker_image_id,
+    updated_tag = create_or_update_tag_for_repo(repo_obj, tag_name, docker_image_id,
                                                 reversion=True)
     tag_manifest.tag = updated_tag
     tag_manifest.save()
@@ -544,8 +544,7 @@ def restore_tag_to_image(repo_obj, tag_name, docker_image_id):
     except DataModelException:
       existing_image = None
 
-    create_or_update_tag(repo_obj.namespace_user.username, repo_obj.name, tag_name,
+    create_or_update_tag_for_repo(repo_obj, tag_name, docker_image_id, reversion=True)
-                         docker_image_id, reversion=True)
     return existing_image
 
 
@@ -589,6 +588,14 @@ def get_active_tag(namespace, repo_name, tag_name):
                     .where(RepositoryTag.name == tag_name, Repository.name == repo_name,
                            Namespace.username == namespace)).get()
 
+def get_active_tag_for_repo(repo, tag_name):
+  try:
+    return _tag_alive(RepositoryTag
+                      .select()
+                      .where(RepositoryTag.name == tag_name,
+                             RepositoryTag.repository == repo)).get()
+  except RepositoryTag.DoesNotExist:
+    return None
 
 def get_possibly_expired_tag(namespace, repo_name, tag_name):
   return (RepositoryTag
@@ -641,6 +648,13 @@ def populate_manifest(repository, manifest, legacy_image, storage_ids):
     return manifest_row
 
 
+def get_tag_manifest(tag):
+  try:
+    return TagManifest.get(tag=tag)
+  except TagManifest.DoesNotExist:
+    return None
+
+
 def load_tag_manifest(namespace, repo_name, tag_name):
   try:
     return (_load_repo_manifests(namespace, repo_name)

data/registry_model/datatype.py

@@ -39,7 +39,7 @@ def requiresinput(input_name):
     @wraps(func)
     def wrapper(self, *args, **kwargs):
       if self._inputs.get(input_name) is None:
-        raise Exception('Cannot invoke function with missing input `%s`', input_name)
+        raise Exception('Cannot invoke function with missing input `%s`' % input_name)
 
       kwargs[input_name] = self._inputs[input_name]
       result = func(self, *args, **kwargs)

data/registry_model/datatypes.py

@@ -22,14 +22,29 @@ class Label(datatype('Label', ['key', 'value', 'uuid', 'source_type_name', 'medi
                  source_type_name=label.source_type.name)
 
 
-class Tag(datatype('Tag', ['name'])):
+class Tag(datatype('Tag', ['name', 'reversion', 'manifest_digest', 'lifetime_start_ts',
+                           'lifetime_end_ts'])):
   """ Tag represents a tag in a repository, which points to a manifest or image. """
   @classmethod
-  def for_repository_tag(cls, repository_tag):
+  def for_repository_tag(cls, repository_tag, manifest_digest=None, legacy_image=None):
     if repository_tag is None:
       return None
 
-    return Tag(db_id=repository_tag.id, name=repository_tag.name)
+    return Tag(db_id=repository_tag.id,
+               name=repository_tag.name,
+               reversion=repository_tag.reversion,
+               lifetime_start_ts=repository_tag.lifetime_start_ts,
+               lifetime_end_ts=repository_tag.lifetime_end_ts,
+               manifest_digest=manifest_digest,
+               inputs=dict(legacy_image=legacy_image))
+
+  @property
+  @requiresinput('legacy_image')
+  def legacy_image(self, legacy_image):
+    """ Returns the legacy Docker V1-style image for this tag. Note that this
+        will be None for tags whose manifests point to other manifests instead of images.
+    """
+    return legacy_image
 
 
 class Manifest(datatype('Manifest', ['digest', 'manifest_bytes'])):
@@ -78,7 +93,7 @@ class LegacyImage(datatype('LegacyImage', ['docker_image_id', 'created', 'commen
         not been loaded before this property is invoked.
     """
     return [LegacyImage.for_image(images_map[ancestor_id], images_map=images_map)
-            for ancestor_id in ancestor_id_list
+            for ancestor_id in reversed(ancestor_id_list)
             if images_map.get(ancestor_id)]
 
   @property

data/registry_model/interface.py

@@ -70,3 +70,51 @@ class RegistryDataInterface(object):
     """ Delete the label with the specified UUID on the manifest. Returns the label deleted
         or None if none.
     """
+
+  @abstractmethod
+  def list_repository_tags(self, repository_ref, include_legacy_images=False):
+    """
+    Returns a list of all the active tags in the repository. Note that this can be a *heavy*
+    operation on repositories with a lot of tags, and should be avoided for more targetted
+    operations wherever possible.
+    """
+
+  @abstractmethod
+  def list_repository_tag_history(self, repository_ref, page=1, size=100, specific_tag_name=None):
+    """
+    Returns the history of all tags in the repository (unless filtered). This includes tags that
+    have been made in-active due to newer versions of those tags coming into service.
+    """
+
+  @abstractmethod
+  def get_repo_tag(self, repository_ref, tag_name, include_legacy_image=False):
+    """
+    Returns the latest, *active* tag found in the repository, with the matching name
+    or None if none.
+    """
+
+  @abstractmethod
+  def retarget_tag(self, repository_ref, tag_name, manifest_or_legacy_image,
+                   is_reversion=False):
+    """
+    Creates, updates or moves a tag to a new entry in history, pointing to the manifest or
+    legacy image specified. If is_reversion is set to True, this operation is considered a
+    reversion over a previous tag move operation. Returns the updated Tag or None on error.
+    """
+
+  @abstractmethod
+  def delete_tag(self, repository_ref, tag_name):
+    """
+    Deletes the latest, *active* tag with the given name in the repository.
+    """
+
+  @abstractmethod
+  def change_repository_tag_expiration(self, tag, expiration_date):
+    """ Sets the expiration date of the tag under the matching repository to that given. If the
+        expiration date is None, then the tag will not expire. Returns a tuple of the previous
+        expiration timestamp in seconds (if any), and whether the operation succeeded.
+    """
+
+  @abstractmethod
+  def get_legacy_images_owned_by_tag(self, tag):
+    """ Returns all legacy images *solely owned and used* by the given tag. """

data/registry_model/registry_pre_oci_model.py

@@ -133,4 +133,139 @@ class PreOCIModel(RegistryDataInterface):
     """
     return Label.for_label(model.label.delete_manifest_label(label_uuid, manifest._db_id))
 
+  def list_repository_tags(self, repository_ref, include_legacy_images=False):
+    """
+    Returns a list of all the active tags in the repository. Note that this can be a *heavy*
+    operation on repositories with a lot of tags, and should be avoided for more targetted
+    operations wherever possible.
+    """
+    # NOTE: include_legacy_images isn't used here because `list_active_repo_tags` includes the
+    # information already, so we might as well just use it. However, the new model classes will
+    # *not* include it by default, so we make it a parameter now.
+    tags = model.tag.list_active_repo_tags(repository_ref._db_id)
+    return [Tag.for_repository_tag(tag,
+                                   legacy_image=LegacyImage.for_image(tag.image),
+                                   manifest_digest=(tag.tagmanifest.digest
+                                                    if hasattr(tag, 'tagmanifest')
+                                                    else None))
+            for tag in tags]
+
+  def list_repository_tag_history(self, repository_ref, page=1, size=100, specific_tag_name=None):
+    """
+    Returns the history of all tags in the repository (unless filtered). This includes tags that
+    have been made in-active due to newer versions of those tags coming into service.
+    """
+    tags, manifest_map, has_more = model.tag.list_repository_tag_history(repository_ref._db_id,
+                                                                         page, size,
+                                                                         specific_tag_name)
+    return [Tag.for_repository_tag(tag, manifest_map.get(tag.id),
+                                   legacy_image=LegacyImage.for_image(tag.image))
+            for tag in tags], has_more
+
+  def get_repo_tag(self, repository_ref, tag_name, include_legacy_image=False):
+    """
+    Returns the latest, *active* tag found in the repository, with the matching name
+    or None if none.
+    """
+    tag = model.tag.get_active_tag_for_repo(repository_ref._db_id, tag_name)
+    if tag is None:
+      return None
+
+    legacy_image = LegacyImage.for_image(tag.image) if include_legacy_image else None
+    tag_manifest = model.tag.get_tag_manifest(tag)
+    manifest_digest = tag_manifest.digest if tag_manifest else None
+    return Tag.for_repository_tag(tag, legacy_image=legacy_image, manifest_digest=manifest_digest)
+
+  def retarget_tag(self, repository_ref, tag_name, manifest_or_legacy_image,
+                   is_reversion=False):
+    """
+    Creates, updates or moves a tag to a new entry in history, pointing to the manifest or
+    legacy image specified. If is_reversion is set to True, this operation is considered a
+    reversion over a previous tag move operation. Returns the updated Tag or None on error.
+    """
+    # TODO: unify this.
+    if not is_reversion:
+      if isinstance(manifest_or_legacy_image, Manifest):
+        raise NotImplementedError('Not yet implemented')
+      else:
+        model.tag.create_or_update_tag_for_repo(repository_ref._db_id, tag_name,
+                                                manifest_or_legacy_image.docker_image_id)
+    else:
+      if isinstance(manifest_or_legacy_image, Manifest):
+        image = model.tag.restore_tag_to_manifest(repository_ref._db_id, tag_name,
+                                                  manifest_or_legacy_image.digest)
+        if image is None:
+          return None
+      else:
+        image = model.tag.restore_tag_to_image(repository_ref._db_id, tag_name,
+                                               manifest_or_legacy_image.docker_image_id)
+        if image is None:
+          return None
+
+    return self.get_repo_tag(repository_ref, tag_name, include_legacy_image=True)
+
+  def delete_tag(self, repository_ref, tag_name):
+    """
+    Deletes the latest, *active* tag with the given name in the repository.
+    """
+    repo = model.repository.lookup_repository(repository_ref._db_id)
+    if repo is None:
+      return None
+
+    deleted_tag = model.tag.delete_tag(repo.namespace_user.username, repo.name, tag_name)
+    return Tag.for_repository_tag(deleted_tag)
+
+  def change_repository_tag_expiration(self, tag, expiration_date):
+    """ Sets the expiration date of the tag under the matching repository to that given. If the
+        expiration date is None, then the tag will not expire. Returns a tuple of the previous
+        expiration timestamp in seconds (if any), and whether the operation succeeded.
+    """
+    try:
+      tag_obj = database.RepositoryTag.get(id=tag._db_id)
+    except database.RepositoryTag.DoesNotExist:
+      return (None, False)
+
+    return model.tag.change_tag_expiration(tag_obj, expiration_date)
+
+  def get_legacy_images_owned_by_tag(self, tag):
+    """ Returns all legacy images *solely owned and used* by the given tag. """
+    try:
+      tag_obj = database.RepositoryTag.get(id=tag._db_id)
+    except database.RepositoryTag.DoesNotExist:
+      return None
+
+    # Collect the IDs of all images that the tag uses.
+    tag_image_ids = set()
+    tag_image_ids.add(tag_obj.image.id)
+    tag_image_ids.update(tag_obj.image.ancestor_id_list())
+
+    # Remove any images shared by other tags.
+    for current_tag in model.tag.list_active_repo_tags(tag_obj.repository_id):
+      if current_tag == tag_obj:
+        continue
+
+      tag_image_ids.discard(current_tag.image.id)
+      tag_image_ids = tag_image_ids.difference(current_tag.image.ancestor_id_list())
+      if not tag_image_ids:
+        return []
+
+    if not tag_image_ids:
+      return []
+
+    # Load the images we need to return.
+    images = database.Image.select().where(database.Image.id << list(tag_image_ids))
+    all_image_ids = set()
+    for image in images:
+      all_image_ids.add(image.id)
+      all_image_ids.update(image.ancestor_id_list())
+
+    # Build a map of all the images and their parents.
+    images_map = {}
+    all_images = database.Image.select().where(database.Image.id << list(all_image_ids))
+    for image in all_images:
+      images_map[image.id] = image
+
+    return [LegacyImage.for_image(image, images_map=images_map) for image in images]
+
+
 pre_oci_model = PreOCIModel()

data/registry_model/test/test_pre_oci_model.py

@@ -1,3 +1,5 @@
+from datetime import datetime, timedelta
+
 import pytest
 
 from data import model
@@ -103,7 +105,7 @@ def test_legacy_images(repo_namespace, repo_name, pre_oci_model):
     # Check against the actual DB row.
     model_image = model.image.get_image(repository_ref._db_id, found_image.docker_image_id)
     assert model_image.id == found_image._db_id
-    assert ([pid for pid in model_image.ancestor_id_list()] ==
+    assert ([pid for pid in reversed(model_image.ancestor_id_list())] ==
             [p._db_id for p in found_image.parents])
 
     # Try without parents and ensure it raises an exception.
@@ -145,3 +147,149 @@ def test_manifest_labels(pre_oci_model):
   assert pre_oci_model.delete_manifest_label(found_manifest, created.uuid)
   assert pre_oci_model.get_manifest_label(found_manifest, created.uuid) is None
   assert created not in pre_oci_model.list_manifest_labels(found_manifest)
+
+
+@pytest.mark.parametrize('repo_namespace, repo_name', [
+  ('devtable', 'simple'),
+  ('devtable', 'complex'),
+  ('devtable', 'history'),
+  ('buynlarge', 'orgrepo'),
+])
+def test_repository_tags(repo_namespace, repo_name, pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name)
+  tags = pre_oci_model.list_repository_tags(repository_ref)
+  assert len(tags)
+
+  for tag in tags:
+    found_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True)
+    assert found_tag == tag
+
+    if found_tag.legacy_image is None:
+      continue
+
+    found_image = pre_oci_model.get_legacy_image(repository_ref,
+                                                 found_tag.legacy_image.docker_image_id)
+    assert found_image == found_tag.legacy_image
+
+
+def test_repository_tag_history(pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository('devtable', 'history')
+  history, has_more = pre_oci_model.list_repository_tag_history(repository_ref)
+  assert not has_more
+  assert len(history) == 2
+
+
+@pytest.mark.parametrize('repo_namespace, repo_name', [
+  ('devtable', 'simple'),
+  ('devtable', 'complex'),
+  ('devtable', 'history'),
+  ('buynlarge', 'orgrepo'),
+])
+def test_delete_tags(repo_namespace, repo_name, pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name)
+  tags = pre_oci_model.list_repository_tags(repository_ref)
+  assert len(tags)
+
+  # Save history before the deletions.
+  previous_history, _ = pre_oci_model.list_repository_tag_history(repository_ref, size=1000)
+  assert len(previous_history) >= len(tags)
+
+  # Delete every tag in the repository.
+  for tag in tags:
+    assert pre_oci_model.delete_tag(repository_ref, tag.name)
+
+    # Make sure the tag is no longer found.
+    found_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True)
+    assert found_tag is None
+
+  # Ensure all tags have been deleted.
+  tags = pre_oci_model.list_repository_tags(repository_ref)
+  assert not len(tags)
+
+  # Ensure that the tags all live in history.
+  history, _ = pre_oci_model.list_repository_tag_history(repository_ref, size=1000)
+  assert len(history) == len(previous_history)
+
+
+@pytest.mark.parametrize('use_manifest', [
+  True,
+  False,
+])
+def test_retarget_tag_history(use_manifest, pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository('devtable', 'history')
+  history, _ = pre_oci_model.list_repository_tag_history(repository_ref)
+
+  if use_manifest:
+    manifest_or_legacy_image = pre_oci_model.lookup_manifest_by_digest(repository_ref,
+                                                                       history[1].manifest_digest,
+                                                                       allow_dead=True)
+  else:
+    manifest_or_legacy_image = history[1].legacy_image
+
+  # Retarget the tag.
+  assert manifest_or_legacy_image
+  updated_tag = pre_oci_model.retarget_tag(repository_ref, 'latest', manifest_or_legacy_image,
+                                           is_reversion=True)
+
+  # Ensure the tag has changed targets.
+  if use_manifest:
+    assert updated_tag.manifest_digest == manifest_or_legacy_image.digest
+  else:
+    assert updated_tag.legacy_image == manifest_or_legacy_image
+
+  # Ensure history has been updated.
+  new_history, _ = pre_oci_model.list_repository_tag_history(repository_ref)
+  assert len(new_history) == len(history) + 1
+
+
+def test_retarget_tag(pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository('devtable', 'complex')
+  history, _ = pre_oci_model.list_repository_tag_history(repository_ref)
+
+  prod_tag = pre_oci_model.get_repo_tag(repository_ref, 'prod', include_legacy_image=True)
+
+  # Retarget the tag.
+  updated_tag = pre_oci_model.retarget_tag(repository_ref, 'latest', prod_tag.legacy_image)
+
+  # Ensure the tag has changed targets.
+  assert updated_tag.legacy_image == prod_tag.legacy_image
+
+  # Ensure history has been updated.
+  new_history, _ = pre_oci_model.list_repository_tag_history(repository_ref)
+  assert len(new_history) == len(history) + 1
+
+
+def test_change_repository_tag_expiration(pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository('devtable', 'simple')
+  tag = pre_oci_model.get_repo_tag(repository_ref, 'latest')
+  assert tag.lifetime_end_ts is None
+
+  new_datetime = datetime.utcnow() + timedelta(days=2)
+  previous, okay = pre_oci_model.change_repository_tag_expiration(tag, new_datetime)
+
+  assert okay
+  assert previous is None
+
+  tag = pre_oci_model.get_repo_tag(repository_ref, 'latest')
+  assert tag.lifetime_end_ts is not None
+
+
+@pytest.mark.parametrize('repo_namespace, repo_name, expected_non_empty', [
+  ('devtable', 'simple', []),
+  ('devtable', 'complex', ['prod', 'v2.0']),
+  ('devtable', 'history', ['latest']),
+  ('buynlarge', 'orgrepo', []),
+  ('devtable', 'gargantuan', ['v2.0', 'v3.0', 'v4.0', 'v5.0', 'v6.0']),
+])
+def test_get_legacy_images_owned_by_tag(repo_namespace, repo_name, expected_non_empty,
+                                        pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name)
+  tags = pre_oci_model.list_repository_tags(repository_ref)
+  assert len(tags)
+
+  non_empty = set()
+  for tag in tags:
+    if pre_oci_model.get_legacy_images_owned_by_tag(tag):
+      non_empty.add(tag.name)
+
+  assert non_empty == set(expected_non_empty)