Merge pull request #3290 from quay/joseph.schorr/QUAY-1124/implement-new-data-model

Implement the new OCI-based registry data model

data/model/oci/__init__.py

@@ -0,0 +1,8 @@
+# There MUST NOT be any circular dependencies between these subsections. If there are fix it by
+# moving the minimal number of things to shared
+from data.model.oci import (
+  label,
+  manifest,
+  shared,
+  tag,
+)

data/model/oci/label.py

@@ -0,0 +1,126 @@
+import logging
+
+
+from data.model import InvalidLabelKeyException, InvalidMediaTypeException, DataModelException
+from data.database import (Label, Manifest, TagManifestLabel, MediaType, LabelSourceType,
+                           db_transaction, ManifestLabel, TagManifestLabelMap,
+                           TagManifestToManifest)
+from data.text import prefix_search
+from util.validation import validate_label_key
+from util.validation import is_json
+
+logger = logging.getLogger(__name__)
+
+def list_manifest_labels(manifest_id, prefix_filter=None):
+  """ Lists all labels found on the given manifest, with an optional filter by key prefix. """
+  query = (Label
+           .select(Label, MediaType)
+           .join(MediaType)
+           .switch(Label)
+           .join(LabelSourceType)
+           .switch(Label)
+           .join(ManifestLabel)
+           .where(ManifestLabel.manifest == manifest_id))
+
+  if prefix_filter is not None:
+    query = query.where(prefix_search(Label.key, prefix_filter))
+
+  return query
+
+
+def get_manifest_label(label_uuid, manifest):
+  """ Retrieves the manifest label on the manifest with the given UUID or None if none. """
+  try:
+    return (Label
+            .select(Label, LabelSourceType)
+            .join(LabelSourceType)
+            .where(Label.uuid == label_uuid)
+            .switch(Label)
+            .join(ManifestLabel)
+            .where(ManifestLabel.manifest == manifest)
+            .get())
+  except Label.DoesNotExist:
+    return None
+
+
+def create_manifest_label(manifest_id, key, value, source_type_name, media_type_name=None):
+  """ Creates a new manifest label on a specific tag manifest. """
+  if not key:
+    raise InvalidLabelKeyException()
+
+  # Note that we don't prevent invalid label names coming from the manifest to be stored, as Docker
+  # does not currently prevent them from being put into said manifests.
+  if not validate_label_key(key) and source_type_name != 'manifest':
+    raise InvalidLabelKeyException('Key `%s` is invalid' % key)
+
+  # Find the matching media type. If none specified, we infer.
+  if media_type_name is None:
+    media_type_name = 'text/plain'
+    if is_json(value):
+      media_type_name = 'application/json'
+
+  try:
+    media_type_id = Label.media_type.get_id(media_type_name)
+  except MediaType.DoesNotExist:
+    raise InvalidMediaTypeException()
+
+  source_type_id = Label.source_type.get_id(source_type_name)
+
+  # Ensure the manifest exists.
+  try:
+    manifest = Manifest.get(id=manifest_id)
+  except Manifest.DoesNotExist:
+    return None
+
+  with db_transaction():
+    label = Label.create(key=key, value=value, source_type=source_type_id, media_type=media_type_id)
+    manifest_label = ManifestLabel.create(manifest=manifest_id, label=label,
+                                          repository=manifest.repository)
+
+    # If there exists a mapping to a TagManifest, add the old-style label.
+    # TODO(jschorr): Remove this code once the TagManifest table is gone.
+    try:
+      mapping_row = TagManifestToManifest.get(manifest=manifest)
+      tag_manifest_label = TagManifestLabel.create(annotated=mapping_row.tag_manifest, label=label,
+                                                   repository=manifest.repository)
+      TagManifestLabelMap.create(manifest_label=manifest_label,
+                                 tag_manifest_label=tag_manifest_label,
+                                 label=label,
+                                 manifest=manifest,
+                                 tag_manifest=mapping_row.tag_manifest)
+    except TagManifestToManifest.DoesNotExist:
+      pass
+
+  return label
+
+
+def delete_manifest_label(label_uuid, manifest):
+  """ Deletes the manifest label on the tag manifest with the given ID. Returns the label deleted
+      or None if none.
+  """
+  # Find the label itself.
+  label = get_manifest_label(label_uuid, manifest)
+  if label is None:
+    return None
+
+  if not label.source_type.mutable:
+    raise DataModelException('Cannot delete immutable label')
+
+  # Delete the mapping records and label.
+  # TODO(jschorr): Remove this code once the TagManifest table is gone.
+  with db_transaction():
+    (TagManifestLabelMap
+     .delete()
+     .where(TagManifestLabelMap.label == label)
+     .execute())
+
+    deleted_count = TagManifestLabel.delete().where(TagManifestLabel.label == label).execute()
+    if deleted_count != 1:
+      logger.warning('More than a single label deleted for matching label %s', label_uuid)
+
+    deleted_count = ManifestLabel.delete().where(ManifestLabel.label == label).execute()
+    if deleted_count != 1:
+      logger.warning('More than a single label deleted for matching label %s', label_uuid)
+
+    label.delete_instance(recursive=False)
+    return label

data/model/oci/manifest.py

@@ -0,0 +1,134 @@
+import logging
+
+from peewee import IntegrityError
+
+from data.database import Tag, Manifest, ManifestBlob, ManifestLegacyImage, db_transaction
+from data.model.oci.tag import filter_to_alive_tags
+from data.model.storage import lookup_repo_storages_by_content_checksum
+from data.model.image import lookup_repository_images, get_image, synthesize_v1_image
+from image.docker.schema1 import DockerSchema1Manifest, ManifestException
+
+logger = logging.getLogger(__name__)
+
+def lookup_manifest(repository_id, manifest_digest, allow_dead=False):
+  """ Returns the manifest with the specified digest under the specified repository
+      or None if none. If allow_dead is True, then manifests referenced by only
+      dead tags will also be returned.
+  """
+  query = (Manifest
+           .select()
+           .where(Manifest.repository == repository_id)
+           .where(Manifest.digest == manifest_digest))
+
+  if not allow_dead:
+    query = filter_to_alive_tags(query.join(Tag)).group_by(Manifest.id)
+
+  try:
+    return query.get()
+  except Manifest.DoesNotExist:
+    return None
+
+
+def get_or_create_manifest(repository_id, manifest_interface_instance):
+  """ Returns a tuple of the manifest in the specified repository with the matching digest
+      (if it already exists) or, if not yet created, creates and returns the manifest, as well as
+      if the manifest was created. Returns (None, None) if there was an error creating the manifest.
+      Note that *all* blobs referenced by the manifest must exist already in the repository or this
+      method will fail with a (None, None).
+  """
+  existing = lookup_manifest(repository_id, manifest_interface_instance.digest, allow_dead=True)
+  if existing is not None:
+    return existing, False
+
+  assert len(list(manifest_interface_instance.layers)) > 0
+
+  # TODO(jschorr): Switch this to supporting schema2 once we're ready.
+  assert isinstance(manifest_interface_instance, DockerSchema1Manifest)
+
+  # Ensure all the blobs in the manifest exist.
+  digests = manifest_interface_instance.checksums
+  query = lookup_repo_storages_by_content_checksum(repository_id, digests)
+  blob_map = {s.content_checksum: s for s in query}
+  for digest_str in manifest_interface_instance.blob_digests:
+    if digest_str not in blob_map:
+      logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str,
+                     manifest_interface_instance.digest, repository_id)
+      return None, None
+
+  # Determine and populate the legacy image if necessary.
+  legacy_image_id = _populate_legacy_image(repository_id, manifest_interface_instance, blob_map)
+  if legacy_image_id is None:
+    return None, None
+
+  legacy_image = get_image(repository_id, legacy_image_id)
+  if legacy_image is None:
+    return None, None
+
+  # Create the manifest and its blobs.
+  media_type = Manifest.media_type.get_id(manifest_interface_instance.content_type)
+  storage_ids = {storage.id for storage in blob_map.values()}
+
+  with db_transaction():
+    # Create the manifest.
+    try:
+      manifest = Manifest.create(repository=repository_id,
+                                 digest=manifest_interface_instance.digest,
+                                 media_type=media_type,
+                                 manifest_bytes=manifest_interface_instance.bytes)
+    except IntegrityError:
+      manifest = Manifest.get(repository=repository_id, digest=manifest_interface_instance.digest)
+      return manifest, False
+
+    # Insert the blobs.
+    blobs_to_insert = [dict(manifest=manifest, repository=repository_id,
+                            blob=storage_id) for storage_id in storage_ids]
+    if blobs_to_insert:
+      ManifestBlob.insert_many(blobs_to_insert).execute()
+
+    # Set the legacy image (if applicable).
+    ManifestLegacyImage.create(repository=repository_id, image=legacy_image, manifest=manifest)
+
+    return manifest, True
+
+
+def _populate_legacy_image(repository_id, manifest_interface_instance, blob_map):
+  # Lookup all the images and their parent images (if any) inside the manifest.
+  # This will let us know which v1 images we need to synthesize and which ones are invalid.
+  docker_image_ids = list(manifest_interface_instance.legacy_image_ids)
+  images_query = lookup_repository_images(repository_id, docker_image_ids)
+  image_storage_map = {i.docker_image_id: i.storage for i in images_query}
+
+  # Rewrite any v1 image IDs that do not match the checksum in the database.
+  try:
+    rewritten_images = manifest_interface_instance.rewrite_invalid_image_ids(image_storage_map)
+    rewritten_images = list(rewritten_images)
+    parent_image_map = {}
+
+    for rewritten_image in rewritten_images:
+      if not rewritten_image.image_id in image_storage_map:
+        parent_image = None
+        if rewritten_image.parent_image_id:
+          parent_image = parent_image_map.get(rewritten_image.parent_image_id)
+          if parent_image is None:
+            parent_image = get_image(repository_id, rewritten_image.parent_image_id)
+            if parent_image is None:
+              return None
+
+        synthesized = synthesize_v1_image(
+          repository_id,
+          blob_map[rewritten_image.content_checksum].id,
+          blob_map[rewritten_image.content_checksum].image_size,
+          rewritten_image.image_id,
+          rewritten_image.created,
+          rewritten_image.comment,
+          rewritten_image.command,
+          rewritten_image.compat_json,
+          parent_image,
+        )
+
+        parent_image_map[rewritten_image.image_id] = synthesized
+  except ManifestException:
+    logger.exception("exception when rewriting v1 metadata")
+    return None
+
+  return rewritten_images[-1].image_id

data/model/oci/shared.py

@@ -0,0 +1,24 @@
+from data.database import Manifest, ManifestLegacyImage, Image
+
+def get_legacy_image_for_manifest(manifest_id):
+  """ Returns the legacy image associated with the given manifest, if any, or None if none. """
+  try:
+    query = (ManifestLegacyImage
+             .select(ManifestLegacyImage, Image)
+             .join(Image)
+             .where(ManifestLegacyImage.manifest == manifest_id))
+    return query.get().image
+  except ManifestLegacyImage.DoesNotExist:
+    return None
+
+
+def get_manifest_for_legacy_image(image_id):
+  """ Returns a manifest that is associated with the given image, if any, or None if none. """
+  try:
+    query = (ManifestLegacyImage
+             .select(ManifestLegacyImage, Manifest)
+             .join(Manifest)
+             .where(ManifestLegacyImage.image == image_id))
+    return query.get().manifest
+  except ManifestLegacyImage.DoesNotExist:
+    return None

data/model/oci/tag.py

@@ -0,0 +1,372 @@
+import logging
+
+from calendar import timegm
+
+from data.database import (Tag, Manifest, ManifestLegacyImage, Image, ImageStorage,
+                           MediaType, RepositoryTag, TagManifest, TagManifestToManifest,
+                           get_epoch_timestamp_ms, db_transaction)
+from data.database import TagToRepositoryTag, RepositoryTag, db_for_update
+from data.model.oci.shared import get_legacy_image_for_manifest
+from data.model import config
+from image.docker.schema1 import (DOCKER_SCHEMA1_CONTENT_TYPES, DockerSchema1Manifest,
+                                  MalformedSchema1Manifest)
+from util.timedeltastring import convert_to_timedelta
+
+logger = logging.getLogger(__name__)
+
+
+def get_tag_by_id(tag_id):
+  """ Returns the tag with the given ID, joined with its manifest or None if none. """
+  try:
+    return Tag.select(Tag, Manifest).join(Manifest).where(Tag.id == tag_id).get()
+  except Tag.DoesNotExist:
+    return None
+
+
+def get_tag(repository_id, tag_name):
+  """ Returns the alive, non-hidden tag with the given name under the specified repository or
+      None if none. The tag is returned joined with its manifest.
+  """
+  query = (Tag
+           .select(Tag, Manifest)
+           .join(Manifest)
+           .where(Tag.repository == repository_id)
+           .where(Tag.name == tag_name))
+
+  query = filter_to_visible_tags(query)
+  query = filter_to_alive_tags(query)
+
+  try:
+    return query.get()
+  except Tag.DoesNotExist:
+    return None
+
+
+def list_alive_tags(repository_id, start_pagination_id=None, limit=None):
+  """ Returns a list of all the tags alive in the specified repository, with optional limits.
+      Tag's returned are joined with their manifest.
+  """
+  query = (Tag
+           .select(Tag, Manifest)
+           .join(Manifest)
+           .where(Tag.repository == repository_id))
+
+  if start_pagination_id is not None:
+    query = query.where(Tag.id >= start_pagination_id)
+
+  if limit is not None:
+    query = query.limit(limit)
+
+  return filter_to_visible_tags(filter_to_alive_tags(query))
+
+
+def list_repository_tag_history(repository_id, page, page_size, specific_tag_name=None,
+                                active_tags_only=False):
+  """ Returns a tuple of the full set of tags found in the specified repository, including those
+      that are no longer alive (unless active_tags_only is True), and whether additional tags exist.
+      If specific_tag_name is given, the tags are further filtered by name.
+  """
+  query = (Tag
+           .select(Tag, Manifest)
+           .join(Manifest)
+           .where(Tag.repository == repository_id)
+           .order_by(Tag.lifetime_start_ms.desc(), Tag.name)
+           .limit(page_size + 1)
+           .offset(page_size * (page - 1)))
+
+  if specific_tag_name is not None:
+    query = query.where(Tag.name == specific_tag_name)
+
+  if active_tags_only:
+    query = filter_to_alive_tags(query)
+
+  query = filter_to_visible_tags(query)
+  results = list(query)
+
+  return results[0:page_size], len(results) > page_size
+
+
+def get_legacy_images_for_tags(tags):
+  """ Returns a map from tag ID to the legacy image for the tag. """
+  if not tags:
+    return {}
+
+  query = (ManifestLegacyImage
+           .select(ManifestLegacyImage, Image, ImageStorage)
+           .join(Image)
+           .join(ImageStorage)
+           .where(ManifestLegacyImage.manifest << [tag.manifest_id for tag in tags]))
+
+  by_manifest = {mli.manifest_id: mli.image for mli in query}
+  return {tag.id: by_manifest[tag.manifest_id] for tag in tags}
+
+
+def find_matching_tag(repository_id, tag_names, tag_kinds=None):
+  """ Finds an alive tag in the specified repository with one of the specified tag names and
+      returns it or None if none. Tag's returned are joined with their manifest.
+  """
+  assert repository_id
+  assert tag_names
+
+  query = (Tag
+           .select(Tag, Manifest)
+           .join(Manifest)
+           .where(Tag.repository == repository_id)
+           .where(Tag.name << tag_names))
+
+  if tag_kinds:
+    query = query.where(Tag.tag_kind << tag_kinds)
+
+  try:
+    return filter_to_visible_tags(filter_to_alive_tags(query)).get()
+  except Tag.DoesNotExist:
+    return None
+
+
+def get_most_recent_tag(repository_id):
+  """ Returns the most recently pushed alive tag in the specified repository or None if none.
+      The Tag returned is joined with its manifest.
+  """
+  assert repository_id
+
+  query = (Tag
+           .select(Tag, Manifest)
+           .join(Manifest)
+           .where(Tag.repository == repository_id)
+           .order_by(Tag.lifetime_start_ms.desc()))
+
+  try:
+    return filter_to_visible_tags(filter_to_alive_tags(query)).get()
+  except Tag.DoesNotExist:
+    return None
+
+
+def get_expired_tag(repository_id, tag_name):
+  """ Returns a tag with the given name that is expired in the repository or None if none.
+  """
+  try:
+    return (Tag
+            .select()
+            .where(Tag.name == tag_name, Tag.repository == repository_id)
+            .where(~(Tag.lifetime_end_ms >> None))
+            .where(Tag.lifetime_end_ms <= get_epoch_timestamp_ms())
+            .get())
+  except Tag.DoesNotExist:
+    return None
+
+
+def retarget_tag(tag_name, manifest_id, is_reversion=False, now_ms=None):
+  """ Creates or updates a tag with the specified name to point to the given manifest under
+      its repository. If this action is a reversion to a previous manifest, is_reversion
+      should be set to True. Returns the newly created tag row or None on error.
+  """
+  try:
+    manifest = (Manifest
+                .select(Manifest, MediaType)
+                .join(MediaType)
+                .where(Manifest.id == manifest_id)
+                .get())
+  except Manifest.DoesNotExist:
+    return None
+
+  # CHECK: Make sure that we are not mistargeting a schema 1 manifest to a tag with a different
+  # name.
+  if manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES:
+    try:
+      parsed = DockerSchema1Manifest(manifest.manifest_bytes, validate=False)
+      if parsed.tag != tag_name:
+        logger.error('Tried to re-target schema1 manifest with tag `%s` to tag `%s', parsed.tag,
+                     tag_name)
+        return None
+    except MalformedSchema1Manifest:
+      logger.exception('Could not parse schema1 manifest')
+      return None
+
+  legacy_image = get_legacy_image_for_manifest(manifest)
+  now_ms = now_ms or get_epoch_timestamp_ms()
+  now_ts = int(now_ms / 1000)
+
+  with db_transaction():
+    # Lookup an existing tag in the repository with the same name and, if present, mark it
+    # as expired.
+    existing_tag = get_tag(manifest.repository_id, tag_name)
+    if existing_tag is not None:
+      _, okay = set_tag_end_ms(existing_tag, now_ms)
+
+      # TODO: should we retry here and/or use a for-update?
+      if not okay:
+        return None
+
+    # Create a new tag pointing to the manifest with a lifetime start of now.
+    created = Tag.create(name=tag_name, repository=manifest.repository_id, lifetime_start_ms=now_ms,
+                         reversion=is_reversion, manifest=manifest,
+                         tag_kind=Tag.tag_kind.get_id('tag'))
+
+    # TODO(jschorr): Remove the linkage code once RepositoryTag is gone.
+    # If this is a schema 1 manifest, then add a TagManifest linkage to it. Otherwise, it will only
+    # be pullable via the new OCI model.
+    if manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES and legacy_image is not None:
+      old_style_tag = RepositoryTag.create(repository=manifest.repository_id, image=legacy_image,
+                                           name=tag_name, lifetime_start_ts=now_ts,
+                                           reversion=is_reversion)
+      TagToRepositoryTag.create(tag=created, repository_tag=old_style_tag,
+                                repository=manifest.repository_id)
+
+      tag_manifest = TagManifest.create(tag=old_style_tag, digest=manifest.digest,
+                                        json_data=manifest.manifest_bytes)
+      TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=manifest,
+                                   repository=manifest.repository_id)
+
+    return created
+
+
+def delete_tag(repository_id, tag_name):
+  """ Deletes the alive tag with the given name in the specified repository and returns the deleted
+      tag. If the tag did not exist, returns None.
+  """
+  tag = get_tag(repository_id, tag_name)
+  if tag is None:
+    return None
+
+  return _delete_tag(tag, get_epoch_timestamp_ms())
+
+
+def _delete_tag(tag, now_ms):
+  """ Deletes the given tag by marking it as expired. """
+  now_ts = int(now_ms / 1000)
+
+  with db_transaction():
+    updated = (Tag
+               .update(lifetime_end_ms=now_ms)
+               .where(Tag.id == tag.id, Tag.lifetime_end_ms == tag.lifetime_end_ms)
+               .execute())
+    if updated != 1:
+      return None
+
+    # TODO(jschorr): Remove the linkage code once RepositoryTag is gone.
+    try:
+      old_style_tag = (TagToRepositoryTag
+                       .select(TagToRepositoryTag, RepositoryTag)
+                       .join(RepositoryTag)
+                       .where(TagToRepositoryTag.tag == tag)
+                       .get()).repository_tag
+
+      old_style_tag.lifetime_end_ts = now_ts
+      old_style_tag.save()
+    except TagToRepositoryTag.DoesNotExist:
+      pass
+
+    return tag
+
+
+def delete_tags_for_manifest(manifest):
+  """ Deletes all tags pointing to the given manifest. Returns the list of tags 
+      deleted.
+  """
+  tags = list(Tag.select().where(Tag.manifest == manifest))
+  now_ms = get_epoch_timestamp_ms()
+
+  with db_transaction():
+    for tag in tags:
+      _delete_tag(tag, now_ms)
+
+  return tags
+
+
+def filter_to_visible_tags(query):
+  """ Adjusts the specified Tag query to only return those tags that are visible.
+  """
+  return query.where(Tag.hidden == False)
+
+
+def filter_to_alive_tags(query, now_ms=None):
+  """ Adjusts the specified Tag query to only return those tags alive. If now_ms is specified,
+      the given timestamp (in MS) is used in place of the current timestamp for determining wherther
+      a tag is alive.
+  """
+  if now_ms is None:
+    now_ms = get_epoch_timestamp_ms()
+
+  return query.where((Tag.lifetime_end_ms >> None) | (Tag.lifetime_end_ms > now_ms))
+
+
+def set_tag_expiration_sec_for_manifest(manifest_id, expiration_seconds):
+  """ Sets the tag expiration for any tags that point to the given manifest ID. """
+  query = Tag.select().where(Tag.manifest == manifest_id)
+  query = filter_to_alive_tags(query)
+  query = filter_to_visible_tags(query)
+  tags = list(query)
+  for tag in tags:
+    set_tag_end_ms(tag, tag.lifetime_start_ms + (expiration_seconds * 1000))
+
+  return tags
+
+
+def set_tag_expiration_for_manifest(manifest_id, expiration_datetime):
+  """ Sets the tag expiration for any tags that point to the given manifest ID. """
+  query = Tag.select().where(Tag.manifest == manifest_id)
+  query = filter_to_alive_tags(query)
+  query = filter_to_visible_tags(query)
+  tags = list(query)
+  for tag in tags:
+    change_tag_expiration(tag, expiration_datetime)
+
+  return tags
+
+
+def change_tag_expiration(tag_id, expiration_datetime):
+  """ Changes the expiration of the specified tag to the given expiration datetime. If
+      the expiration datetime is None, then the tag is marked as not expiring. Returns
+      a tuple of the previous expiration timestamp in seconds (if any), and whether the
+      operation succeeded.
+  """
+  try:
+    tag = Tag.get(id=tag_id)
+  except Tag.DoesNotExist:
+    return (None, False)
+
+  new_end_ms = None
+  min_expire_sec = convert_to_timedelta(config.app_config.get('LABELED_EXPIRATION_MINIMUM', '1h'))
+  max_expire_sec = convert_to_timedelta(config.app_config.get('LABELED_EXPIRATION_MAXIMUM', '104w'))
+
+  if expiration_datetime is not None:
+    lifetime_start_ts = int(tag.lifetime_start_ms / 1000)
+
+    offset = timegm(expiration_datetime.utctimetuple()) - lifetime_start_ts
+    offset = min(max(offset, min_expire_sec.total_seconds()), max_expire_sec.total_seconds())
+    new_end_ms = tag.lifetime_start_ms + (offset * 1000)
+
+  if new_end_ms == tag.lifetime_end_ms:
+    return (None, True)
+
+  return set_tag_end_ms(tag, new_end_ms)
+
+
+def set_tag_end_ms(tag, end_ms):
+  """ Sets the end timestamp for a tag. Should only be called by change_tag_expiration
+      or tests.
+  """
+
+  with db_transaction():
+    updated = (Tag
+               .update(lifetime_end_ms=end_ms)
+               .where(Tag.id == tag)
+               .where(Tag.lifetime_end_ms == tag.lifetime_end_ms)
+               .execute())
+    if updated != 1:
+      return (None, False)
+
+    # TODO(jschorr): Remove the linkage code once RepositoryTag is gone.
+    try:
+      old_style_tag = (TagToRepositoryTag
+                       .select(TagToRepositoryTag, RepositoryTag)
+                       .join(RepositoryTag)
+                       .where(TagToRepositoryTag.tag == tag)
+                       .get()).repository_tag
+
+      old_style_tag.lifetime_end_ts = end_ms / 1000
+      old_style_tag.save()
+    except TagToRepositoryTag.DoesNotExist:
+      pass
+
+    return (tag.lifetime_end_ms, True)

data/model/oci/test/test_oci_label.py

@@ -0,0 +1,87 @@
+import pytest
+
+from playhouse.test_utils import assert_query_count
+
+from data.database import Manifest, ManifestLabel
+from data.model.oci.label import (create_manifest_label, list_manifest_labels, get_manifest_label,
+                                  delete_manifest_label, DataModelException)
+
+from test.fixtures import *
+
+
+@pytest.mark.parametrize('key, value, source_type, expected_error', [
+  ('foo', 'bar', 'manifest', None),
+
+  pytest.param('..foo', 'bar', 'manifest', None, id='invalid key on manifest'),
+  pytest.param('..foo', 'bar', 'api', 'is invalid', id='invalid key on api'),
+])
+def test_create_manifest_label(key, value, source_type, expected_error, initialized_db):
+  manifest = Manifest.get()
+
+  if expected_error:
+    with pytest.raises(DataModelException) as ex:
+      create_manifest_label(manifest, key, value, source_type)
+
+    assert ex.match(expected_error)
+    return
+
+  label = create_manifest_label(manifest, key, value, source_type)
+  labels = [ml.label_id for ml in ManifestLabel.select().where(ManifestLabel.manifest == manifest)]
+  assert label.id in labels
+
+  with assert_query_count(1):
+    assert label in list_manifest_labels(manifest)
+
+  assert label not in list_manifest_labels(manifest, 'someprefix')
+  assert label in list_manifest_labels(manifest, key[0:2])
+
+  with assert_query_count(1):
+    assert get_manifest_label(label.uuid, manifest) == label
+
+
+def test_list_manifest_labels(initialized_db):
+  manifest = Manifest.get()
+
+  label1 = create_manifest_label(manifest, 'foo', '1', 'manifest')
+  label2 = create_manifest_label(manifest, 'bar', '2', 'api')
+  label3 = create_manifest_label(manifest, 'baz', '3', 'internal')
+
+  assert label1 in list_manifest_labels(manifest)
+  assert label2 in list_manifest_labels(manifest)
+  assert label3 in list_manifest_labels(manifest)
+
+  other_manifest = Manifest.select().where(Manifest.id != manifest.id).get()
+  assert label1 not in list_manifest_labels(other_manifest)
+  assert label2 not in list_manifest_labels(other_manifest)
+  assert label3 not in list_manifest_labels(other_manifest)
+
+
+def test_get_manifest_label(initialized_db):
+  found = False
+  for manifest_label in ManifestLabel.select():
+    assert (get_manifest_label(manifest_label.label.uuid, manifest_label.manifest) ==
+            manifest_label.label)
+    assert manifest_label.label in list_manifest_labels(manifest_label.manifest)
+    found = True
+
+  assert found
+
+
+def test_delete_manifest_label(initialized_db):
+  found = False
+  for manifest_label in list(ManifestLabel.select()):
+    assert (get_manifest_label(manifest_label.label.uuid, manifest_label.manifest) ==
+            manifest_label.label)
+    assert manifest_label.label in list_manifest_labels(manifest_label.manifest)
+
+    if manifest_label.label.source_type.mutable:
+      assert delete_manifest_label(manifest_label.label.uuid, manifest_label.manifest)
+      assert manifest_label.label not in list_manifest_labels(manifest_label.manifest)
+      assert get_manifest_label(manifest_label.label.uuid, manifest_label.manifest) is None
+    else:
+      with pytest.raises(DataModelException):
+        delete_manifest_label(manifest_label.label.uuid, manifest_label.manifest)
+
+    found = True
+
+  assert found

data/model/oci/test/test_oci_manifest.py

@@ -0,0 +1,83 @@
+from playhouse.test_utils import assert_query_count
+
+from app import docker_v2_signing_key
+
+from data.database import Tag, ManifestBlob, get_epoch_timestamp_ms
+from data.model.oci.manifest import lookup_manifest, get_or_create_manifest
+from data.model.oci.tag import filter_to_alive_tags, get_tag
+from data.model.oci.shared import get_legacy_image_for_manifest
+from data.model.repository import get_repository
+from image.docker.schema1 import DockerSchema1ManifestBuilder, DockerSchema1Manifest
+
+from test.fixtures import *
+
+def test_lookup_manifest(initialized_db):
+  found = False
+  for tag in filter_to_alive_tags(Tag.select()):
+    found = True
+    repo = tag.repository
+    digest = tag.manifest.digest
+    with assert_query_count(1):
+      assert lookup_manifest(repo, digest) == tag.manifest
+
+  assert found
+
+  for tag in Tag.select():
+    repo = tag.repository
+    digest = tag.manifest.digest
+    with assert_query_count(1):
+      assert lookup_manifest(repo, digest, allow_dead=True) == tag.manifest
+
+
+def test_lookup_manifest_dead_tag(initialized_db):
+  dead_tag = Tag.select().where(Tag.lifetime_end_ms <= get_epoch_timestamp_ms()).get()
+  assert dead_tag.lifetime_end_ms <= get_epoch_timestamp_ms()
+
+  assert lookup_manifest(dead_tag.repository, dead_tag.manifest.digest) is None
+  assert (lookup_manifest(dead_tag.repository, dead_tag.manifest.digest, allow_dead=True) ==
+          dead_tag.manifest)
+
+
+def test_get_or_create_manifest(initialized_db):
+  repository = get_repository('devtable', 'simple')
+
+  latest_tag = get_tag(repository, 'latest')
+  legacy_image = get_legacy_image_for_manifest(latest_tag.manifest)
+  parsed = DockerSchema1Manifest(latest_tag.manifest.manifest_bytes, validate=False)
+
+  builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag')
+  builder.add_layer(parsed.blob_digests[0], '{"id": "%s"}' % legacy_image.docker_image_id)
+  sample_manifest_instance = builder.build(docker_v2_signing_key)
+
+  # Create a new manifest.
+  created, newly_created = get_or_create_manifest(repository, sample_manifest_instance)
+  assert newly_created
+  assert created is not None
+  assert created.digest == sample_manifest_instance.digest
+  assert created.manifest_bytes == sample_manifest_instance.bytes
+
+  assert get_legacy_image_for_manifest(created) is not None
+
+  blob_digests = [mb.blob.content_checksum for mb
+                  in ManifestBlob.select().where(ManifestBlob.manifest == created)]
+  assert parsed.blob_digests[0] in blob_digests
+
+  # Retrieve it again and ensure it is the same manifest.
+  created2, newly_created2 = get_or_create_manifest(repository, sample_manifest_instance)
+  assert not newly_created2
+  assert created2 == created
+
+
+def test_get_or_create_manifest_invalid_image(initialized_db):
+  repository = get_repository('devtable', 'simple')
+
+  latest_tag = get_tag(repository, 'latest')
+  parsed = DockerSchema1Manifest(latest_tag.manifest.manifest_bytes, validate=False)
+
+  builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag')
+  builder.add_layer(parsed.blob_digests[0], '{"id": "foo", "parent": "someinvalidimageid"}')
+  sample_manifest_instance = builder.build(docker_v2_signing_key)
+
+  created, newly_created = get_or_create_manifest(repository, sample_manifest_instance)
+  assert created is None
+  assert newly_created is None

data/model/oci/test/test_oci_tag.py

@@ -0,0 +1,253 @@
+from calendar import timegm
+from datetime import timedelta, datetime
+
+from playhouse.test_utils import assert_query_count
+
+from data.database import (Tag, ManifestLegacyImage, TagToRepositoryTag, TagManifestToManifest,
+                           TagManifest, Manifest)
+from data.model.oci.tag import (find_matching_tag, get_most_recent_tag, list_alive_tags,
+                                get_legacy_images_for_tags, filter_to_alive_tags,
+                                filter_to_visible_tags, list_repository_tag_history,
+                                get_expired_tag, get_tag, delete_tag,
+                                delete_tags_for_manifest, change_tag_expiration,
+                                set_tag_expiration_for_manifest, retarget_tag)
+from data.model.repository import get_repository, create_repository
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('namespace_name, repo_name, tag_names, expected', [
+  ('devtable', 'simple', ['latest'], 'latest'),
+  ('devtable', 'simple', ['unknown', 'latest'], 'latest'),
+  ('devtable', 'simple', ['unknown'], None),
+])
+def test_find_matching_tag(namespace_name, repo_name, tag_names, expected, initialized_db):
+  repo = get_repository(namespace_name, repo_name)
+  if expected is not None:
+    with assert_query_count(1):
+      found = find_matching_tag(repo, tag_names)
+
+    assert found is not None
+    assert found.name == expected
+    assert not found.lifetime_end_ms
+  else:
+    with assert_query_count(1):
+      assert find_matching_tag(repo, tag_names) is None
+
+
+def test_get_most_recent_tag(initialized_db):
+  repo = get_repository('outsideorg', 'coolrepo')
+
+  with assert_query_count(1):
+    assert get_most_recent_tag(repo).name == 'latest'
+
+
+def test_get_most_recent_tag_empty_repo(initialized_db):
+  empty_repo = create_repository('devtable', 'empty', None)
+
+  with assert_query_count(1):
+    assert get_most_recent_tag(empty_repo) is None
+
+
+def test_list_alive_tags(initialized_db):
+  found = False
+  for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())):
+    tags = list_alive_tags(tag.repository)
+    assert tag in tags
+
+    with assert_query_count(1):
+      legacy_images = get_legacy_images_for_tags(tags)
+
+    for tag in tags:
+      assert ManifestLegacyImage.get(manifest=tag.manifest).image == legacy_images[tag.id]
+
+    found = True
+
+  assert found
+
+  # Ensure hidden tags cannot be listed.
+  tag = Tag.get()
+  tag.hidden = True
+  tag.save()
+
+  tags = list_alive_tags(tag.repository)
+  assert tag not in tags
+
+
+def test_get_tag(initialized_db):
+  found = False
+  for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())):
+    repo = tag.repository
+
+    with assert_query_count(1):
+      assert get_tag(repo, tag.name) == tag
+    found = True
+
+  assert found
+
+
+@pytest.mark.parametrize('namespace_name, repo_name', [
+  ('devtable', 'simple'),
+  ('devtable', 'complex'),
+])
+def test_list_repository_tag_history(namespace_name, repo_name, initialized_db):
+  repo = get_repository(namespace_name, repo_name)
+
+  with assert_query_count(1):
+    results, has_more = list_repository_tag_history(repo, 1, 100)
+
+  assert results
+  assert not has_more
+
+
+def test_list_repository_tag_history_with_history(initialized_db):
+  repo = get_repository('devtable', 'history')
+
+  with assert_query_count(1):
+    results, _ = list_repository_tag_history(repo, 1, 100)
+
+  assert len(results) == 2
+  assert results[0].lifetime_end_ms is None
+  assert results[1].lifetime_end_ms is not None
+
+  with assert_query_count(1):
+    results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name='latest')
+
+  assert len(results) == 2
+  assert results[0].lifetime_end_ms is None
+  assert results[1].lifetime_end_ms is not None
+
+  with assert_query_count(1):
+    results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name='foobar')
+
+  assert len(results) == 0
+
+
+def test_list_repository_tag_history_all_tags(initialized_db):
+  for tag in Tag.select():
+    repo = tag.repository
+    with assert_query_count(1):
+      results, _ = list_repository_tag_history(repo, 1, 1000)
+
+    assert (tag in results) == (not tag.hidden)
+
+
+@pytest.mark.parametrize('namespace_name, repo_name, tag_name, expected', [
+  ('devtable', 'simple', 'latest', False),
+  ('devtable', 'simple', 'unknown', False),
+  ('devtable', 'complex', 'latest', False),
+
+  ('devtable', 'history', 'latest', True),
+])
+def test_get_expired_tag(namespace_name, repo_name, tag_name, expected, initialized_db):
+  repo = get_repository(namespace_name, repo_name)
+
+  with assert_query_count(1):
+    assert bool(get_expired_tag(repo, tag_name)) == expected
+
+
+def test_delete_tag(initialized_db):
+  found = False
+  for tag in list(filter_to_visible_tags(filter_to_alive_tags(Tag.select()))):
+    repo = tag.repository
+
+    assert get_tag(repo, tag.name) == tag
+    assert tag.lifetime_end_ms is None
+
+    with assert_query_count(4):
+      assert delete_tag(repo, tag.name) == tag
+
+    assert get_tag(repo, tag.name) is None
+    found = True
+
+  assert found
+
+
+def test_delete_tags_for_manifest(initialized_db):
+  for tag in list(filter_to_visible_tags(filter_to_alive_tags(Tag.select()))):
+    repo = tag.repository
+    assert get_tag(repo, tag.name) == tag
+
+    with assert_query_count(5):
+      assert delete_tags_for_manifest(tag.manifest) == [tag]
+
+    assert get_tag(repo, tag.name) is None
+
+
+@pytest.mark.parametrize('timedelta, expected_timedelta', [
+  pytest.param(timedelta(seconds=1), timedelta(hours=1), id='less than minimum'),
+  pytest.param(timedelta(weeks=300), timedelta(weeks=104), id='more than maxium'),
+  pytest.param(timedelta(weeks=1), timedelta(weeks=1), id='within range'),
+])
+def test_change_tag_expiration(timedelta, expected_timedelta, initialized_db):
+  now = datetime.utcnow()
+  now_ms = timegm(now.utctimetuple()) * 1000
+
+  tag = Tag.get()
+  tag.lifetime_start_ms = now_ms
+  tag.save()
+
+  original_end_ms, okay = change_tag_expiration(tag, datetime.utcnow() + timedelta)
+  assert okay
+  assert original_end_ms == tag.lifetime_end_ms
+
+  updated_tag = Tag.get(id=tag.id)
+  offset = expected_timedelta.total_seconds() * 1000
+  expected_ms = (updated_tag.lifetime_start_ms + offset)
+  assert updated_tag.lifetime_end_ms == expected_ms
+
+
+def test_set_tag_expiration_for_manifest(initialized_db):
+  tag = Tag.get()
+  manifest = tag.manifest
+  assert manifest is not None
+
+  set_tag_expiration_for_manifest(manifest, datetime.utcnow() + timedelta(weeks=1))
+
+  updated_tag = Tag.get(id=tag.id)
+  assert updated_tag.lifetime_end_ms is not None
+
+
+def test_retarget_tag(initialized_db):
+  repo = get_repository('devtable', 'history')
+  results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name='latest')
+
+  assert len(results) == 2
+  assert results[0].lifetime_end_ms is None
+  assert results[1].lifetime_end_ms is not None
+
+  # Revert back to the original manifest.
+  created = retarget_tag('latest', results[0].manifest, is_reversion=True,
+                         now_ms=results[1].lifetime_end_ms + 10000)
+  assert created.lifetime_end_ms is None
+  assert created.reversion
+  assert created.name == 'latest'
+  assert created.manifest == results[0].manifest
+
+  # Verify in the history.
+  results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name='latest')
+
+  assert len(results) == 3
+  assert results[0].lifetime_end_ms is None
+  assert results[1].lifetime_end_ms is not None
+  assert results[2].lifetime_end_ms is not None
+
+  assert results[0] == created
+
+  # Verify old-style tables.
+  repository_tag = TagToRepositoryTag.get(tag=created).repository_tag
+  assert repository_tag.lifetime_start_ts == int(created.lifetime_start_ms / 1000)
+
+  tag_manifest = TagManifest.get(tag=repository_tag)
+  assert TagManifestToManifest.get(tag_manifest=tag_manifest).manifest == created.manifest
+
+
+def test_retarget_tag_wrong_name(initialized_db):
+  repo = get_repository('devtable', 'history')
+  results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name='latest')
+  assert len(results) == 2
+
+  created = retarget_tag('someothername', results[1].manifest, is_reversion=True)
+  assert created is None
+
+  results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name='latest')
+  assert len(results) == 2

data/model/tag.py

@@ -251,17 +251,18 @@ def list_repository_tags(namespace_name, repository_name, include_hidden=False,
 
 
 def create_or_update_tag(namespace_name, repository_name, tag_name, tag_docker_image_id,
-                         reversion=False):
+                         reversion=False, now_ms=None):
   try:
     repo = _basequery.get_existing_repository(namespace_name, repository_name)
   except Repository.DoesNotExist:
     raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name))
 
-  return create_or_update_tag_for_repo(repo.id, tag_name, tag_docker_image_id, reversion=reversion)
+  return create_or_update_tag_for_repo(repo.id, tag_name, tag_docker_image_id, reversion=reversion,
+                                       now_ms=now_ms)
 
 def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id, reversion=False,
-                                  oci_manifest=None):
-  now_ms = get_epoch_timestamp_ms()
+                                  oci_manifest=None, now_ms=None):
+  now_ms = now_ms or get_epoch_timestamp_ms()
   now_ts = int(now_ms / 1000)
 
   with db_transaction():