initial import for Open Source 🎉

data/model/oci/test/test_oci_label.py

@@ -0,0 +1,87 @@
+import pytest
+
+from playhouse.test_utils import assert_query_count
+
+from data.database import Manifest, ManifestLabel
+from data.model.oci.label import (create_manifest_label, list_manifest_labels, get_manifest_label,
+                                  delete_manifest_label, DataModelException)
+
+from test.fixtures import *
+
+
+@pytest.mark.parametrize('key, value, source_type, expected_error', [
+  ('foo', 'bar', 'manifest', None),
+
+  pytest.param('..foo', 'bar', 'manifest', None, id='invalid key on manifest'),
+  pytest.param('..foo', 'bar', 'api', 'is invalid', id='invalid key on api'),
+])
+def test_create_manifest_label(key, value, source_type, expected_error, initialized_db):
+  manifest = Manifest.get()
+
+  if expected_error:
+    with pytest.raises(DataModelException) as ex:
+      create_manifest_label(manifest, key, value, source_type)
+
+    assert ex.match(expected_error)
+    return
+
+  label = create_manifest_label(manifest, key, value, source_type)
+  labels = [ml.label_id for ml in ManifestLabel.select().where(ManifestLabel.manifest == manifest)]
+  assert label.id in labels
+
+  with assert_query_count(1):
+    assert label in list_manifest_labels(manifest)
+
+  assert label not in list_manifest_labels(manifest, 'someprefix')
+  assert label in list_manifest_labels(manifest, key[0:2])
+
+  with assert_query_count(1):
+    assert get_manifest_label(label.uuid, manifest) == label
+
+
+def test_list_manifest_labels(initialized_db):
+  manifest = Manifest.get()
+
+  label1 = create_manifest_label(manifest, 'foo', '1', 'manifest')
+  label2 = create_manifest_label(manifest, 'bar', '2', 'api')
+  label3 = create_manifest_label(manifest, 'baz', '3', 'internal')
+
+  assert label1 in list_manifest_labels(manifest)
+  assert label2 in list_manifest_labels(manifest)
+  assert label3 in list_manifest_labels(manifest)
+
+  other_manifest = Manifest.select().where(Manifest.id != manifest.id).get()
+  assert label1 not in list_manifest_labels(other_manifest)
+  assert label2 not in list_manifest_labels(other_manifest)
+  assert label3 not in list_manifest_labels(other_manifest)
+
+
+def test_get_manifest_label(initialized_db):
+  found = False
+  for manifest_label in ManifestLabel.select():
+    assert (get_manifest_label(manifest_label.label.uuid, manifest_label.manifest) ==
+            manifest_label.label)
+    assert manifest_label.label in list_manifest_labels(manifest_label.manifest)
+    found = True
+
+  assert found
+
+
+def test_delete_manifest_label(initialized_db):
+  found = False
+  for manifest_label in list(ManifestLabel.select()):
+    assert (get_manifest_label(manifest_label.label.uuid, manifest_label.manifest) ==
+            manifest_label.label)
+    assert manifest_label.label in list_manifest_labels(manifest_label.manifest)
+
+    if manifest_label.label.source_type.mutable:
+      assert delete_manifest_label(manifest_label.label.uuid, manifest_label.manifest)
+      assert manifest_label.label not in list_manifest_labels(manifest_label.manifest)
+      assert get_manifest_label(manifest_label.label.uuid, manifest_label.manifest) is None
+    else:
+      with pytest.raises(DataModelException):
+        delete_manifest_label(manifest_label.label.uuid, manifest_label.manifest)
+
+    found = True
+
+  assert found

data/model/oci/test/test_oci_manifest.py

@@ -0,0 +1,560 @@
+import json
+
+from playhouse.test_utils import assert_query_count
+
+from app import docker_v2_signing_key, storage
+
+from digest.digest_tools import sha256_digest
+from data.database import (Tag, ManifestBlob, ImageStorageLocation, ManifestChild,
+                           ImageStorage, Image, RepositoryTag, get_epoch_timestamp_ms)
+from data.model.oci.manifest import lookup_manifest, get_or_create_manifest
+from data.model.oci.tag import filter_to_alive_tags, get_tag
+from data.model.oci.shared import get_legacy_image_for_manifest
+from data.model.oci.label import list_manifest_labels
+from data.model.oci.retriever import RepositoryContentRetriever
+from data.model.repository import get_repository, create_repository
+from data.model.image import find_create_or_link_image
+from data.model.blob import store_blob_record_and_temp_link
+from data.model.storage import get_layer_path
+from image.docker.schema1 import DockerSchema1ManifestBuilder, DockerSchema1Manifest
+from image.docker.schema2.manifest import DockerSchema2ManifestBuilder
+from image.docker.schema2.list import DockerSchema2ManifestListBuilder
+from util.bytes import Bytes
+
+from test.fixtures import *
+
+def test_lookup_manifest(initialized_db):
+  found = False
+  for tag in filter_to_alive_tags(Tag.select()):
+    found = True
+    repo = tag.repository
+    digest = tag.manifest.digest
+    with assert_query_count(1):
+      assert lookup_manifest(repo, digest) == tag.manifest
+
+  assert found
+
+  for tag in Tag.select():
+    repo = tag.repository
+    digest = tag.manifest.digest
+    with assert_query_count(1):
+      assert lookup_manifest(repo, digest, allow_dead=True) == tag.manifest
+
+
+def test_lookup_manifest_dead_tag(initialized_db):
+  dead_tag = Tag.select().where(Tag.lifetime_end_ms <= get_epoch_timestamp_ms()).get()
+  assert dead_tag.lifetime_end_ms <= get_epoch_timestamp_ms()
+
+  assert lookup_manifest(dead_tag.repository, dead_tag.manifest.digest) is None
+  assert (lookup_manifest(dead_tag.repository, dead_tag.manifest.digest, allow_dead=True) ==
+          dead_tag.manifest)
+
+
+def create_manifest_for_testing(repository, differentiation_field='1'):
+  # Populate a manifest.
+  layer_json = json.dumps({
+    'config': {},
+    "rootfs": {
+      "type": "layers",
+      "diff_ids": []
+    },
+    "history": [],
+  })
+
+  # Add a blob containing the config.
+  _, config_digest = _populate_blob(layer_json)
+
+  remote_digest = sha256_digest('something')
+  builder = DockerSchema2ManifestBuilder()
+  builder.set_config_digest(config_digest, len(layer_json))
+  builder.add_layer(remote_digest, 1234, urls=['http://hello/world' + differentiation_field])
+  manifest = builder.build()
+
+  created = get_or_create_manifest(repository, manifest, storage)
+  assert created
+  return created.manifest, manifest
+
+
+def test_lookup_manifest_child_tag(initialized_db):
+  repository = create_repository('devtable', 'newrepo', None)
+  manifest, manifest_impl = create_manifest_for_testing(repository)
+
+  # Mark the hidden tag as dead.
+  hidden_tag = Tag.get(manifest=manifest, hidden=True)
+  hidden_tag.lifetime_end_ms = hidden_tag.lifetime_start_ms
+  hidden_tag.save()
+
+  # Ensure the manifest cannot currently be looked up, as it is not pointed to by an alive tag.
+  assert lookup_manifest(repository, manifest.digest) is None
+  assert lookup_manifest(repository, manifest.digest, allow_dead=True) is not None
+
+  # Populate a manifest list.
+  list_builder = DockerSchema2ManifestListBuilder()
+  list_builder.add_manifest(manifest_impl, 'amd64', 'linux')
+  manifest_list = list_builder.build()
+
+  # Write the manifest list, which should also write the manifests themselves.
+  created_tuple = get_or_create_manifest(repository, manifest_list, storage)
+  assert created_tuple is not None
+
+  # Since the manifests are not yet referenced by a tag, they cannot be found.
+  assert lookup_manifest(repository, manifest.digest) is None
+  assert lookup_manifest(repository, manifest_list.digest) is None
+
+  # Unless we ask for "dead" manifests.
+  assert lookup_manifest(repository, manifest.digest, allow_dead=True) is not None
+  assert lookup_manifest(repository, manifest_list.digest, allow_dead=True) is not None
+
+
+def _populate_blob(content):
+  digest = str(sha256_digest(content))
+  location = ImageStorageLocation.get(name='local_us')
+  blob = store_blob_record_and_temp_link('devtable', 'newrepo', digest, location,
+                                         len(content), 120)
+  storage.put_content(['local_us'], get_layer_path(blob), content)
+  return blob, digest
+
+
+@pytest.mark.parametrize('schema_version', [
+  1,
+  2,
+])
+def test_get_or_create_manifest(schema_version, initialized_db):
+  repository = create_repository('devtable', 'newrepo', None)
+
+  expected_labels = {
+    'Foo': 'Bar',
+    'Baz': 'Meh',
+  }
+
+  layer_json = json.dumps({
+    'id': 'somelegacyid',
+    'config': {
+      'Labels': expected_labels,
+    },
+    "rootfs": {
+      "type": "layers",
+      "diff_ids": []
+    },
+    "history": [
+      {
+        "created": "2018-04-03T18:37:09.284840891Z",
+        "created_by": "do something",
+      },
+    ],
+  })
+
+  # Create a legacy image.
+  find_create_or_link_image('somelegacyid', repository, 'devtable', {}, 'local_us')
+
+  # Add a blob containing the config.
+  _, config_digest = _populate_blob(layer_json)
+
+  # Add a blob of random data.
+  random_data = 'hello world'
+  _, random_digest = _populate_blob(random_data)
+
+  # Build the manifest.
+  if schema_version == 1:
+    builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag')
+    builder.add_layer(random_digest, layer_json)
+    sample_manifest_instance = builder.build(docker_v2_signing_key)
+  elif schema_version == 2:
+    builder = DockerSchema2ManifestBuilder()
+    builder.set_config_digest(config_digest, len(layer_json))
+    builder.add_layer(random_digest, len(random_data))
+    sample_manifest_instance = builder.build()
+
+  # Create a new manifest.
+  created_manifest = get_or_create_manifest(repository, sample_manifest_instance, storage)
+  created = created_manifest.manifest
+  newly_created = created_manifest.newly_created
+
+  assert newly_created
+  assert created is not None
+  assert created.media_type.name == sample_manifest_instance.media_type
+  assert created.digest == sample_manifest_instance.digest
+  assert created.manifest_bytes == sample_manifest_instance.bytes.as_encoded_str()
+  assert created_manifest.labels_to_apply == expected_labels
+
+  # Verify it has a temporary tag pointing to it.
+  assert Tag.get(manifest=created, hidden=True).lifetime_end_ms
+
+  # Verify the legacy image.
+  legacy_image = get_legacy_image_for_manifest(created)
+  assert legacy_image is not None
+  assert legacy_image.storage.content_checksum == random_digest
+
+  # Verify the linked blobs.
+  blob_digests = [mb.blob.content_checksum for mb
+                  in ManifestBlob.select().where(ManifestBlob.manifest == created)]
+
+  assert random_digest in blob_digests
+  if schema_version == 2:
+    assert config_digest in blob_digests
+
+  # Retrieve it again and ensure it is the same manifest.
+  created_manifest2 = get_or_create_manifest(repository, sample_manifest_instance, storage)
+  created2 = created_manifest2.manifest
+  newly_created2 = created_manifest2.newly_created
+
+  assert not newly_created2
+  assert created2 == created
+
+  # Ensure it again has a temporary tag.
+  assert Tag.get(manifest=created2, hidden=True).lifetime_end_ms
+
+  # Ensure the labels were added.
+  labels = list(list_manifest_labels(created))
+  assert len(labels) == 2
+
+  labels_dict = {label.key: label.value for label in labels}
+  assert labels_dict == expected_labels
+
+
+def test_get_or_create_manifest_invalid_image(initialized_db):
+  repository = get_repository('devtable', 'simple')
+
+  latest_tag = get_tag(repository, 'latest')
+  parsed = DockerSchema1Manifest(Bytes.for_string_or_unicode(latest_tag.manifest.manifest_bytes),
+                                 validate=False)
+
+  builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag')
+  builder.add_layer(parsed.blob_digests[0], '{"id": "foo", "parent": "someinvalidimageid"}')
+  sample_manifest_instance = builder.build(docker_v2_signing_key)
+
+  created_manifest = get_or_create_manifest(repository, sample_manifest_instance, storage)
+  assert created_manifest is None
+
+
+def test_get_or_create_manifest_list(initialized_db):
+  repository = create_repository('devtable', 'newrepo', None)
+
+  expected_labels = {
+    'Foo': 'Bar',
+    'Baz': 'Meh',
+  }
+
+  layer_json = json.dumps({
+    'id': 'somelegacyid',
+    'config': {
+      'Labels': expected_labels,
+    },
+    "rootfs": {
+      "type": "layers",
+      "diff_ids": []
+    },
+    "history": [
+      {
+        "created": "2018-04-03T18:37:09.284840891Z",
+        "created_by": "do something",
+      },
+    ],
+  })
+
+  # Create a legacy image.
+  find_create_or_link_image('somelegacyid', repository, 'devtable', {}, 'local_us')
+
+  # Add a blob containing the config.
+  _, config_digest = _populate_blob(layer_json)
+
+  # Add a blob of random data.
+  random_data = 'hello world'
+  _, random_digest = _populate_blob(random_data)
+
+  # Build the manifests.
+  v1_builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag')
+  v1_builder.add_layer(random_digest, layer_json)
+  v1_manifest = v1_builder.build(docker_v2_signing_key).unsigned()
+
+  v2_builder = DockerSchema2ManifestBuilder()
+  v2_builder.set_config_digest(config_digest, len(layer_json))
+  v2_builder.add_layer(random_digest, len(random_data))
+  v2_manifest = v2_builder.build()
+
+  # Write the manifests.
+  v1_created = get_or_create_manifest(repository, v1_manifest, storage)
+  assert v1_created
+  assert v1_created.manifest.digest == v1_manifest.digest
+
+  v2_created = get_or_create_manifest(repository, v2_manifest, storage)
+  assert v2_created
+  assert v2_created.manifest.digest == v2_manifest.digest
+
+  # Build the manifest list.
+  list_builder = DockerSchema2ManifestListBuilder()
+  list_builder.add_manifest(v1_manifest, 'amd64', 'linux')
+  list_builder.add_manifest(v2_manifest, 'amd32', 'linux')
+  manifest_list = list_builder.build()
+
+  # Write the manifest list, which should also write the manifests themselves.
+  created_tuple = get_or_create_manifest(repository, manifest_list, storage)
+  assert created_tuple is not None
+
+  created_list = created_tuple.manifest
+  assert created_list
+  assert created_list.media_type.name == manifest_list.media_type
+  assert created_list.digest == manifest_list.digest
+
+  # Ensure the child manifest links exist.
+  child_manifests = {cm.child_manifest.digest: cm.child_manifest
+                     for cm in ManifestChild.select().where(ManifestChild.manifest == created_list)}
+  assert len(child_manifests) == 2
+  assert v1_manifest.digest in child_manifests
+  assert v2_manifest.digest in child_manifests
+
+  assert child_manifests[v1_manifest.digest].media_type.name == v1_manifest.media_type
+  assert child_manifests[v2_manifest.digest].media_type.name == v2_manifest.media_type
+
+
+def test_get_or_create_manifest_list_duplicate_child_manifest(initialized_db):
+  repository = create_repository('devtable', 'newrepo', None)
+
+  expected_labels = {
+    'Foo': 'Bar',
+    'Baz': 'Meh',
+  }
+
+  layer_json = json.dumps({
+    'id': 'somelegacyid',
+    'config': {
+      'Labels': expected_labels,
+    },
+    "rootfs": {
+      "type": "layers",
+      "diff_ids": []
+    },
+    "history": [
+      {
+        "created": "2018-04-03T18:37:09.284840891Z",
+        "created_by": "do something",
+      },
+    ],
+  })
+
+  # Create a legacy image.
+  find_create_or_link_image('somelegacyid', repository, 'devtable', {}, 'local_us')
+
+  # Add a blob containing the config.
+  _, config_digest = _populate_blob(layer_json)
+
+  # Add a blob of random data.
+  random_data = 'hello world'
+  _, random_digest = _populate_blob(random_data)
+
+  # Build the manifest.
+  v2_builder = DockerSchema2ManifestBuilder()
+  v2_builder.set_config_digest(config_digest, len(layer_json))
+  v2_builder.add_layer(random_digest, len(random_data))
+  v2_manifest = v2_builder.build()
+
+  # Write the manifest.
+  v2_created = get_or_create_manifest(repository, v2_manifest, storage)
+  assert v2_created
+  assert v2_created.manifest.digest == v2_manifest.digest
+
+  # Build the manifest list, with the child manifest repeated.
+  list_builder = DockerSchema2ManifestListBuilder()
+  list_builder.add_manifest(v2_manifest, 'amd64', 'linux')
+  list_builder.add_manifest(v2_manifest, 'amd32', 'linux')
+  manifest_list = list_builder.build()
+
+  # Write the manifest list, which should also write the manifests themselves.
+  created_tuple = get_or_create_manifest(repository, manifest_list, storage)
+  assert created_tuple is not None
+
+  created_list = created_tuple.manifest
+  assert created_list
+  assert created_list.media_type.name == manifest_list.media_type
+  assert created_list.digest == manifest_list.digest
+
+  # Ensure the child manifest links exist.
+  child_manifests = {cm.child_manifest.digest: cm.child_manifest
+                     for cm in ManifestChild.select().where(ManifestChild.manifest == created_list)}
+  assert len(child_manifests) == 1
+  assert v2_manifest.digest in child_manifests
+  assert child_manifests[v2_manifest.digest].media_type.name == v2_manifest.media_type
+
+  # Try to create again and ensure we get back the same manifest list.
+  created2_tuple = get_or_create_manifest(repository, manifest_list, storage)
+  assert created2_tuple is not None
+  assert created2_tuple.manifest == created_list
+
+
+def test_get_or_create_manifest_with_remote_layers(initialized_db):
+  repository = create_repository('devtable', 'newrepo', None)
+
+  layer_json = json.dumps({
+    'config': {},
+    "rootfs": {
+      "type": "layers",
+      "diff_ids": []
+    },
+    "history": [
+      {
+        "created": "2018-04-03T18:37:09.284840891Z",
+        "created_by": "do something",
+      },
+      {
+        "created": "2018-04-03T18:37:09.284840891Z",
+        "created_by": "do something",
+      },
+    ],
+  })
+
+  # Add a blob containing the config.
+  _, config_digest = _populate_blob(layer_json)
+
+  # Add a blob of random data.
+  random_data = 'hello world'
+  _, random_digest = _populate_blob(random_data)
+
+  remote_digest = sha256_digest('something')
+
+  builder = DockerSchema2ManifestBuilder()
+  builder.set_config_digest(config_digest, len(layer_json))
+  builder.add_layer(remote_digest, 1234, urls=['http://hello/world'])
+  builder.add_layer(random_digest, len(random_data))
+  manifest = builder.build()
+
+  assert remote_digest in manifest.blob_digests
+  assert remote_digest not in manifest.local_blob_digests
+
+  assert manifest.has_remote_layer
+  assert not manifest.has_legacy_image
+  assert manifest.get_schema1_manifest('foo', 'bar', 'baz', None) is None
+
+  # Write the manifest.
+  created_tuple = get_or_create_manifest(repository, manifest, storage)
+  assert created_tuple is not None
+
+  created_manifest = created_tuple.manifest
+  assert created_manifest
+  assert created_manifest.media_type.name == manifest.media_type
+  assert created_manifest.digest == manifest.digest
+
+  # Verify the legacy image.
+  legacy_image = get_legacy_image_for_manifest(created_manifest)
+  assert legacy_image is None
+
+  # Verify the linked blobs.
+  blob_digests = {mb.blob.content_checksum for mb
+                  in ManifestBlob.select().where(ManifestBlob.manifest == created_manifest)}
+
+  assert random_digest in blob_digests
+  assert config_digest in blob_digests
+  assert remote_digest not in blob_digests
+
+
+def create_manifest_for_testing(repository, differentiation_field='1', include_shared_blob=False):
+  # Populate a manifest.
+  layer_json = json.dumps({
+    'config': {},
+    "rootfs": {
+      "type": "layers",
+      "diff_ids": []
+    },
+    "history": [],
+  })
+
+  # Add a blob containing the config.
+  _, config_digest = _populate_blob(layer_json)
+
+  remote_digest = sha256_digest('something')
+  builder = DockerSchema2ManifestBuilder()
+  builder.set_config_digest(config_digest, len(layer_json))
+  builder.add_layer(remote_digest, 1234, urls=['http://hello/world' + differentiation_field])
+
+  if include_shared_blob:
+    _, blob_digest = _populate_blob('some data here')
+    builder.add_layer(blob_digest, 4567)
+
+  manifest = builder.build()
+
+  created = get_or_create_manifest(repository, manifest, storage)
+  assert created
+  return created.manifest, manifest
+
+
+def test_retriever(initialized_db):
+  repository = create_repository('devtable', 'newrepo', None)
+
+  layer_json = json.dumps({
+    'config': {},
+    "rootfs": {
+      "type": "layers",
+      "diff_ids": []
+    },
+    "history": [
+      {
+        "created": "2018-04-03T18:37:09.284840891Z",
+        "created_by": "do something",
+      },
+      {
+        "created": "2018-04-03T18:37:09.284840891Z",
+        "created_by": "do something",
+      },
+    ],
+  })
+
+  # Add a blob containing the config.
+  _, config_digest = _populate_blob(layer_json)
+
+  # Add a blob of random data.
+  random_data = 'hello world'
+  _, random_digest = _populate_blob(random_data)
+
+  # Add another blob of random data.
+  other_random_data = 'hi place'
+  _, other_random_digest = _populate_blob(other_random_data)
+
+  remote_digest = sha256_digest('something')
+
+  builder = DockerSchema2ManifestBuilder()
+  builder.set_config_digest(config_digest, len(layer_json))
+  builder.add_layer(other_random_digest, len(other_random_data))
+  builder.add_layer(random_digest, len(random_data))
+  manifest = builder.build()
+
+  assert config_digest in manifest.blob_digests
+  assert random_digest in manifest.blob_digests
+  assert other_random_digest in manifest.blob_digests
+
+  assert config_digest in manifest.local_blob_digests
+  assert random_digest in manifest.local_blob_digests
+  assert other_random_digest in manifest.local_blob_digests
+
+  # Write the manifest.
+  created_tuple = get_or_create_manifest(repository, manifest, storage)
+  assert created_tuple is not None
+
+  created_manifest = created_tuple.manifest
+  assert created_manifest
+  assert created_manifest.media_type.name == manifest.media_type
+  assert created_manifest.digest == manifest.digest
+
+  # Verify the linked blobs.
+  blob_digests = {mb.blob.content_checksum for mb
+                  in ManifestBlob.select().where(ManifestBlob.manifest == created_manifest)}
+
+  assert random_digest in blob_digests
+  assert other_random_digest in blob_digests
+  assert config_digest in blob_digests
+
+  # Delete any Image rows linking to the blobs from temp tags.
+  for blob_digest in blob_digests:
+    storage_row = ImageStorage.get(content_checksum=blob_digest)
+    for image in list(Image.select().where(Image.storage == storage_row)):
+      all_temp = all([rt.hidden for rt
+                      in RepositoryTag.select().where(RepositoryTag.image == image)])
+      if all_temp:
+        RepositoryTag.delete().where(RepositoryTag.image == image).execute()
+        image.delete_instance(recursive=True)
+
+  # Verify the blobs in the retriever.
+  retriever = RepositoryContentRetriever(repository, storage)
+  assert (retriever.get_manifest_bytes_with_digest(created_manifest.digest) ==
+          manifest.bytes.as_encoded_str())
+
+  for blob_digest in blob_digests:
+    assert retriever.get_blob_bytes_with_digest(blob_digest) is not None

data/model/oci/test/test_oci_tag.py

@@ -0,0 +1,378 @@
+from calendar import timegm
+from datetime import timedelta, datetime
+
+from playhouse.test_utils import assert_query_count
+
+from data.database import (Tag, ManifestLegacyImage, TagToRepositoryTag, TagManifestToManifest,
+                           TagManifest, Manifest, Repository)
+from data.model.oci.test.test_oci_manifest import create_manifest_for_testing
+from data.model.oci.tag import (find_matching_tag, get_most_recent_tag,
+                                get_most_recent_tag_lifetime_start, list_alive_tags,
+                                get_legacy_images_for_tags, filter_to_alive_tags,
+                                filter_to_visible_tags, list_repository_tag_history,
+                                get_expired_tag, get_tag, delete_tag,
+                                delete_tags_for_manifest, change_tag_expiration,
+                                set_tag_expiration_for_manifest, retarget_tag,
+                                create_temporary_tag_if_necessary,
+                                lookup_alive_tags_shallow,
+                                lookup_unrecoverable_tags,
+                                get_epoch_timestamp_ms)
+from data.model.repository import get_repository, create_repository
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('namespace_name, repo_name, tag_names, expected', [
+  ('devtable', 'simple', ['latest'], 'latest'),
+  ('devtable', 'simple', ['unknown', 'latest'], 'latest'),
+  ('devtable', 'simple', ['unknown'], None),
+])
+def test_find_matching_tag(namespace_name, repo_name, tag_names, expected, initialized_db):
+  repo = get_repository(namespace_name, repo_name)
+  if expected is not None:
+    with assert_query_count(1):
+      found = find_matching_tag(repo, tag_names)
+
+    assert found is not None
+    assert found.name == expected
+    assert not found.lifetime_end_ms
+  else:
+    with assert_query_count(1):
+      assert find_matching_tag(repo, tag_names) is None
+
+
+def test_get_most_recent_tag_lifetime_start(initialized_db):
+  repo = get_repository('devtable', 'simple')
+  tag = get_most_recent_tag(repo)
+
+  with assert_query_count(1):
+    tags = get_most_recent_tag_lifetime_start([repo])
+    assert tags[repo.id] == tag.lifetime_start_ms
+
+
+def test_get_most_recent_tag(initialized_db):
+  repo = get_repository('outsideorg', 'coolrepo')
+
+  with assert_query_count(1):
+    assert get_most_recent_tag(repo).name == 'latest'
+
+
+def test_get_most_recent_tag_empty_repo(initialized_db):
+  empty_repo = create_repository('devtable', 'empty', None)
+
+  with assert_query_count(1):
+    assert get_most_recent_tag(empty_repo) is None
+
+
+def test_list_alive_tags(initialized_db):
+  found = False
+  for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())):
+    tags = list_alive_tags(tag.repository)
+    assert tag in tags
+
+    with assert_query_count(1):
+      legacy_images = get_legacy_images_for_tags(tags)
+
+    for tag in tags:
+      assert ManifestLegacyImage.get(manifest=tag.manifest).image == legacy_images[tag.id]
+
+    found = True
+
+  assert found
+
+  # Ensure hidden tags cannot be listed.
+  tag = Tag.get()
+  tag.hidden = True
+  tag.save()
+
+  tags = list_alive_tags(tag.repository)
+  assert tag not in tags
+
+
+def test_lookup_alive_tags_shallow(initialized_db):
+  found = False
+  for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())):
+    tags = lookup_alive_tags_shallow(tag.repository)
+    found = True
+    assert tag in tags
+
+  assert found
+
+  # Ensure hidden tags cannot be listed.
+  tag = Tag.get()
+  tag.hidden = True
+  tag.save()
+
+  tags = lookup_alive_tags_shallow(tag.repository)
+  assert tag not in tags
+
+
+def test_get_tag(initialized_db):
+  found = False
+  for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())):
+    repo = tag.repository
+
+    with assert_query_count(1):
+      assert get_tag(repo, tag.name) == tag
+    found = True
+
+  assert found
+
+
+@pytest.mark.parametrize('namespace_name, repo_name', [
+  ('devtable', 'simple'),
+  ('devtable', 'complex'),
+])
+def test_list_repository_tag_history(namespace_name, repo_name, initialized_db):
+  repo = get_repository(namespace_name, repo_name)
+
+  with assert_query_count(1):
+    results, has_more = list_repository_tag_history(repo, 1, 100)
+
+  assert results
+  assert not has_more
+
+
+def test_list_repository_tag_history_with_history(initialized_db):
+  repo = get_repository('devtable', 'history')
+
+  with assert_query_count(1):
+    results, _ = list_repository_tag_history(repo, 1, 100)
+
+  assert len(results) == 2
+  assert results[0].lifetime_end_ms is None
+  assert results[1].lifetime_end_ms is not None
+
+  with assert_query_count(1):
+    results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name='latest')
+
+  assert len(results) == 2
+  assert results[0].lifetime_end_ms is None
+  assert results[1].lifetime_end_ms is not None
+
+  with assert_query_count(1):
+    results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name='foobar')
+
+  assert len(results) == 0
+
+
+def test_list_repository_tag_history_all_tags(initialized_db):
+  for tag in Tag.select():
+    repo = tag.repository
+    with assert_query_count(1):
+      results, _ = list_repository_tag_history(repo, 1, 1000)
+
+    assert (tag in results) == (not tag.hidden)
+
+
+@pytest.mark.parametrize('namespace_name, repo_name, tag_name, expected', [
+  ('devtable', 'simple', 'latest', False),
+  ('devtable', 'simple', 'unknown', False),
+  ('devtable', 'complex', 'latest', False),
+
+  ('devtable', 'history', 'latest', True),
+])
+def test_get_expired_tag(namespace_name, repo_name, tag_name, expected, initialized_db):
+  repo = get_repository(namespace_name, repo_name)
+
+  with assert_query_count(1):
+    assert bool(get_expired_tag(repo, tag_name)) == expected
+
+
+def test_delete_tag(initialized_db):
+  found = False
+  for tag in list(filter_to_visible_tags(filter_to_alive_tags(Tag.select()))):
+    repo = tag.repository
+
+    assert get_tag(repo, tag.name) == tag
+    assert tag.lifetime_end_ms is None
+
+    with assert_query_count(4):
+      assert delete_tag(repo, tag.name) == tag
+
+    assert get_tag(repo, tag.name) is None
+    found = True
+
+  assert found
+
+
+def test_delete_tags_for_manifest(initialized_db):
+  for tag in list(filter_to_visible_tags(filter_to_alive_tags(Tag.select()))):
+    repo = tag.repository
+    assert get_tag(repo, tag.name) == tag
+
+    with assert_query_count(5):
+      assert delete_tags_for_manifest(tag.manifest) == [tag]
+
+    assert get_tag(repo, tag.name) is None
+
+
+def test_delete_tags_for_manifest_same_manifest(initialized_db):
+  new_repo = model.repository.create_repository('devtable', 'newrepo', None)
+  manifest_1, _ = create_manifest_for_testing(new_repo, '1')
+  manifest_2, _ = create_manifest_for_testing(new_repo, '2')
+
+  assert manifest_1.digest != manifest_2.digest
+
+  # Add some tag history, moving a tag back and forth between two manifests.
+  retarget_tag('latest', manifest_1)
+  retarget_tag('latest', manifest_2)
+  retarget_tag('latest', manifest_1)
+  retarget_tag('latest', manifest_2)
+
+  retarget_tag('another1', manifest_1)
+  retarget_tag('another2', manifest_2)
+
+  # Delete all tags pointing to the first manifest.
+  delete_tags_for_manifest(manifest_1)
+
+  assert get_tag(new_repo, 'latest').manifest == manifest_2
+  assert get_tag(new_repo, 'another1') is None
+  assert get_tag(new_repo, 'another2').manifest == manifest_2
+
+  # Delete all tags pointing to the second manifest, which should actually delete the `latest`
+  # tag now.
+  delete_tags_for_manifest(manifest_2)
+  assert get_tag(new_repo, 'latest') is None
+  assert get_tag(new_repo, 'another1') is None
+  assert get_tag(new_repo, 'another2') is None
+
+
+@pytest.mark.parametrize('timedelta, expected_timedelta', [
+  pytest.param(timedelta(seconds=1), timedelta(hours=1), id='less than minimum'),
+  pytest.param(timedelta(weeks=300), timedelta(weeks=104), id='more than maxium'),
+  pytest.param(timedelta(weeks=1), timedelta(weeks=1), id='within range'),
+])
+def test_change_tag_expiration(timedelta, expected_timedelta, initialized_db):
+  now = datetime.utcnow()
+  now_ms = timegm(now.utctimetuple()) * 1000
+
+  tag = Tag.get()
+  tag.lifetime_start_ms = now_ms
+  tag.save()
+
+  original_end_ms, okay = change_tag_expiration(tag, now + timedelta)
+  assert okay
+  assert original_end_ms == tag.lifetime_end_ms
+
+  updated_tag = Tag.get(id=tag.id)
+  offset = expected_timedelta.total_seconds() * 1000
+  expected_ms = (updated_tag.lifetime_start_ms + offset)
+  assert updated_tag.lifetime_end_ms == expected_ms
+
+  original_end_ms, okay = change_tag_expiration(tag, None)
+  assert okay
+  assert original_end_ms == expected_ms
+
+  updated_tag = Tag.get(id=tag.id)
+  assert updated_tag.lifetime_end_ms is None
+
+
+def test_set_tag_expiration_for_manifest(initialized_db):
+  tag = Tag.get()
+  manifest = tag.manifest
+  assert manifest is not None
+
+  set_tag_expiration_for_manifest(manifest, datetime.utcnow() + timedelta(weeks=1))
+
+  updated_tag = Tag.get(id=tag.id)
+  assert updated_tag.lifetime_end_ms is not None
+
+
+def test_create_temporary_tag_if_necessary(initialized_db):
+  tag = Tag.get()
+  manifest = tag.manifest
+  assert manifest is not None
+
+  # Ensure no tag is created, since an existing one is present.
+  created = create_temporary_tag_if_necessary(manifest, 60)
+  assert created is None
+
+  # Mark the tag as deleted.
+  tag.lifetime_end_ms = 1
+  tag.save()
+
+  # Now create a temp tag.
+  created = create_temporary_tag_if_necessary(manifest, 60)
+  assert created is not None
+  assert created.hidden
+  assert created.name.startswith('$temp-')
+  assert created.manifest == manifest
+  assert created.lifetime_end_ms is not None
+  assert created.lifetime_end_ms == (created.lifetime_start_ms + 60000)
+
+  # Try again and ensure it is not created.
+  created = create_temporary_tag_if_necessary(manifest, 30)
+  assert created is None
+
+
+def test_retarget_tag(initialized_db):
+  repo = get_repository('devtable', 'history')
+  results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name='latest')
+
+  assert len(results) == 2
+  assert results[0].lifetime_end_ms is None
+  assert results[1].lifetime_end_ms is not None
+
+  # Revert back to the original manifest.
+  created = retarget_tag('latest', results[0].manifest, is_reversion=True,
+                         now_ms=results[1].lifetime_end_ms + 10000)
+  assert created.lifetime_end_ms is None
+  assert created.reversion
+  assert created.name == 'latest'
+  assert created.manifest == results[0].manifest
+
+  # Verify in the history.
+  results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name='latest')
+
+  assert len(results) == 3
+  assert results[0].lifetime_end_ms is None
+  assert results[1].lifetime_end_ms is not None
+  assert results[2].lifetime_end_ms is not None
+
+  assert results[0] == created
+
+  # Verify old-style tables.
+  repository_tag = TagToRepositoryTag.get(tag=created).repository_tag
+  assert repository_tag.lifetime_start_ts == int(created.lifetime_start_ms / 1000)
+
+  tag_manifest = TagManifest.get(tag=repository_tag)
+  assert TagManifestToManifest.get(tag_manifest=tag_manifest).manifest == created.manifest
+
+
+def test_retarget_tag_wrong_name(initialized_db):
+  repo = get_repository('devtable', 'history')
+  results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name='latest')
+  assert len(results) == 2
+
+  created = retarget_tag('someothername', results[1].manifest, is_reversion=True)
+  assert created is None
+
+  results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name='latest')
+  assert len(results) == 2
+
+
+def test_lookup_unrecoverable_tags(initialized_db):
+  # Ensure no existing tags are found.
+  for repo in Repository.select():
+    assert not list(lookup_unrecoverable_tags(repo))
+
+  # Mark a tag as outside the expiration window and ensure it is found.
+  repo = get_repository('devtable', 'history')
+  results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name='latest')
+  assert len(results) == 2
+
+  results[1].lifetime_end_ms = 1
+  results[1].save()
+
+  # Ensure the tag is now found.
+  found = list(lookup_unrecoverable_tags(repo))
+  assert found
+  assert len(found) == 1
+  assert found[0] == results[1]
+
+  # Mark the tag as expiring in the future and ensure it is no longer found.
+  results[1].lifetime_end_ms = get_epoch_timestamp_ms() + 1000000
+  results[1].save()
+
+  found = list(lookup_unrecoverable_tags(repo))
+  assert not found