Have all tag code add, modify and delete both old and new style tags
This preps us for being able to use the new data model with existing repositories
This commit is contained in:
Joseph Schorr
parent
36db133b86
commit
114e2c3bf2
10 changed files with 274 additions and 54 deletions
|
|
@ -12,7 +12,7 @@ from playhouse.test_utils import assert_query_count
|
|||
from data import model, database
|
||||
from data.database import (Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel,
|
||||
ApprBlob, Manifest, TagManifest, TagManifestToManifest, ManifestLabel,
|
||||
TagManifestLabelMap, ManifestBlob)
|
||||
TagManifestLabelMap, ManifestBlob, Tag, TagToRepositoryTag)
|
||||
from image.docker.schema1 import DockerSchema1ManifestBuilder
|
||||
from test.fixtures import *
|
||||
|
||||
|
|
@ -223,6 +223,11 @@ def assert_gc_integrity(expect_storage_removed=True):
|
|||
for blob_row in ApprBlob.select():
|
||||
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
|
||||
|
||||
# Ensure there are no danglings OCI tags.
|
||||
oci_tags = {t.id for t in Tag.select()}
|
||||
referenced_oci_tags = {t.tag_id for t in TagToRepositoryTag.select()}
|
||||
assert not (oci_tags - referenced_oci_tags)
|
||||
|
||||
|
||||
def test_has_garbage(default_tag_policy, initialized_db):
|
||||
""" Remove all existing repositories, then add one without garbage, check, then add one with
|
||||
|
|
|
|||
|
|
@ -9,12 +9,13 @@ from mock import patch
|
|||
|
||||
from app import docker_v2_signing_key
|
||||
from data.database import (Image, RepositoryTag, ImageStorage, Repository, Manifest, ManifestBlob,
|
||||
ManifestLegacyImage, TagManifestToManifest)
|
||||
ManifestLegacyImage, TagManifestToManifest, Tag, TagToRepositoryTag)
|
||||
from data.model.repository import create_repository
|
||||
from data.model.tag import (list_active_repo_tags, create_or_update_tag, delete_tag,
|
||||
get_matching_tags, _tag_alive, get_matching_tags_for_images,
|
||||
change_tag_expiration, get_active_tag, store_tag_manifest_for_testing,
|
||||
get_most_recent_tag, get_active_tag_for_repo)
|
||||
get_most_recent_tag, get_active_tag_for_repo,
|
||||
create_or_update_tag_for_repo, set_tag_end_ts)
|
||||
from data.model.image import find_create_or_link_image
|
||||
from image.docker.schema1 import DockerSchema1ManifestBuilder
|
||||
from util.timedeltastring import convert_to_timedelta
|
||||
|
|
@ -24,12 +25,13 @@ from test.fixtures import *
|
|||
|
||||
def _get_expected_tags(image):
|
||||
expected_query = (RepositoryTag
|
||||
.select()
|
||||
.join(Image)
|
||||
.where(RepositoryTag.hidden == False)
|
||||
.where((Image.id == image.id) | (Image.ancestors ** ('%%/%s/%%' % image.id))))
|
||||
.select()
|
||||
.join(Image)
|
||||
.where(RepositoryTag.hidden == False)
|
||||
.where((Image.id == image.id) | (Image.ancestors ** ('%%/%s/%%' % image.id))))
|
||||
return set([tag.id for tag in _tag_alive(expected_query)])
|
||||
|
||||
|
||||
@pytest.mark.parametrize('max_subqueries,max_image_lookup_count', [
|
||||
(1, 1),
|
||||
(10, 10),
|
||||
|
|
@ -45,6 +47,12 @@ def test_get_matching_tags(max_subqueries, max_image_lookup_count, initialized_d
|
|||
expected_tags = _get_expected_tags(image)
|
||||
assert matching_tags == expected_tags, "mismatch for image %s" % image.id
|
||||
|
||||
oci_tags = list(Tag
|
||||
.select()
|
||||
.join(TagToRepositoryTag)
|
||||
.where(TagToRepositoryTag.repository_tag << expected_tags))
|
||||
assert len(oci_tags) == len(expected_tags)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('max_subqueries,max_image_lookup_count', [
|
||||
(1, 1),
|
||||
|
|
@ -116,6 +124,13 @@ def test_get_matching_tag_ids_images_filtered(initialized_db):
|
|||
assert matching_tags_ids == expected_tag_ids
|
||||
|
||||
|
||||
def _get_oci_tag(tag):
|
||||
return (Tag
|
||||
.select()
|
||||
.join(TagToRepositoryTag)
|
||||
.where(TagToRepositoryTag.repository_tag == tag)).get()
|
||||
|
||||
|
||||
def assert_tags(repository, *args):
|
||||
tags = list(list_active_repo_tags(repository))
|
||||
assert len(tags) == len(args)
|
||||
|
|
@ -128,12 +143,23 @@ def assert_tags(repository, *args):
|
|||
|
||||
tags_dict[tag.name] = tag
|
||||
|
||||
oci_tag = _get_oci_tag(tag)
|
||||
assert oci_tag.name == tag.name
|
||||
assert not oci_tag.hidden
|
||||
|
||||
if tag.lifetime_end_ts:
|
||||
assert oci_tag.lifetime_end_ms == (tag.lifetime_end_ts * 1000)
|
||||
else:
|
||||
assert oci_tag.lifetime_end_ms is None
|
||||
|
||||
for expected in args:
|
||||
assert expected in tags_dict
|
||||
|
||||
|
||||
def test_list_active_tags(initialized_db):
|
||||
# Create a new repository.
|
||||
repository = create_repository('devtable', 'somenewrepo', None)
|
||||
manifest = Manifest.get()
|
||||
|
||||
# Create some images.
|
||||
image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us')
|
||||
|
|
@ -143,50 +169,62 @@ def test_list_active_tags(initialized_db):
|
|||
assert_tags(repository)
|
||||
|
||||
# Add some new tags.
|
||||
footag = create_or_update_tag('devtable', 'somenewrepo', 'foo', image1.docker_image_id)
|
||||
bartag = create_or_update_tag('devtable', 'somenewrepo', 'bar', image1.docker_image_id)
|
||||
footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id,
|
||||
oci_manifest=manifest)
|
||||
bartag = create_or_update_tag_for_repo(repository, 'bar', image1.docker_image_id,
|
||||
oci_manifest=manifest)
|
||||
|
||||
# Since timestamps are stored on a second-granuality, we need to make the tags "start"
|
||||
# Since timestamps are stored on a second-granularity, we need to make the tags "start"
|
||||
# before "now", so when we recreate them below, they don't conflict.
|
||||
footag.lifetime_start_ts -= 5
|
||||
bartag.lifetime_start_ts -= 5
|
||||
|
||||
footag.save()
|
||||
|
||||
bartag.lifetime_start_ts -= 5
|
||||
bartag.save()
|
||||
|
||||
footag_oci = _get_oci_tag(footag)
|
||||
footag_oci.lifetime_start_ms -= 5000
|
||||
footag_oci.save()
|
||||
|
||||
bartag_oci = _get_oci_tag(bartag)
|
||||
bartag_oci.lifetime_start_ms -= 5000
|
||||
bartag_oci.save()
|
||||
|
||||
# Make sure they are returned.
|
||||
assert_tags(repository, 'foo', 'bar')
|
||||
|
||||
# Mark as a tag as expiring in the far future, and make sure it is still returned.
|
||||
footag.lifetime_end_ts = footag.lifetime_start_ts + 10000000
|
||||
footag.save()
|
||||
set_tag_end_ts(footag, footag.lifetime_start_ts + 10000000)
|
||||
|
||||
# Make sure they are returned.
|
||||
assert_tags(repository, 'foo', 'bar')
|
||||
|
||||
# Delete a tag and make sure it isn't returned.
|
||||
footag = delete_tag('devtable', 'somenewrepo', 'foo')
|
||||
footag.lifetime_end_ts -= 4
|
||||
footag.save()
|
||||
set_tag_end_ts(footag, footag.lifetime_end_ts - 4)
|
||||
|
||||
assert_tags(repository, 'bar')
|
||||
|
||||
# Add a new foo again.
|
||||
footag = create_or_update_tag('devtable', 'somenewrepo', 'foo', image1.docker_image_id)
|
||||
footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id,
|
||||
oci_manifest=manifest)
|
||||
footag.lifetime_start_ts -= 3
|
||||
footag.save()
|
||||
|
||||
footag_oci = _get_oci_tag(footag)
|
||||
footag_oci.lifetime_start_ms -= 3000
|
||||
footag_oci.save()
|
||||
|
||||
assert_tags(repository, 'foo', 'bar')
|
||||
|
||||
# Mark as a tag as expiring in the far future, and make sure it is still returned.
|
||||
footag.lifetime_end_ts = footag.lifetime_start_ts + 10000000
|
||||
footag.save()
|
||||
set_tag_end_ts(footag, footag.lifetime_start_ts + 10000000)
|
||||
|
||||
# Make sure they are returned.
|
||||
assert_tags(repository, 'foo', 'bar')
|
||||
|
||||
# "Move" foo by updating it and make sure we don't get duplicates.
|
||||
create_or_update_tag('devtable', 'somenewrepo', 'foo', image2.docker_image_id)
|
||||
create_or_update_tag_for_repo(repository, 'foo', image2.docker_image_id, oci_manifest=manifest)
|
||||
assert_tags(repository, 'foo', 'bar')
|
||||
|
||||
|
||||
|
|
@ -201,7 +239,10 @@ def test_list_active_tags(initialized_db):
|
|||
def test_change_tag_expiration(expiration_offset, expected_offset, initialized_db):
|
||||
repository = create_repository('devtable', 'somenewrepo', None)
|
||||
image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us')
|
||||
footag = create_or_update_tag('devtable', 'somenewrepo', 'foo', image1.docker_image_id)
|
||||
|
||||
manifest = Manifest.get()
|
||||
footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id,
|
||||
oci_manifest=manifest)
|
||||
|
||||
expiration_date = None
|
||||
if expiration_offset is not None:
|
||||
|
|
@ -211,15 +252,19 @@ def test_change_tag_expiration(expiration_offset, expected_offset, initialized_d
|
|||
|
||||
# Lookup the tag again.
|
||||
footag_updated = get_active_tag('devtable', 'somenewrepo', 'foo')
|
||||
oci_tag = _get_oci_tag(footag_updated)
|
||||
|
||||
if expected_offset is None:
|
||||
assert footag_updated.lifetime_end_ts is None
|
||||
assert oci_tag.lifetime_end_ms is None
|
||||
else:
|
||||
start_date = datetime.utcfromtimestamp(footag_updated.lifetime_start_ts)
|
||||
end_date = datetime.utcfromtimestamp(footag_updated.lifetime_end_ts)
|
||||
expected_end_date = start_date + convert_to_timedelta(expected_offset)
|
||||
assert (expected_end_date - end_date).total_seconds() < 5 # variance in test
|
||||
|
||||
assert oci_tag.lifetime_end_ms == (footag_updated.lifetime_end_ts * 1000)
|
||||
|
||||
|
||||
def random_storages():
|
||||
return list(ImageStorage.select().where(~(ImageStorage.content_checksum >> None)).limit(10))
|
||||
|
|
|
|||
Reference in a new issue