Have all tag code add, modify and delete both old and new style tags
This preps us for being able to use the new data model with existing repositories
This commit is contained in:
Joseph Schorr
parent
36db133b86
commit
114e2c3bf2
10 changed files with 274 additions and 54 deletions
|
|
@ -12,7 +12,7 @@ from data.database import (RepositoryTag, Repository, Image, ImageStorage, Names
|
|||
RepositoryNotification, Label, TagManifestLabel, get_epoch_timestamp,
|
||||
db_for_update, Manifest, ManifestLabel, ManifestBlob,
|
||||
ManifestLegacyImage, TagManifestToManifest,
|
||||
TagManifestLabelMap)
|
||||
TagManifestLabelMap, TagToRepositoryTag, Tag, get_epoch_timestamp_ms)
|
||||
from util.timedeltastring import convert_to_timedelta
|
||||
|
||||
|
||||
|
|
@ -259,8 +259,10 @@ def create_or_update_tag(namespace_name, repository_name, tag_name, tag_docker_i
|
|||
|
||||
return create_or_update_tag_for_repo(repo.id, tag_name, tag_docker_image_id, reversion=reversion)
|
||||
|
||||
def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id, reversion=False):
|
||||
now_ts = get_epoch_timestamp()
|
||||
def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id, reversion=False,
|
||||
oci_manifest=None):
|
||||
now_ms = get_epoch_timestamp_ms()
|
||||
now_ts = int(now_ms / 1000)
|
||||
|
||||
with db_transaction():
|
||||
try:
|
||||
|
|
@ -270,6 +272,17 @@ def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id,
|
|||
RepositoryTag.name == tag_name), now_ts)).get()
|
||||
tag.lifetime_end_ts = now_ts
|
||||
tag.save()
|
||||
|
||||
# Check for an OCI tag.
|
||||
try:
|
||||
oci_tag = db_for_update(Tag
|
||||
.select()
|
||||
.join(TagToRepositoryTag)
|
||||
.where(TagToRepositoryTag.repository_tag == tag)).get()
|
||||
oci_tag.lifetime_end_ms = now_ms
|
||||
oci_tag.save()
|
||||
except Tag.DoesNotExist:
|
||||
pass
|
||||
except RepositoryTag.DoesNotExist:
|
||||
pass
|
||||
except IntegrityError:
|
||||
|
|
@ -283,8 +296,16 @@ def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id,
|
|||
raise DataModelException('Invalid image with id: %s' % tag_docker_image_id)
|
||||
|
||||
try:
|
||||
return RepositoryTag.create(repository=repository_id, image=image_obj, name=tag_name,
|
||||
lifetime_start_ts=now_ts, reversion=reversion)
|
||||
created = RepositoryTag.create(repository=repository_id, image=image_obj, name=tag_name,
|
||||
lifetime_start_ts=now_ts, reversion=reversion)
|
||||
if oci_manifest:
|
||||
# Create the OCI tag as well.
|
||||
oci_tag = Tag.create(repository=repository_id, manifest=oci_manifest, name=tag_name,
|
||||
lifetime_start_ms=now_ms, reversion=reversion,
|
||||
tag_kind=Tag.tag_kind.get_id('tag'))
|
||||
TagToRepositoryTag.create(tag=oci_tag, repository_tag=created, repository=repository_id)
|
||||
|
||||
return created
|
||||
except IntegrityError:
|
||||
msg = 'Tag with name %s and lifetime start %s already exists'
|
||||
raise TagAlreadyCreatedException(msg % (tag_name, now_ts))
|
||||
|
|
@ -302,7 +323,9 @@ def create_temporary_hidden_tag(repo, image_obj, expiration_s):
|
|||
|
||||
|
||||
def delete_tag(namespace_name, repository_name, tag_name):
|
||||
now_ts = get_epoch_timestamp()
|
||||
now_ms = get_epoch_timestamp_ms()
|
||||
now_ts = int(now_ms / 1000)
|
||||
|
||||
with db_transaction():
|
||||
try:
|
||||
query = _tag_alive(RepositoryTag
|
||||
|
|
@ -320,6 +343,15 @@ def delete_tag(namespace_name, repository_name, tag_name):
|
|||
|
||||
found.lifetime_end_ts = now_ts
|
||||
found.save()
|
||||
|
||||
try:
|
||||
oci_tag_query = TagToRepositoryTag.select().where(TagToRepositoryTag.repository_tag == found)
|
||||
oci_tag = db_for_update(oci_tag_query).get().tag
|
||||
oci_tag.lifetime_end_ms = now_ms
|
||||
oci_tag.save()
|
||||
except TagToRepositoryTag.DoesNotExist:
|
||||
pass
|
||||
|
||||
return found
|
||||
|
||||
|
||||
|
|
@ -362,6 +394,24 @@ def _delete_tags(repo, query_modifier=None):
|
|||
return set()
|
||||
|
||||
with db_transaction():
|
||||
# Delete any associated new-style OCI tags.
|
||||
# NOTE: This will *not* work once we have tags pointing to other tags (e.g. for channels),
|
||||
# but this should all be changed over to new-style-only before we make *that* change.
|
||||
oci_tags_to_delete = list(Tag
|
||||
.select()
|
||||
.join(TagToRepositoryTag)
|
||||
.where(TagToRepositoryTag.repository == repo,
|
||||
TagToRepositoryTag.repository_tag << tags_to_delete))
|
||||
|
||||
if oci_tags_to_delete:
|
||||
# Delete the mapping entries.
|
||||
(TagToRepositoryTag.delete().where(TagToRepositoryTag.repository == repo,
|
||||
TagToRepositoryTag.repository_tag << tags_to_delete)
|
||||
.execute())
|
||||
|
||||
# Delete the tags themselves.
|
||||
Tag.delete().where(Tag.id << oci_tags_to_delete).execute()
|
||||
|
||||
# TODO(jschorr): Update to not use TagManifest once that table has been deprecated.
|
||||
tag_manifests_to_delete = list(TagManifest
|
||||
.select()
|
||||
|
|
@ -548,10 +598,16 @@ def restore_tag_to_manifest(repo_obj, tag_name, manifest_digest):
|
|||
except DataModelException:
|
||||
existing_image = None
|
||||
|
||||
# Change the tag manifest to point to the updated image.
|
||||
docker_image_id = tag_manifest.tag.image.docker_image_id
|
||||
oci_manifest = None
|
||||
try:
|
||||
oci_manifest = Manifest.get(repository=repo_obj, digest=manifest_digest)
|
||||
except Manifest.DoesNotExist:
|
||||
pass
|
||||
|
||||
# Change the tag and tag manifest to point to the updated image.
|
||||
updated_tag = create_or_update_tag_for_repo(repo_obj, tag_name, docker_image_id,
|
||||
reversion=True)
|
||||
reversion=True, oci_manifest=oci_manifest)
|
||||
tag_manifest.tag = updated_tag
|
||||
tag_manifest.save()
|
||||
return existing_image
|
||||
|
|
@ -601,9 +657,13 @@ def store_tag_manifest_for_repo(repository_id, tag_name, manifest, leaf_layer_id
|
|||
""" Stores a tag manifest for a specific tag name in the database. Returns the TagManifest
|
||||
object, as well as a boolean indicating whether the TagManifest was created.
|
||||
"""
|
||||
# Create the new-style OCI manifest and its blobs.
|
||||
oci_manifest = _populate_manifest_and_blobs(repository_id, manifest, storage_id_map,
|
||||
leaf_layer_id=leaf_layer_id)
|
||||
|
||||
# Create the tag for the tag manifest.
|
||||
tag = create_or_update_tag_for_repo(repository_id, tag_name, leaf_layer_id,
|
||||
reversion=reversion)
|
||||
reversion=reversion, oci_manifest=oci_manifest)
|
||||
|
||||
# Add a tag manifest pointing to that tag.
|
||||
try:
|
||||
|
|
@ -612,7 +672,8 @@ def store_tag_manifest_for_repo(repository_id, tag_name, manifest, leaf_layer_id
|
|||
manifest.save()
|
||||
return manifest, False
|
||||
except TagManifest.DoesNotExist:
|
||||
return _create_manifest(tag, manifest, storage_id_map), True
|
||||
created = _associate_manifest(tag, oci_manifest)
|
||||
return created, True
|
||||
|
||||
|
||||
def get_active_tag(namespace, repo_name, tag_name):
|
||||
|
|
@ -661,10 +722,42 @@ def associate_generated_tag_manifest(namespace, repo_name, tag_name, manifest, s
|
|||
manifest.save()
|
||||
return manifest, False
|
||||
except TagManifest.DoesNotExist:
|
||||
return _create_manifest(tag, manifest, storage_id_map), True
|
||||
oci_manifest = _populate_manifest_and_blobs(tag.repository, manifest, storage_id_map)
|
||||
|
||||
with db_transaction():
|
||||
try:
|
||||
(Tag
|
||||
.select()
|
||||
.join(TagToRepositoryTag)
|
||||
.where(TagToRepositoryTag.repository_tag == tag)).get()
|
||||
except Tag.DoesNotExist:
|
||||
oci_tag = Tag.create(repository=tag.repository, manifest=oci_manifest, name=tag_name,
|
||||
reversion=tag.reversion,
|
||||
lifetime_start_ms=tag.lifetime_start_ts * 1000,
|
||||
lifetime_end_ms=(tag.lifetime_end_ts * 1000
|
||||
if tag.lifetime_end_ts else None),
|
||||
tag_kind=Tag.tag_kind.get_id('tag'))
|
||||
TagToRepositoryTag.create(tag=oci_tag, repository_tag=tag, repository=tag.repository)
|
||||
|
||||
return _associate_manifest(tag, oci_manifest), True
|
||||
|
||||
|
||||
def _create_manifest(tag, manifest, storage_id_map):
|
||||
def _associate_manifest(tag, oci_manifest):
|
||||
with db_transaction():
|
||||
tag_manifest = TagManifest.create(tag=tag, digest=oci_manifest.digest,
|
||||
json_data=oci_manifest.manifest_bytes)
|
||||
TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=oci_manifest)
|
||||
return tag_manifest
|
||||
|
||||
|
||||
def _populate_manifest_and_blobs(repository, manifest, storage_id_map, leaf_layer_id=None):
|
||||
leaf_layer_id = leaf_layer_id or manifest.leaf_layer_v1_image_id
|
||||
try:
|
||||
legacy_image = Image.get(Image.docker_image_id == leaf_layer_id,
|
||||
Image.repository == repository)
|
||||
except Image.DoesNotExist:
|
||||
raise DataModelException('Invalid image with id: %s' % leaf_layer_id)
|
||||
|
||||
storage_ids = set()
|
||||
for blob_digest in manifest.blob_digests:
|
||||
image_storage_id = storage_id_map.get(blob_digest)
|
||||
|
|
@ -677,12 +770,7 @@ def _create_manifest(tag, manifest, storage_id_map):
|
|||
|
||||
storage_ids.add(image_storage_id)
|
||||
|
||||
manifest_row = populate_manifest(tag.repository, manifest, tag.image, storage_ids)
|
||||
|
||||
with db_transaction():
|
||||
tag_manifest = TagManifest.create(tag=tag, digest=manifest.digest, json_data=manifest.bytes)
|
||||
TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=manifest_row)
|
||||
return tag_manifest
|
||||
return populate_manifest(repository, manifest, legacy_image, storage_ids)
|
||||
|
||||
|
||||
def populate_manifest(repository, manifest, legacy_image, storage_ids):
|
||||
|
|
@ -800,15 +888,41 @@ def change_tag_expiration(tag, expiration_date):
|
|||
if end_ts == tag.lifetime_end_ts:
|
||||
return (None, True)
|
||||
|
||||
# Note: We check not just the ID of the tag but also its lifetime_end_ts, to ensure that it has
|
||||
# not changed while we were updatings it expiration.
|
||||
result = (RepositoryTag
|
||||
.update(lifetime_end_ts=end_ts)
|
||||
.where(RepositoryTag.id == tag.id,
|
||||
RepositoryTag.lifetime_end_ts == tag.lifetime_end_ts)
|
||||
.execute())
|
||||
return set_tag_end_ts(tag, end_ts)
|
||||
|
||||
return (tag.lifetime_end_ts, result > 0)
|
||||
|
||||
def set_tag_end_ts(tag, end_ts):
|
||||
""" Sets the end timestamp for a tag. Should only be called by change_tag_expiration
|
||||
or tests.
|
||||
"""
|
||||
end_ms = end_ts * 1000
|
||||
|
||||
with db_transaction():
|
||||
# Note: We check not just the ID of the tag but also its lifetime_end_ts, to ensure that it has
|
||||
# not changed while we were updating it expiration.
|
||||
result = (RepositoryTag
|
||||
.update(lifetime_end_ts=end_ts)
|
||||
.where(RepositoryTag.id == tag.id,
|
||||
RepositoryTag.lifetime_end_ts == tag.lifetime_end_ts)
|
||||
.execute())
|
||||
|
||||
# Check for a mapping to an OCI tag.
|
||||
try:
|
||||
oci_tag = (Tag
|
||||
.select()
|
||||
.join(TagToRepositoryTag)
|
||||
.where(TagToRepositoryTag.repository_tag == tag)
|
||||
.get())
|
||||
|
||||
(Tag
|
||||
.update(lifetime_end_ms=end_ms)
|
||||
.where(Tag.id == oci_tag.id,
|
||||
Tag.lifetime_end_ms == oci_tag.lifetime_end_ms)
|
||||
.execute())
|
||||
except Tag.DoesNotExist:
|
||||
pass
|
||||
|
||||
return (tag.lifetime_end_ts, result > 0)
|
||||
|
||||
|
||||
def find_matching_tag(repo_id, tag_names):
|
||||
|
|
|
|||
Reference in a new issue