vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add support for direct pushing of schema 2 manifests without tags

Browse source 
This is required for manifest lists
This commit is contained in:
Joseph Schorr 2018-11-19 13:23:29 +02:00
parent 8a3427e55a
commit e6c2ddfa93
6 changed files with 158 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
data/model/oci/tag.py
View file

@ -1,3 +1,4 @@
import uuid
import logging
from calendar import timegm
@ -155,6 +156,47 @@ def get_expired_tag(repository_id, tag_name):
return None
def create_temporary_tag(manifest, expiration_sec):
""" Creates a temporary tag pointing to the given manifest, with the given expiration in seconds.
"""
tag_name = '$temp-%s' % str(uuid.uuid4())
now_ms = get_epoch_timestamp_ms()
end_ms = now_ms + (expiration_sec * 1000)
legacy_image = get_legacy_image_for_manifest(manifest)
with db_transaction():
created_tag = Tag.create(name=tag_name,
repository=manifest.repository_id,
lifetime_start_ms=now_ms,
lifetime_end_ms=end_ms,
reversion=False,
hidden=True,
manifest=manifest,
tag_kind=Tag.tag_kind.get_id('tag'))
# TODO(jschorr): Remove the linkage code once RepositoryTag is gone.
# If this is a schema 1 manifest, then add a TagManifest linkage to it. Otherwise, it will only
# be pullable via the new OCI model.
if manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES and legacy_image is not None:
now_ts = int(now_ms / 1000)
end_ts = int(end_ms / 1000)
old_style_tag = RepositoryTag.create(repository=manifest.repository_id, image=legacy_image,
name=tag_name, lifetime_start_ts=now_ts,
lifetime_end_ts=end_ts,
reversion=False, hidden=True)
TagToRepositoryTag.create(tag=created_tag, repository_tag=old_style_tag,
repository=manifest.repository_id)
tag_manifest = TagManifest.create(tag=old_style_tag, digest=manifest.digest,
json_data=manifest.manifest_bytes)
TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=manifest,
repository=manifest.repository_id)
return created_tag
def retarget_tag(tag_name, manifest_id, is_reversion=False, now_ms=None):
""" Creates or updates a tag with the specified name to point to the given manifest under
its repository. If this action is a reversion to a previous manifest, is_reversion

28
data/model/oci/test/test_oci_tag.py
View file

@ -10,7 +10,8 @@ from data.model.oci.tag import (find_matching_tag, get_most_recent_tag, list_ali
filter_to_visible_tags, list_repository_tag_history,
get_expired_tag, get_tag, delete_tag,
delete_tags_for_manifest, change_tag_expiration,
set_tag_expiration_for_manifest, retarget_tag)
set_tag_expiration_for_manifest, retarget_tag,
create_temporary_tag)
from data.model.repository import get_repository, create_repository
from test.fixtures import *
@ -207,6 +208,31 @@ def test_set_tag_expiration_for_manifest(initialized_db):
assert updated_tag.lifetime_end_ms is not None
def test_create_temporary_tag(initialized_db):
tag = Tag.get()
manifest = tag.manifest
assert manifest is not None
created = create_temporary_tag(manifest, 30)
assert created is not None
assert created.hidden
assert created.name.startswith('$temp-')
assert created.manifest == manifest
assert created.lifetime_end_ms is not None
assert created.lifetime_end_ms == (created.lifetime_start_ms + 30000)
# Verify old-style tables.
repository_tag = TagToRepositoryTag.get(tag=created).repository_tag
assert repository_tag.lifetime_start_ts == int(created.lifetime_start_ms / 1000)
assert repository_tag.lifetime_end_ts == int(created.lifetime_end_ms / 1000)
assert repository_tag.name == created.name
assert repository_tag.hidden
tag_manifest = TagManifest.get(tag=repository_tag)
assert TagManifestToManifest.get(tag_manifest=tag_manifest).manifest == manifest
def test_retarget_tag(initialized_db):
repo = get_repository('devtable', 'history')
results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name='latest')

7
data/registry_model/interface.py
View file

@ -303,3 +303,10 @@ class RegistryDataInterface(object):
@abstractmethod
def get_schema1_parsed_manifest(self, manifest, namespace_name, repo_name, tag_name, storage):
""" Returns the schema 1 version of this manifest, or None if none. """
@abstractmethod
def create_manifest_with_temp_tag(self, repository_ref, manifest_interface_instance,
expiration_sec, storage):
""" Creates a manifest under the repository and sets a temporary tag to point to it.
Returns the manifest object created or None on error.
"""

33
data/registry_model/registry_oci_model.py
View file

@ -219,17 +219,6 @@ class OCIModel(SharedModel, RegistryDataInterface):
Returns a reference to the (created manifest, tag) or (None, None) on error.
"""
def _retrieve_repo_blob(digest):
blob_found = self.get_repo_blob_by_digest(repository_ref, digest, include_placements=True)
if blob_found is None:
return None
try:
return storage.get_content(blob_found.placements, blob_found.storage_path)
except IOError:
logger.exception('Could not retrieve configuration blob `%s`', digest)
return None
# Get or create the manifest itself.
created_manifest = oci.manifest.get_or_create_manifest(repository_ref._db_id,
manifest_interface_instance,
@ -460,4 +449,26 @@ class OCIModel(SharedModel, RegistryDataInterface):
retriever = RepositoryContentRetriever(manifest_row.repository_id, storage)
return parsed.get_v1_compatible_manifest(namespace_name, repo_name, tag_name, retriever)
def create_manifest_with_temp_tag(self, repository_ref, manifest_interface_instance,
expiration_sec, storage):
""" Creates a manifest under the repository and sets a temporary tag to point to it.
Returns the manifest object created or None on error.
"""
# Get or create the manifest itself.
created_manifest = oci.manifest.get_or_create_manifest(repository_ref._db_id,
manifest_interface_instance,
storage)
if created_manifest is None:
return None
# Point a temporary tag to the manifest.
tag = oci.tag.create_temporary_tag(created_manifest.manifest, expiration_sec)
if tag is None:
return None
legacy_image = oci.shared.get_legacy_image_for_manifest(created_manifest.manifest)
li = LegacyImage.for_image(legacy_image)
return Manifest.for_manifest(created_manifest.manifest, li)
oci_model = OCIModel()

7
data/registry_model/registry_pre_oci_model.py
View file

@ -541,5 +541,12 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
except ManifestException:
return None
def create_manifest_with_temp_tag(self, repository_ref, manifest_interface_instance,
expiration_sec, storage):
""" Creates a manifest under the repository and sets a temporary tag to point to it.
Returns the manifest object created or None on error.
"""
raise NotImplementedError('Unsupported in pre OCI model')
pre_oci_model = PreOCIModel()

88
endpoints/v2/manifest.py
View file

@ -62,7 +62,8 @@ def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
logger.exception('Got exception when trying to parse manifest `%s`', manifest_ref)
raise ManifestInvalid()
manifest = _rewrite_to_schema1_if_necessary(namespace_name, repo_name, manifest_ref, parsed)
manifest = _rewrite_to_schema1_if_necessary(namespace_name, repo_name, manifest_ref, manifest,
parsed)
if manifest is None:
raise ManifestUnknown()
@ -100,7 +101,8 @@ def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref):
logger.exception('Got exception when trying to parse manifest `%s`', manifest_ref)
raise ManifestInvalid()
manifest = _rewrite_to_schema1_if_necessary(namespace_name, repo_name, '$digest', parsed)
manifest = _rewrite_to_schema1_if_necessary(namespace_name, repo_name, '$digest', manifest,
parsed)
if manifest is None:
raise ManifestUnknown()
@ -113,24 +115,17 @@ def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref):
})
def _rewrite_to_schema1_if_necessary(namespace_name, repo_name, tag_name, manifest):
def _rewrite_to_schema1_if_necessary(namespace_name, repo_name, tag_name, manifest, parsed):
# As per the Docker protocol, if the manifest is not schema version 1 and the manifest's
# media type is not in the Accept header, we return a schema 1 version of the manifest for
# the amd64+linux platform, if any, or None if none.
# See: https://docs.docker.com/registry/spec/manifest-v2-2
mimetypes = [mimetype for mimetype, _ in request.accept_mimetypes]
if manifest.media_type in mimetypes:
return manifest
if parsed.media_type in mimetypes:
return parsed
def lookup_fn(config_or_manifest_digest):
blob = registry_model.get_cached_repo_blob(model_cache, namespace_name, repo_name,
config_or_manifest_digest)
if blob is None:
return None
return storage.get_content(blob.placements, blob.storage_path)
return manifest.get_v1_compatible_manifest(namespace_name, repo_name, tag_name, lookup_fn)
return registry_model.get_schema1_parsed_manifest(manifest, namespace_name, repo_name, tag_name,
storage)
def _reject_manifest2_schema2(func):
@ -162,19 +157,8 @@ def _doesnt_accept_schema_v1():
@require_repo_write
@anon_protect
def write_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
content_type = request.content_type or DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
if content_type == 'application/json':
# For back-compat.
content_type = DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
try:
manifest = parse_manifest_from_bytes(request.data, content_type)
except ManifestException as me:
logger.exception("failed to parse manifest when writing by tagname")
raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message})
return _write_manifest_and_log(namespace_name, repo_name, manifest_ref, manifest)
parsed = _parse_manifest()
return _write_manifest_and_log(namespace_name, repo_name, manifest_ref, parsed)
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['PUT'])
@ -184,16 +168,50 @@ def write_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
@require_repo_write
@anon_protect
def write_manifest_by_digest(namespace_name, repo_name, manifest_ref):
try:
manifest = DockerSchema1Manifest(request.data)
except ManifestException as me:
logger.exception("failed to parse manifest when writing by digest")
raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message})
if manifest.digest != manifest_ref:
parsed = _parse_manifest()
if parsed.digest != manifest_ref:
raise ManifestInvalid(detail={'message': 'manifest digest mismatch'})
return _write_manifest_and_log(namespace_name, repo_name, manifest.tag, manifest)
if parsed.schema_version != 2:
return _write_manifest_and_log(namespace_name, repo_name, parsed.tag, parsed)
# If the manifest is schema version 2, then this cannot be a normal tag-based push, as the
# manifest does not contain the tag and this call was not given a tag name. Instead, we write the
# manifest with a temporary tag, as it is being pushed as part of a call for a manifest list.
repository_ref = registry_model.lookup_repository(namespace_name, repo_name)
if repository_ref is None:
raise NameUnknown()
expiration_sec = app.config['PUSH_TEMP_TAG_EXPIRATION_SEC']
manifest = registry_model.create_manifest_with_temp_tag(repository_ref, parsed, expiration_sec,
storage)
if manifest is None:
raise ManifestInvalid()
return Response(
'OK',
status=202,
headers={
'Docker-Content-Digest': manifest.digest,
'Location':
url_for('v2.fetch_manifest_by_digest',
repository='%s/%s' % (namespace_name, repo_name),
manifest_ref=manifest.digest),
},
)
def _parse_manifest():
content_type = request.content_type or DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
if content_type == 'application/json':
# For back-compat.
content_type = DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
try:
return parse_manifest_from_bytes(request.data, content_type)
except ManifestException as me:
logger.exception("failed to parse manifest when writing by tagname")
raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message})
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE'])