diff --git a/data/model/oci/tag.py b/data/model/oci/tag.py index 3ebe67f05..bb2cc85d9 100644 --- a/data/model/oci/tag.py +++ b/data/model/oci/tag.py @@ -1,3 +1,4 @@ +import uuid import logging from calendar import timegm @@ -155,6 +156,47 @@ def get_expired_tag(repository_id, tag_name): return None +def create_temporary_tag(manifest, expiration_sec): + """ Creates a temporary tag pointing to the given manifest, with the given expiration in seconds. + """ + tag_name = '$temp-%s' % str(uuid.uuid4()) + now_ms = get_epoch_timestamp_ms() + end_ms = now_ms + (expiration_sec * 1000) + + legacy_image = get_legacy_image_for_manifest(manifest) + + with db_transaction(): + created_tag = Tag.create(name=tag_name, + repository=manifest.repository_id, + lifetime_start_ms=now_ms, + lifetime_end_ms=end_ms, + reversion=False, + hidden=True, + manifest=manifest, + tag_kind=Tag.tag_kind.get_id('tag')) + + # TODO(jschorr): Remove the linkage code once RepositoryTag is gone. + # If this is a schema 1 manifest, then add a TagManifest linkage to it. Otherwise, it will only + # be pullable via the new OCI model. + if manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES and legacy_image is not None: + now_ts = int(now_ms / 1000) + end_ts = int(end_ms / 1000) + + old_style_tag = RepositoryTag.create(repository=manifest.repository_id, image=legacy_image, + name=tag_name, lifetime_start_ts=now_ts, + lifetime_end_ts=end_ts, + reversion=False, hidden=True) + TagToRepositoryTag.create(tag=created_tag, repository_tag=old_style_tag, + repository=manifest.repository_id) + + tag_manifest = TagManifest.create(tag=old_style_tag, digest=manifest.digest, + json_data=manifest.manifest_bytes) + TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=manifest, + repository=manifest.repository_id) + + return created_tag + + def retarget_tag(tag_name, manifest_id, is_reversion=False, now_ms=None): """ Creates or updates a tag with the specified name to point to the given manifest under its repository. If this action is a reversion to a previous manifest, is_reversion diff --git a/data/model/oci/test/test_oci_tag.py b/data/model/oci/test/test_oci_tag.py index 624131be5..d31256abb 100644 --- a/data/model/oci/test/test_oci_tag.py +++ b/data/model/oci/test/test_oci_tag.py @@ -10,7 +10,8 @@ from data.model.oci.tag import (find_matching_tag, get_most_recent_tag, list_ali filter_to_visible_tags, list_repository_tag_history, get_expired_tag, get_tag, delete_tag, delete_tags_for_manifest, change_tag_expiration, - set_tag_expiration_for_manifest, retarget_tag) + set_tag_expiration_for_manifest, retarget_tag, + create_temporary_tag) from data.model.repository import get_repository, create_repository from test.fixtures import * @@ -207,6 +208,31 @@ def test_set_tag_expiration_for_manifest(initialized_db): assert updated_tag.lifetime_end_ms is not None +def test_create_temporary_tag(initialized_db): + tag = Tag.get() + manifest = tag.manifest + assert manifest is not None + + created = create_temporary_tag(manifest, 30) + assert created is not None + + assert created.hidden + assert created.name.startswith('$temp-') + assert created.manifest == manifest + assert created.lifetime_end_ms is not None + assert created.lifetime_end_ms == (created.lifetime_start_ms + 30000) + + # Verify old-style tables. + repository_tag = TagToRepositoryTag.get(tag=created).repository_tag + assert repository_tag.lifetime_start_ts == int(created.lifetime_start_ms / 1000) + assert repository_tag.lifetime_end_ts == int(created.lifetime_end_ms / 1000) + assert repository_tag.name == created.name + assert repository_tag.hidden + + tag_manifest = TagManifest.get(tag=repository_tag) + assert TagManifestToManifest.get(tag_manifest=tag_manifest).manifest == manifest + + def test_retarget_tag(initialized_db): repo = get_repository('devtable', 'history') results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name='latest') diff --git a/data/registry_model/interface.py b/data/registry_model/interface.py index 917f6763c..82486f40e 100644 --- a/data/registry_model/interface.py +++ b/data/registry_model/interface.py @@ -303,3 +303,10 @@ class RegistryDataInterface(object): @abstractmethod def get_schema1_parsed_manifest(self, manifest, namespace_name, repo_name, tag_name, storage): """ Returns the schema 1 version of this manifest, or None if none. """ + + @abstractmethod + def create_manifest_with_temp_tag(self, repository_ref, manifest_interface_instance, + expiration_sec, storage): + """ Creates a manifest under the repository and sets a temporary tag to point to it. + Returns the manifest object created or None on error. + """ diff --git a/data/registry_model/registry_oci_model.py b/data/registry_model/registry_oci_model.py index cf288d0e2..bd2edac56 100644 --- a/data/registry_model/registry_oci_model.py +++ b/data/registry_model/registry_oci_model.py @@ -219,17 +219,6 @@ class OCIModel(SharedModel, RegistryDataInterface): Returns a reference to the (created manifest, tag) or (None, None) on error. """ - def _retrieve_repo_blob(digest): - blob_found = self.get_repo_blob_by_digest(repository_ref, digest, include_placements=True) - if blob_found is None: - return None - - try: - return storage.get_content(blob_found.placements, blob_found.storage_path) - except IOError: - logger.exception('Could not retrieve configuration blob `%s`', digest) - return None - # Get or create the manifest itself. created_manifest = oci.manifest.get_or_create_manifest(repository_ref._db_id, manifest_interface_instance, @@ -460,4 +449,26 @@ class OCIModel(SharedModel, RegistryDataInterface): retriever = RepositoryContentRetriever(manifest_row.repository_id, storage) return parsed.get_v1_compatible_manifest(namespace_name, repo_name, tag_name, retriever) + def create_manifest_with_temp_tag(self, repository_ref, manifest_interface_instance, + expiration_sec, storage): + """ Creates a manifest under the repository and sets a temporary tag to point to it. + Returns the manifest object created or None on error. + """ + # Get or create the manifest itself. + created_manifest = oci.manifest.get_or_create_manifest(repository_ref._db_id, + manifest_interface_instance, + storage) + if created_manifest is None: + return None + + # Point a temporary tag to the manifest. + tag = oci.tag.create_temporary_tag(created_manifest.manifest, expiration_sec) + if tag is None: + return None + + legacy_image = oci.shared.get_legacy_image_for_manifest(created_manifest.manifest) + li = LegacyImage.for_image(legacy_image) + return Manifest.for_manifest(created_manifest.manifest, li) + + oci_model = OCIModel() diff --git a/data/registry_model/registry_pre_oci_model.py b/data/registry_model/registry_pre_oci_model.py index a32b37868..9172fd134 100644 --- a/data/registry_model/registry_pre_oci_model.py +++ b/data/registry_model/registry_pre_oci_model.py @@ -541,5 +541,12 @@ class PreOCIModel(SharedModel, RegistryDataInterface): except ManifestException: return None + def create_manifest_with_temp_tag(self, repository_ref, manifest_interface_instance, + expiration_sec, storage): + """ Creates a manifest under the repository and sets a temporary tag to point to it. + Returns the manifest object created or None on error. + """ + raise NotImplementedError('Unsupported in pre OCI model') + pre_oci_model = PreOCIModel() diff --git a/endpoints/v2/manifest.py b/endpoints/v2/manifest.py index b4f3259d6..c8a81ea6f 100644 --- a/endpoints/v2/manifest.py +++ b/endpoints/v2/manifest.py @@ -62,7 +62,8 @@ def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref): logger.exception('Got exception when trying to parse manifest `%s`', manifest_ref) raise ManifestInvalid() - manifest = _rewrite_to_schema1_if_necessary(namespace_name, repo_name, manifest_ref, parsed) + manifest = _rewrite_to_schema1_if_necessary(namespace_name, repo_name, manifest_ref, manifest, + parsed) if manifest is None: raise ManifestUnknown() @@ -100,7 +101,8 @@ def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref): logger.exception('Got exception when trying to parse manifest `%s`', manifest_ref) raise ManifestInvalid() - manifest = _rewrite_to_schema1_if_necessary(namespace_name, repo_name, '$digest', parsed) + manifest = _rewrite_to_schema1_if_necessary(namespace_name, repo_name, '$digest', manifest, + parsed) if manifest is None: raise ManifestUnknown() @@ -113,24 +115,17 @@ def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref): }) -def _rewrite_to_schema1_if_necessary(namespace_name, repo_name, tag_name, manifest): +def _rewrite_to_schema1_if_necessary(namespace_name, repo_name, tag_name, manifest, parsed): # As per the Docker protocol, if the manifest is not schema version 1 and the manifest's # media type is not in the Accept header, we return a schema 1 version of the manifest for # the amd64+linux platform, if any, or None if none. # See: https://docs.docker.com/registry/spec/manifest-v2-2 mimetypes = [mimetype for mimetype, _ in request.accept_mimetypes] - if manifest.media_type in mimetypes: - return manifest + if parsed.media_type in mimetypes: + return parsed - def lookup_fn(config_or_manifest_digest): - blob = registry_model.get_cached_repo_blob(model_cache, namespace_name, repo_name, - config_or_manifest_digest) - if blob is None: - return None - - return storage.get_content(blob.placements, blob.storage_path) - - return manifest.get_v1_compatible_manifest(namespace_name, repo_name, tag_name, lookup_fn) + return registry_model.get_schema1_parsed_manifest(manifest, namespace_name, repo_name, tag_name, + storage) def _reject_manifest2_schema2(func): @@ -162,19 +157,8 @@ def _doesnt_accept_schema_v1(): @require_repo_write @anon_protect def write_manifest_by_tagname(namespace_name, repo_name, manifest_ref): - content_type = request.content_type or DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE - - if content_type == 'application/json': - # For back-compat. - content_type = DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE - - try: - manifest = parse_manifest_from_bytes(request.data, content_type) - except ManifestException as me: - logger.exception("failed to parse manifest when writing by tagname") - raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message}) - - return _write_manifest_and_log(namespace_name, repo_name, manifest_ref, manifest) + parsed = _parse_manifest() + return _write_manifest_and_log(namespace_name, repo_name, manifest_ref, parsed) @v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['PUT']) @@ -184,16 +168,50 @@ def write_manifest_by_tagname(namespace_name, repo_name, manifest_ref): @require_repo_write @anon_protect def write_manifest_by_digest(namespace_name, repo_name, manifest_ref): - try: - manifest = DockerSchema1Manifest(request.data) - except ManifestException as me: - logger.exception("failed to parse manifest when writing by digest") - raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message}) - - if manifest.digest != manifest_ref: + parsed = _parse_manifest() + if parsed.digest != manifest_ref: raise ManifestInvalid(detail={'message': 'manifest digest mismatch'}) - return _write_manifest_and_log(namespace_name, repo_name, manifest.tag, manifest) + if parsed.schema_version != 2: + return _write_manifest_and_log(namespace_name, repo_name, parsed.tag, parsed) + + # If the manifest is schema version 2, then this cannot be a normal tag-based push, as the + # manifest does not contain the tag and this call was not given a tag name. Instead, we write the + # manifest with a temporary tag, as it is being pushed as part of a call for a manifest list. + repository_ref = registry_model.lookup_repository(namespace_name, repo_name) + if repository_ref is None: + raise NameUnknown() + + expiration_sec = app.config['PUSH_TEMP_TAG_EXPIRATION_SEC'] + manifest = registry_model.create_manifest_with_temp_tag(repository_ref, parsed, expiration_sec, + storage) + if manifest is None: + raise ManifestInvalid() + + return Response( + 'OK', + status=202, + headers={ + 'Docker-Content-Digest': manifest.digest, + 'Location': + url_for('v2.fetch_manifest_by_digest', + repository='%s/%s' % (namespace_name, repo_name), + manifest_ref=manifest.digest), + }, + ) + + +def _parse_manifest(): + content_type = request.content_type or DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE + if content_type == 'application/json': + # For back-compat. + content_type = DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE + + try: + return parse_manifest_from_bytes(request.data, content_type) + except ManifestException as me: + logger.exception("failed to parse manifest when writing by tagname") + raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message}) @v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE'])