From ca883e56622b478ecbaab9c39a00314992774bad Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 20 Sep 2016 22:09:25 -0400 Subject: [PATCH] port label support to refactored v2 registry --- data/database.py | 17 ++++++++++++++++- data/interfaces/v2.py | 34 +++++++++++++++++++++++++++++++--- data/model/label.py | 3 ++- endpoints/v2/manifest.py | 18 ++++++++++++++---- image/docker/schema1.py | 5 +++-- 5 files changed, 66 insertions(+), 11 deletions(-) diff --git a/data/database.py b/data/database.py index d94c42b6f..f3c5912b8 100644 --- a/data/database.py +++ b/data/database.py @@ -957,7 +957,7 @@ class ServiceKey(BaseModel): rotation_duration = IntegerField(null=True) approval = ForeignKeyField(ServiceKeyApproval, null=True) -''' + class MediaType(BaseModel): """ MediaType is an enumeration of the possible formats of various objects in the data model. """ name = CharField(index=True, unique=True) @@ -992,6 +992,21 @@ class TagManifestLabel(BaseModel): (('annotated', 'label'), True), ) + +''' + +class ManifestLabel(BaseModel): + repository = ForeignKeyField(Repository, index=True) + annotated = ForeignKeyField(Manifest, index=True) + label = ForeignKeyField(Label) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('repository', 'annotated', 'label'), True), + ) + class Blob(BaseModel): """ Blob represents a content-addressable object stored outside of the database. """ digest = CharField(index=True, unique=True) diff --git a/data/interfaces/v2.py b/data/interfaces/v2.py index 891fe08fd..dce6baa28 100644 --- a/data/interfaces/v2.py +++ b/data/interfaces/v2.py @@ -4,7 +4,7 @@ from namedlist import namedlist from peewee import IntegrityError from data import model, database -from data.model import DataModelException +from data.model import DataModelException, TagAlreadyCreatedException from image.docker.v1 import DockerV1Metadata _MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v1+prettyjws" @@ -48,6 +48,11 @@ class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'name RepositoryReference represents a reference to a Repository, without its full metadata. """ +class Label(namedtuple('Label', ['key', 'value', 'source_type', 'media_type'])): + """ + Label represents a key-value pair that describes a particular Manifest. + """ + class DockerRegistryV2DataInterface(object): """ @@ -158,6 +163,8 @@ class DockerRegistryV2DataInterface(object): """ Saves a manifest pointing to the given leaf image, with the given manifest, under the matching repository as a tag with the given name. + + Returns a boolean whether or not the tag was newly created or not. """ raise NotImplementedError() @@ -246,6 +253,14 @@ class DockerRegistryV2DataInterface(object): """ raise NotImplementedError() + @classmethod + def create_manifest_labels(cls, namespace_name, repo_name, manifest_digest, labels): + """ + Creates a new labels for the provided manifest. + """ + raise NotImplementedError() + + @classmethod def get_blob_path(cls, blob): """ @@ -407,8 +422,10 @@ class PreOCIModel(DockerRegistryV2DataInterface): @classmethod def save_manifest(cls, namespace_name, repo_name, tag_name, leaf_layer_docker_id, manifest_digest, manifest_bytes): - model.tag.store_tag_manifest(namespace_name, repo_name, tag_name, leaf_layer_docker_id, - manifest_digest, manifest_bytes) + (_, newly_created) = model.tag.store_tag_manifest(namespace_name, repo_name, tag_name, + leaf_layer_docker_id, manifest_digest, + manifest_bytes) + return newly_created @classmethod def repository_tags(cls, namespace_name, repo_name, limit, offset): @@ -540,6 +557,17 @@ class PreOCIModel(DockerRegistryV2DataInterface): blob_record = model.storage.get_storage_by_uuid(blob.uuid) model.storage.save_torrent_info(blob_record, piece_size, piece_bytes) + @classmethod + def create_manifest_labels(cls, namespace_name, repo_name, manifest_digest, labels): + if not labels: + # No point in doing anything more. + return + + tag_manifest = model.tag.load_manifest_by_digest(namespace_name, repo_name, manifest_digest) + for label in labels: + model.label.create_manifest_label(tag_manifest, label.key, label.value, label.source_type, + label.media_type) + @classmethod def get_blob_path(cls, blob): blob_record = model.storage.get_storage_by_uuid(blob.uuid) diff --git a/data/model/label.py b/data/model/label.py index ad5eadc7d..bd783e168 100644 --- a/data/model/label.py +++ b/data/model/label.py @@ -1,11 +1,12 @@ import logging +from cachetools import lru_cache + from data.database import Label, TagManifestLabel, MediaType, LabelSourceType, db_transaction from data.model import InvalidLabelKeyException, InvalidMediaTypeException, DataModelException from data.model._basequery import prefix_search from util.validation import validate_label_key from util.validation import is_json -from cachetools import lru_cache logger = logging.getLogger(__name__) diff --git a/endpoints/v2/manifest.py b/endpoints/v2/manifest.py index 4acc6abc2..2cc716103 100644 --- a/endpoints/v2/manifest.py +++ b/endpoints/v2/manifest.py @@ -8,7 +8,7 @@ import features from app import docker_v2_signing_key, app, metric_queue from auth.registry_jwt_auth import process_registry_jwt_auth -from data.interfaces.v2 import PreOCIModel as model +from data.interfaces.v2 import PreOCIModel as model, Label from digest import digest_tools from endpoints.common import parse_repository_name from endpoints.decorators import anon_protect @@ -20,8 +20,9 @@ from endpoints.notificationhelper import spawn_notification from image.docker import ManifestException from image.docker.schema1 import DockerSchema1Manifest, DockerSchema1ManifestBuilder from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES -from util.registry.replication import queue_storage_replication from util.names import VALID_TAG_PATTERN +from util.registry.replication import queue_storage_replication +from util.validation import is_json logger = logging.getLogger(__name__) @@ -175,8 +176,14 @@ def _write_manifest(namespace_name, repo_name, manifest): # Store the manifest pointing to the tag. leaf_layer_id = rewritten_images[-1].image_id - model.save_manifest(namespace_name, repo_name, manifest.tag, leaf_layer_id, manifest.digest, - manifest.bytes) + newly_created = model.save_manifest(namespace_name, repo_name, manifest.tag, leaf_layer_id, + manifest.digest, manifest.bytes) + if newly_created: + labels = [] + for key, value in manifest.layers[-1].v1_metadata.labels.iteritems(): + media_type = 'application/json' if is_json(value) else 'text/plain' + labels.append(Label(key=key, value=value, source_type='manifest', media_type=media_type)) + model.create_manifest_labels(namespace_name, repo_name, manifest.digest, labels) return repo, storage_map @@ -257,3 +264,6 @@ def _generate_and_store_manifest(namespace_name, repo_name, tag_name): model.create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest.digest, manifest.bytes) return manifest + +def _determine_media_type(value): + media_type_name = 'application/json' if is_json(value) else 'text/plain' diff --git a/image/docker/schema1.py b/image/docker/schema1.py index 6e54ef3f4..72b1aa8d2 100644 --- a/image/docker/schema1.py +++ b/image/docker/schema1.py @@ -76,7 +76,7 @@ class Schema1Layer(namedtuple('Schema1Layer', ['digest', 'v1_metadata', 'raw_v1_ class Schema1V1Metadata(namedtuple('Schema1V1Metadata', ['image_id', 'parent_image_id', 'created', - 'comment', 'command'])): + 'comment', 'command', 'labels'])): """ Represents the necessary data extracted from the v1 compatibility string in a given layer of a Manifest. @@ -207,9 +207,10 @@ class DockerSchema1Manifest(object): if not 'id' in v1_metadata: raise MalformedSchema1Manifest('id field missing from v1Compatibility JSON') + labels = v1_metadata.get('config', {}).get('Labels', {}) or {} extracted = Schema1V1Metadata(v1_metadata['id'], v1_metadata.get('parent'), v1_metadata.get('created'), v1_metadata.get('comment'), - command) + command, labels) yield Schema1Layer(image_digest, extracted, metadata_string) @property