diff --git a/app.py b/app.py index 9060b38a4..dc9671001 100644 --- a/app.py +++ b/app.py @@ -40,6 +40,7 @@ from util.config.superusermanager import SuperUserManager from util.secscan.api import SecurityScannerAPI from util.metrics.metricqueue import MetricQueue from util.metrics.prometheus import PrometheusPlugin +from util.label_validator import LabelValidator OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/' OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml' @@ -183,6 +184,7 @@ userevents = UserEventsBuilderModule(app) superusers = SuperUserManager(app) signer = Signer(app, config_provider) instance_keys = InstanceKeys(app) +label_validator = LabelValidator(app) start_cloudwatch_sender(metric_queue, app) diff --git a/config.py b/config.py index 7bb083710..e0ae3f8fb 100644 --- a/config.py +++ b/config.py @@ -357,3 +357,15 @@ class DefaultConfig(object): # URL that specifies the location of the prometheus stats aggregator. PROMETHEUS_AGGREGATOR_URL = 'http://localhost:9092' + + # Reverse DNS prefixes that are reserved for internal use on labels and should not be allowable + # to be set via the API. + DEFAULT_LABEL_KEY_RESERVED_PREFIXES = ['com.docker.', 'io.docker.', 'org.dockerproject.', + 'org.opencontainers.', 'io.cncf.', + 'io.kubernetes.', 'io.k8s.', + 'io.quay', 'com.coreos', 'com.tectonic', + 'internal', 'quay'] + + # Overridable list of reverse DNS prefixes that are reserved for internal use on labels. + LABEL_KEY_RESERVED_PREFIXES = [] + diff --git a/data/database.py b/data/database.py index 94d469f07..367dac336 100644 --- a/data/database.py +++ b/data/database.py @@ -423,8 +423,7 @@ class Repository(BaseModel): # These models don't need to use transitive deletes, because the referenced objects # are cleaned up directly skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger, BlobUpload, - Image, TagManifest, DerivedStorageForImage} - + Image, TagManifest, TagManifestLabel, Label, DerivedStorageForImage} delete_instance_filtered(self, Repository, delete_nullable, skip_transitive_deletes) @@ -920,5 +919,40 @@ class ServiceKey(BaseModel): approval = ForeignKeyField(ServiceKeyApproval, null=True) +class MediaType(BaseModel): + """ MediaType is an enumeration of the possible formats of various objects in the data model. """ + name = CharField(index=True, unique=True) + + +class LabelSourceType(BaseModel): + """ LabelSourceType is an enumeration of the possible sources for a label. """ + name = CharField(index=True, unique=True) + mutable = BooleanField(default=False) + + +class Label(BaseModel): + """ Label represents user-facing metadata associated with another entry in the + database (e.g. a Manifest). """ + uuid = CharField(default=uuid_generator, index=True, unique=True) + key = CharField(index=True) + value = TextField() + media_type = ForeignKeyField(MediaType) + source_type = ForeignKeyField(LabelSourceType) + + +class TagManifestLabel(BaseModel): + """ Mapping from a tag manifest to a label. """ + repository = ForeignKeyField(Repository, index=True) + annotated = ForeignKeyField(TagManifest, index=True) + label = ForeignKeyField(Label) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('annotated', 'label'), True), + ) + + is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)] diff --git a/data/migrations/versions/c9b91bee7554_add_labels_to_the_schema.py b/data/migrations/versions/c9b91bee7554_add_labels_to_the_schema.py new file mode 100644 index 000000000..21f1113ff --- /dev/null +++ b/data/migrations/versions/c9b91bee7554_add_labels_to_the_schema.py @@ -0,0 +1,89 @@ +"""Add labels to the schema + +Revision ID: c9b91bee7554 +Revises: 983247d75af3 +Create Date: 2016-08-22 15:40:25.226541 + +""" + +# revision identifiers, used by Alembic. +revision = 'c9b91bee7554' +down_revision = '983247d75af3' + +from alembic import op +import sqlalchemy as sa +from util.migrate import UTF8LongText, UTF8CharField + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('labelsourcetype', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('mutable', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_labelsourcetype')) + ) + op.create_index('labelsourcetype_name', 'labelsourcetype', ['name'], unique=True) + op.create_table('mediatype', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_mediatype')) + ) + op.create_index('mediatype_name', 'mediatype', ['name'], unique=True) + op.create_table('label', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', sa.String(length=255), nullable=False), + sa.Column('key', UTF8CharField(length=255), nullable=False), + sa.Column('value', UTF8LongText(), nullable=False), + sa.Column('media_type_id', sa.Integer(), nullable=False), + sa.Column('source_type_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_label_media_type_id_mediatype')), + sa.ForeignKeyConstraint(['source_type_id'], ['labelsourcetype.id'], name=op.f('fk_label_source_type_id_labelsourcetype')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_label')) + ) + op.create_index('label_key', 'label', ['key'], unique=False) + op.create_index('label_media_type_id', 'label', ['media_type_id'], unique=False) + op.create_index('label_source_type_id', 'label', ['source_type_id'], unique=False) + op.create_index('label_uuid', 'label', ['uuid'], unique=True) + op.create_table('tagmanifestlabel', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('repository_id', sa.Integer(), nullable=False), + sa.Column('annotated_id', sa.Integer(), nullable=False), + sa.Column('label_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['annotated_id'], ['tagmanifest.id'], name=op.f('fk_tagmanifestlabel_annotated_id_tagmanifest')), + sa.ForeignKeyConstraint(['label_id'], ['label.id'], name=op.f('fk_tagmanifestlabel_label_id_label')), + sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_tagmanifestlabel_repository_id_repository')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_tagmanifestlabel')) + ) + op.create_index('tagmanifestlabel_annotated_id', 'tagmanifestlabel', ['annotated_id'], unique=False) + op.create_index('tagmanifestlabel_annotated_id_label_id', 'tagmanifestlabel', ['annotated_id', 'label_id'], unique=True) + op.create_index('tagmanifestlabel_label_id', 'tagmanifestlabel', ['label_id'], unique=False) + op.create_index('tagmanifestlabel_repository_id', 'tagmanifestlabel', ['repository_id'], unique=False) + ### end Alembic commands ### + + op.bulk_insert(tables.logentrykind, [ + {'name':'manifest_label_add'}, + {'name':'manifest_label_delete'}, + ]) + + op.bulk_insert(tables.mediatype, [ + {'name':'text/plain'}, + {'name':'application/json'}, + ]) + + op.bulk_insert(tables.labelsourcetype, [ + {'name':'manifest', 'mutable': False}, + {'name':'api', 'mutable': True}, + {'name':'internal', 'mutable': False}, + ]) + + +def downgrade(tables): + op.execute(tables.logentrykind.delete().where(tables.logentrykind.c.name == op.inline_literal('manifest_label_add'))) + op.execute(tables.logentrykind.delete().where(tables.logentrykind.c.name == op.inline_literal('manifest_label_delete'))) + + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('tagmanifestlabel') + op.drop_table('label') + op.drop_table('mediatype') + op.drop_table('labelsourcetype') + ### end Alembic commands ### diff --git a/data/model/__init__.py b/data/model/__init__.py index b138bcf35..1aa512120 100644 --- a/data/model/__init__.py +++ b/data/model/__init__.py @@ -5,9 +5,18 @@ class DataModelException(Exception): pass +class InvalidLabelKeyException(DataModelException): + pass + + +class InvalidMediaTypeException(DataModelException): + pass + + class BlobDoesNotExist(DataModelException): pass + class TorrentInfoDoesNotExist(DataModelException): pass @@ -105,7 +114,23 @@ config = Config() # There MUST NOT be any circular dependencies between these subsections. If there are fix it by # moving the minimal number of things to _basequery -# TODO document the methods and modules for each one of the submodules below. -from data.model import (blob, build, image, log, notification, oauth, organization, permission, - repository, service_keys, storage, tag, team, token, user, release, - modelutil) +from data.model import ( + blob, + build, + image, + label, + log, + modelutil, + notification, + oauth, + organization, + permission, + release, + repository, + service_keys, + storage, + tag, + team, + token, + user, +) diff --git a/data/model/label.py b/data/model/label.py new file mode 100644 index 000000000..ad5eadc7d --- /dev/null +++ b/data/model/label.py @@ -0,0 +1,121 @@ +import logging + +from data.database import Label, TagManifestLabel, MediaType, LabelSourceType, db_transaction +from data.model import InvalidLabelKeyException, InvalidMediaTypeException, DataModelException +from data.model._basequery import prefix_search +from util.validation import validate_label_key +from util.validation import is_json +from cachetools import lru_cache + +logger = logging.getLogger(__name__) + + +@lru_cache(maxsize=1) +def get_label_source_types(): + source_type_map = {} + for kind in LabelSourceType.select(): + source_type_map[kind.id] = kind.name + source_type_map[kind.name] = kind.id + + return source_type_map + + +@lru_cache(maxsize=1) +def get_media_types(): + media_type_map = {} + for kind in MediaType.select(): + media_type_map[kind.id] = kind.name + media_type_map[kind.name] = kind.id + + return media_type_map + + +def _get_label_source_type_id(name): + kinds = get_label_source_types() + return kinds[name] + + +def _get_media_type_id(name): + kinds = get_media_types() + return kinds[name] + + +def create_manifest_label(tag_manifest, key, value, source_type_name, media_type_name=None): + """ Creates a new manifest label on a specific tag manifest. """ + if not key: + raise InvalidLabelKeyException() + + # Note that we don't prevent invalid label names coming from the manifest to be stored, as Docker + # does not currently prevent them from being put into said manifests. + if not validate_label_key(key) and source_type_name != 'manifest': + raise InvalidLabelKeyException() + + # Find the matching media type. If none specified, we infer. + if media_type_name is None: + media_type_name = 'text/plain' + if is_json(value): + media_type_name = 'application/json' + + media_type_id = _get_media_type_id(media_type_name) + if media_type_id is None: + raise InvalidMediaTypeException + + source_type_id = _get_label_source_type_id(source_type_name) + + with db_transaction(): + label = Label.create(key=key, value=value, source_type=source_type_id, media_type=media_type_id) + TagManifestLabel.create(annotated=tag_manifest, label=label, + repository=tag_manifest.tag.repository) + + return label + + +def list_manifest_labels(tag_manifest, prefix_filter=None): + """ Lists all labels found on the given tag manifest. """ + query = (Label.select(Label, MediaType) + .join(MediaType) + .switch(Label) + .join(LabelSourceType) + .switch(Label) + .join(TagManifestLabel) + .where(TagManifestLabel.annotated == tag_manifest)) + + if prefix_filter is not None: + query = query.where(prefix_search(Label.key, prefix_filter)) + + return query + + +def get_manifest_label(label_uuid, tag_manifest): + """ Retrieves the manifest label on the tag manifest with the given ID. """ + try: + return (Label.select(Label, LabelSourceType) + .join(LabelSourceType) + .where(Label.uuid == label_uuid) + .switch(Label) + .join(TagManifestLabel) + .where(TagManifestLabel.annotated == tag_manifest) + .get()) + except Label.DoesNotExist: + return None + + +def delete_manifest_label(label_uuid, tag_manifest): + """ Deletes the manifest label on the tag manifest with the given ID. """ + + # Find the label itself. + label = get_manifest_label(label_uuid, tag_manifest) + if label is None: + return None + + if not label.source_type.mutable: + raise DataModelException('Cannot delete immutable label') + + # Delete the mapping record and label. + deleted_count = TagManifestLabel.delete().where(TagManifestLabel.label == label).execute() + if deleted_count != 1: + logger.warning('More than a single label deleted for matching label %s', label_uuid) + + label.delete_instance(recursive=False) + return label + diff --git a/data/model/repository.py b/data/model/repository.py index ca4ff15a9..ebba1029f 100644 --- a/data/model/repository.py +++ b/data/model/repository.py @@ -10,7 +10,8 @@ from data.model import (DataModelException, tag, db_transaction, storage, permis from data.database import (Repository, Namespace, RepositoryTag, Star, Image, User, Visibility, RepositoryPermission, RepositoryActionCount, Role, RepositoryAuthorizedEmail, TagManifest, DerivedStorageForImage, - get_epoch_timestamp, db_random_func) + Label, TagManifestLabel, db_for_update, get_epoch_timestamp, + db_random_func) logger = logging.getLogger(__name__) @@ -50,11 +51,24 @@ def _purge_all_repository_tags(namespace_name, repository_name): raise DataModelException('Invalid repository \'%s/%s\'' % (namespace_name, repository_name)) - # Delete all manifests. + # Finds all the tags to delete. repo_tags = list(RepositoryTag.select().where(RepositoryTag.repository == repo.id)) if not repo_tags: return + # Find all labels to delete. + labels = list(TagManifestLabel + .select(TagManifestLabel.label) + .where(TagManifestLabel.repository == repo)) + + # Delete all the mapping entries. + TagManifestLabel.delete().where(TagManifestLabel.repository == repo).execute() + + # Delete all the matching labels. + if labels: + Label.delete().where(Label.id << [label.id for label in labels]).execute() + + # Delete all the manifests. TagManifest.delete().where(TagManifest.tag << repo_tags).execute() # Delete all tags. diff --git a/data/model/tag.py b/data/model/tag.py index 4b048e748..96aa34e76 100644 --- a/data/model/tag.py +++ b/data/model/tag.py @@ -5,7 +5,8 @@ from uuid import uuid4 from data.model import (image, db_transaction, DataModelException, _basequery, InvalidManifestException) from data.database import (RepositoryTag, Repository, Image, ImageStorage, Namespace, TagManifest, - RepositoryNotification, get_epoch_timestamp, db_for_update) + RepositoryNotification, Label, TagManifestLabel, get_epoch_timestamp, + db_for_update) logger = logging.getLogger(__name__) @@ -150,6 +151,22 @@ def garbage_collect_tags(repo): num_deleted_manifests = 0 if len(manifests_to_delete) > 0: + # Find the set of IDs for all the labels to delete. + labels = list(TagManifestLabel + .select(TagManifestLabel.id) + .where(TagManifestLabel.annotated << manifests_to_delete)) + + if len(labels) > 0: + # Delete the mapping entries. + (TagManifestLabel + .delete() + .where(TagManifestLabel.annotated << manifests_to_delete) + .execute()) + + # Delete the labels themselves. + Label.delete().where(Label.id << [label.id for label in labels]).execute() + + # Delete the tag manifests themselves. num_deleted_manifests = (TagManifest .delete() .where(TagManifest.id << manifests_to_delete) @@ -234,15 +251,22 @@ def revert_tag(repo_obj, tag_name, docker_image_id): def store_tag_manifest(namespace, repo_name, tag_name, docker_image_id, manifest_digest, manifest_data): + """ Stores a tag manifest for a specific tag name in the database. Returns the TagManifest + object, as well as a boolean indicating whether the TagManifest was created or updated. + """ with db_transaction(): tag = create_or_update_tag(namespace, repo_name, tag_name, docker_image_id) try: manifest = TagManifest.get(digest=manifest_digest) + if manifest.tag == tag: + return manifest, False + manifest.tag = tag manifest.save() + return manifest, True except TagManifest.DoesNotExist: - return TagManifest.create(tag=tag, digest=manifest_digest, json_data=manifest_data) + return TagManifest.create(tag=tag, digest=manifest_digest, json_data=manifest_data), True def get_active_tag(namespace, repo_name, tag_name): @@ -282,7 +306,7 @@ def load_manifest_by_digest(namespace, repo_name, digest): def _load_repo_manifests(namespace, repo_name): return _tag_alive(TagManifest - .select(TagManifest, RepositoryTag) + .select(TagManifest, RepositoryTag, Repository) .join(RepositoryTag) .join(Repository) .join(Namespace, on=(Namespace.id == Repository.namespace_user)) diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index ccb83e554..3f0241b7d 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -364,6 +364,7 @@ import endpoints.api.discovery import endpoints.api.error import endpoints.api.image import endpoints.api.logs +import endpoints.api.manifest import endpoints.api.organization import endpoints.api.permission import endpoints.api.prototype diff --git a/endpoints/api/manifest.py b/endpoints/api/manifest.py new file mode 100644 index 000000000..2a0daddd3 --- /dev/null +++ b/endpoints/api/manifest.py @@ -0,0 +1,159 @@ +""" Manage the manifests of a repository. """ + +from app import label_validator +from flask import request +from endpoints.api import (resource, nickname, require_repo_read, require_repo_write, + RepositoryParamResource, log_action, validate_json_request, + path_param, parse_args, query_param, truthy_bool, abort, api) +from endpoints.exception import NotFound +from data import model + +from digest import digest_tools + +BASE_MANIFEST_ROUTE = '/v1/repository//manifest/' +MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN) +ALLOWED_LABEL_MEDIA_TYPES = ['text/plain', 'application/json'] + +def label_view(label): + view = { + 'id': label.uuid, + 'key': label.key, + 'value': label.value, + 'source_type': label.source_type.name, + 'media_type': label.media_type.name, + } + + return view + +@resource(MANIFEST_DIGEST_ROUTE + '/labels') +@path_param('repository', 'The full path of the repository. e.g. namespace/name') +@path_param('manifestref', 'The digest of the manifest') +class RepositoryManifestLabels(RepositoryParamResource): + """ Resource for listing the labels on a specific repository manifest. """ + schemas = { + 'AddLabel': { + 'type': 'object', + 'description': 'Adds a label to a manifest', + 'required': [ + 'key', + 'value', + 'media_type', + ], + 'properties': { + 'key': { + 'type': 'string', + 'description': 'The key for the label', + }, + 'value': { + 'type': 'string', + 'description': 'The value for the label', + }, + 'media_type': { + 'type': ['string'], + 'description': 'The media type for this label', + 'enum': ALLOWED_LABEL_MEDIA_TYPES, + }, + }, + }, + } + + @require_repo_read + @nickname('listManifestLabels') + @parse_args() + @query_param('filter', 'If specified, only labels matching the given prefix will be returned', + type=str, default=None) + def get(self, namespace, repository, manifestref, parsed_args): + try: + tag_manifest = model.tag.load_manifest_by_digest(namespace, repository, manifestref) + except model.DataModelException: + raise NotFound() + + labels = model.label.list_manifest_labels(tag_manifest, prefix_filter=parsed_args['filter']) + return { + 'labels': [label_view(label) for label in labels] + } + + @require_repo_write + @nickname('addManifestLabel') + @validate_json_request('AddLabel') + def post(self, namespace, repository, manifestref): + """ Adds a new label into the tag manifest. """ + try: + tag_manifest = model.tag.load_manifest_by_digest(namespace, repository, manifestref) + except model.DataModelException: + raise NotFound() + + label_data = request.get_json() + + # Check for any reserved prefixes. + if label_validator.has_reserved_prefix(label_data['key']): + abort(400, message='Label has a reserved prefix') + + label = model.label.create_manifest_label(tag_manifest, label_data['key'], + label_data['value'], 'api', + media_type_name=label_data['media_type']) + metadata = { + 'id': label.uuid, + 'key': label_data['key'], + 'value': label_data['value'], + 'manifest_digest': manifestref, + 'media_type': label_data['media_type'], + } + + log_action('manifest_label_add', namespace, metadata, repo=tag_manifest.tag.repository) + + resp = {'label': label_view(label)} + repo_string = '%s/%s' % (namespace, repository) + headers = { + 'Location': api.url_for(ManageRepositoryManifestLabel, repository=repo_string, + manifestref=manifestref, labelid=label.uuid), + } + return resp, 201, headers + + +@resource(MANIFEST_DIGEST_ROUTE + '/labels/') +@path_param('repository', 'The full path of the repository. e.g. namespace/name') +@path_param('manifestref', 'The digest of the manifest') +@path_param('labelid', 'The ID of the label') +class ManageRepositoryManifestLabel(RepositoryParamResource): + """ Resource for managing the labels on a specific repository manifest. """ + @require_repo_read + @nickname('getManifestLabel') + @parse_args() + def get(self, namespace, repository, manifestref, labelid): + """ Retrieves the label with the specific ID under the manifest. """ + try: + tag_manifest = model.tag.load_manifest_by_digest(namespace, repository, manifestref) + except model.DataModelException: + raise NotFound() + + label = model.label.get_manifest_label(labelid, tag_manifest) + if label is None: + raise NotFound() + + return label_view(label) + + + @require_repo_write + @nickname('deleteManifestLabel') + def delete(self, namespace, repository, manifestref, labelid): + """ Deletes an existing label from a manifest. """ + try: + tag_manifest = model.tag.load_manifest_by_digest(namespace, repository, manifestref) + except model.DataModelException: + raise NotFound() + + deleted = model.label.delete_manifest_label(labelid, tag_manifest) + if deleted is None: + raise NotFound() + + metadata = { + 'id': labelid, + 'key': deleted.key, + 'value': deleted.value, + 'manifest_digest': manifestref + } + + log_action('manifest_label_delete', namespace, metadata, repo=tag_manifest.tag.repository) + return 'Deleted', 204 + diff --git a/endpoints/v2/manifest.py b/endpoints/v2/manifest.py index c30ae63f5..f68baa984 100644 --- a/endpoints/v2/manifest.py +++ b/endpoints/v2/manifest.py @@ -47,7 +47,7 @@ JWS_ALGORITHM = 'RS256' ImageMetadata = namedtuple('ImageMetadata', ['digest', 'v1_metadata', 'v1_metadata_str']) ExtractedV1Metadata = namedtuple('ExtractedV1Metadata', ['docker_id', 'parent', 'created', - 'comment', 'command']) + 'comment', 'command', 'labels']) _SIGNATURES_KEY = 'signatures' @@ -144,9 +144,10 @@ class SignedManifest(object): if not 'id' in v1_metadata: raise ManifestInvalid(detail={'message': 'invalid manifest v1 history'}) + labels = v1_metadata.get('config', {}).get('Labels', {}) or {} extracted = ExtractedV1Metadata(v1_metadata['id'], v1_metadata.get('parent'), v1_metadata.get('created'), v1_metadata.get('comment'), - command) + command, labels) yield ImageMetadata(image_digest, extracted, metadata_string) @property @@ -450,8 +451,12 @@ def _write_manifest_itself(namespace_name, repo_name, manifest): # Store the manifest pointing to the tag. manifest_digest = manifest.digest leaf_layer_id = images_map[layers[-1].v1_metadata.docker_id].docker_image_id - model.tag.store_tag_manifest(namespace_name, repo_name, tag_name, leaf_layer_id, manifest_digest, - manifest.bytes) + tag_manifest, manifest_created = model.tag.store_tag_manifest(namespace_name, repo_name, tag_name, + leaf_layer_id, manifest_digest, + manifest.bytes) + if manifest_created: + for key, value in layers[-1].v1_metadata.labels.iteritems(): + model.label.create_manifest_label(tag_manifest, key, value, 'manifest') # Queue all blob manifests for replication. # TODO(jschorr): Find a way to optimize this insertion. diff --git a/initdb.py b/initdb.py index 0e6fab37a..69ccb38e5 100644 --- a/initdb.py +++ b/initdb.py @@ -12,9 +12,6 @@ from peewee import (SqliteDatabase, create_model_tables, drop_model_tables, save from itertools import count from uuid import UUID, uuid4 from threading import Event -from hashlib import sha256 -from Crypto.PublicKey import RSA -from jwkest.jwk import RSAKey from email.utils import formatdate from data.database import (db, all_models, Role, TeamRole, Visibility, LoginService, @@ -22,7 +19,7 @@ from data.database import (db, all_models, Role, TeamRole, Visibility, LoginServ ImageStorageTransformation, ImageStorageSignatureKind, ExternalNotificationEvent, ExternalNotificationMethod, NotificationKind, QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode, - ServiceKeyApprovalType) + ServiceKeyApprovalType, MediaType, LabelSourceType) from data import model from data.queue import WorkQueue from app import app, storage as store, tf @@ -345,6 +342,9 @@ def initialize_database(): LogEntryKind.create(name='take_ownership') + LogEntryKind.create(name='manifest_label_add') + LogEntryKind.create(name='manifest_label_delete') + ImageStorageLocation.create(name='local_eu') ImageStorageLocation.create(name='local_us') @@ -389,6 +389,13 @@ def initialize_database(): QuayRegion.create(name='us') QuayService.create(name='quay') + MediaType.create(name='text/plain') + MediaType.create(name='application/json') + + LabelSourceType.create(name='manifest') + LabelSourceType.create(name='api', mutable=True) + LabelSourceType.create(name='internal') + def wipe_database(): logger.debug('Wiping all data from the DB.') @@ -474,6 +481,24 @@ def populate_database(minimal=False, with_storage=False): simple_repo = __generate_repository(with_storage, new_user_1, 'simple', 'Simple repository.', False, [], (4, [], ['latest', 'prod'])) + + # Add some labels to the latest tag's manifest. + tag_manifest = model.tag.load_tag_manifest(new_user_1.username, 'simple', 'latest') + first_label = model.label.create_manifest_label(tag_manifest, 'foo', 'bar', 'manifest') + model.label.create_manifest_label(tag_manifest, 'foo', 'baz', 'api') + model.label.create_manifest_label(tag_manifest, 'anotherlabel', '1234', 'internal') + + label_metadata = { + 'key': 'foo', + 'value': 'bar', + 'id': first_label.id, + 'manifest_digest': tag_manifest.digest + } + + model.log.log_action('manifest_label_add', new_user_1.username, performer=new_user_1, + timestamp=datetime.now(), metadata=label_metadata, + repository=tag_manifest.tag.repository) + model.blob.initiate_upload(new_user_1.username, simple_repo.name, str(uuid4()), 'local_us', {}) model.notification.create_repo_notification(simple_repo, 'repo_push', 'quay_notification', {}, {}) diff --git a/static/js/directives/ui/logs-view.js b/static/js/directives/ui/logs-view.js index 4c6321c63..a7439ca7b 100644 --- a/static/js/directives/ui/logs-view.js +++ b/static/js/directives/ui/logs-view.js @@ -230,6 +230,9 @@ angular.module('quay').directive('logsView', function () { } }, + 'manifest_label_add': 'Label {key} added to manifest {manifest_digest}', + 'manifest_label_delete': 'Label {key} deleted from manifest {manifest_digest}', + // Note: These are deprecated. 'add_repo_webhook': 'Add webhook in repository {repo}', 'delete_repo_webhook': 'Delete webhook in repository {repo}' @@ -287,6 +290,8 @@ angular.module('quay').directive('logsView', function () { 'service_key_extend': 'Extend Service Key Expiration', 'service_key_rotate': 'Automatic rotation of Service Key', 'take_ownership': 'Take Namespace Ownership', + 'manifest_label_add': 'Add Manifest Label', + 'manifest_label_delete': 'Delete Manifest Label', // Note: these are deprecated. 'add_repo_webhook': 'Add webhook', diff --git a/static/js/services/string-builder-service.js b/static/js/services/string-builder-service.js index 87fe2cd66..cb5ba19f1 100644 --- a/static/js/services/string-builder-service.js +++ b/static/js/services/string-builder-service.js @@ -22,7 +22,8 @@ angular.module('quay').factory('StringBuilderService', ['$sce', 'UtilService', f 'application_name': 'cloud', 'image': 'archive', 'original_image': 'archive', - 'client_id': 'chain' + 'client_id': 'chain', + 'manifest_digest': 'link' }; var filters = { diff --git a/test/data/test.db b/test/data/test.db index 6ceefdff8..183613dcf 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/test/helpers.py b/test/helpers.py index 0fb535427..ee35152aa 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -16,6 +16,7 @@ class assert_action_logged(object): return self def __exit__(self, exc_type, exc_val, exc_tb): - updated_count = self._get_log_count() - error_msg = 'Missing new log entry of kind %s' % self.log_kind - assert self.existing_count == (updated_count - 1), error_msg + if exc_val is None: + updated_count = self._get_log_count() + error_msg = 'Missing new log entry of kind %s' % self.log_kind + assert self.existing_count == (updated_count - 1), error_msg diff --git a/test/registry_tests.py b/test/registry_tests.py index 19a0c9544..538475333 100644 --- a/test/registry_tests.py +++ b/test/registry_tests.py @@ -1129,6 +1129,55 @@ class V1RegistryTests(V1RegistryPullMixin, V1RegistryPushMixin, RegistryTestsMix class V2RegistryTests(V2RegistryPullMixin, V2RegistryPushMixin, RegistryTestsMixin, RegistryTestCaseMixin, LiveServerTestCase): """ Tests for V2 registry. """ + def test_label_invalid_manifest(self): + images = [{ + 'id': 'someid', + 'config': {'Labels': None}, + 'contents': 'somecontent' + }] + + self.do_push('devtable', 'newrepo', 'devtable', 'password', images=images) + self.do_pull('devtable', 'newrepo', 'devtable', 'password') + + def test_labels(self): + # Push a new repo with the latest tag. + images = [{ + 'id': 'someid', + 'config': {'Labels': {'foo': 'bar', 'baz': 'meh', 'theoretically-invalid--label': 'foo'}}, + 'contents': 'somecontent' + }] + + (_, digest) = self.do_push('devtable', 'newrepo', 'devtable', 'password', images=images) + + self.conduct_api_login('devtable', 'password') + labels = self.conduct('GET', '/api/v1/repository/devtable/newrepo/manifest/' + digest + '/labels').json() + self.assertEquals(3, len(labels['labels'])) + + self.assertEquals('manifest', labels['labels'][0]['source_type']) + self.assertEquals('manifest', labels['labels'][1]['source_type']) + self.assertEquals('manifest', labels['labels'][2]['source_type']) + + self.assertEquals('text/plain', labels['labels'][0]['media_type']) + self.assertEquals('text/plain', labels['labels'][1]['media_type']) + self.assertEquals('text/plain', labels['labels'][2]['media_type']) + + def test_json_labels(self): + # Push a new repo with the latest tag. + images = [{ + 'id': 'someid', + 'config': {'Labels': {'foo': 'bar', 'baz': '{"some": "json"}'}}, + 'contents': 'somecontent' + }] + + (_, digest) = self.do_push('devtable', 'newrepo', 'devtable', 'password', images=images) + + self.conduct_api_login('devtable', 'password') + labels = self.conduct('GET', '/api/v1/repository/devtable/newrepo/manifest/' + digest + '/labels').json() + self.assertEquals(2, len(labels['labels'])) + + self.assertEquals('text/plain', labels['labels'][0]['media_type']) + self.assertEquals('application/json', labels['labels'][1]['media_type']) + def test_invalid_manifest_type(self): namespace = 'devtable' repository = 'somerepo' diff --git a/test/test_api_security.py b/test/test_api_security.py index 258efde7d..da7f23a81 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -52,6 +52,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana SuperUserServiceKey, SuperUserServiceKeyApproval, SuperUserTakeOwnership) from endpoints.api.secscan import RepositoryImageSecurity +from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel try: @@ -4298,5 +4299,54 @@ class TestRepositoryImageSecurity(ApiTestCase): self._run_test('GET', 404, 'devtable', None) +class TestRepositoryManifestLabels(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(RepositoryManifestLabels, repository='devtable/simple', manifestref='sha256:abcd') + + def test_get_anonymous(self): + self._run_test('GET', 401, None, None) + + def test_get_freshuser(self): + self._run_test('GET', 403, 'freshuser', None) + + def test_get_reader(self): + self._run_test('GET', 403, 'reader', None) + + def test_get_devtable(self): + self._run_test('GET', 404, 'devtable', None) + + def test_post_anonymous(self): + self._run_test('POST', 401, None, dict(key='foo', value='bar', media_type='text/plain')) + + def test_post_freshuser(self): + self._run_test('POST', 403, 'freshuser', dict(key='foo', value='bar', media_type='text/plain')) + + def test_post_reader(self): + self._run_test('POST', 403, 'reader', dict(key='foo', value='bar', media_type='text/plain')) + + def test_post_devtable(self): + self._run_test('POST', 404, 'devtable', dict(key='foo', value='bar', media_type='text/plain')) + + +class TestManageRepositoryManifestLabel(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(ManageRepositoryManifestLabel, repository='devtable/simple', + manifestref='sha256:abcd', labelid='someid') + + def test_delete_anonymous(self): + self._run_test('DELETE', 401, None, None) + + def test_delete_freshuser(self): + self._run_test('DELETE', 403, 'freshuser', None) + + def test_delete_reader(self): + self._run_test('DELETE', 403, 'reader', None) + + def test_delete_devtable(self): + self._run_test('DELETE', 404, 'devtable', None) + + if __name__ == '__main__': unittest.main() diff --git a/test/test_api_usage.py b/test/test_api_usage.py index f7a8dcb16..8761281f5 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -65,6 +65,8 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana from endpoints.api.secscan import RepositoryImageSecurity from endpoints.api.suconfig import (SuperUserRegistryStatus, SuperUserConfig, SuperUserConfigFile, SuperUserCreateInitialSuperUser) +from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel + try: app.register_blueprint(api_bp, url_prefix='/api') @@ -1765,6 +1767,15 @@ class TestDeleteRepository(ApiTestCase): RepositoryActionCount.create(repository=repository, date=datetime.datetime.now() - datetime.timedelta(days=5), count=6) + # Create some labels. + tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod') + model.label.create_manifest_label(tag_manifest, 'foo', 'bar', 'manifest') + model.label.create_manifest_label(tag_manifest, 'foo', 'baz', 'manifest') + model.label.create_manifest_label(tag_manifest, 'something', '{}', 'api', + media_type_name='application/json') + + model.label.create_manifest_label(tag_manifest, 'something', '{"some": "json"}', 'manifest') + # Delete the repository. with check_transitive_deletes(): self.deleteResponse(Repository, params=dict(repository=self.COMPLEX_REPO)) @@ -3941,6 +3952,150 @@ class TestSuperUserKeyManagement(ApiTestCase): self.assertEquals('whazzup!?', json['approval']['notes']) +class TestRepositoryManifestLabels(ApiTestCase): + def test_basic_labels(self): + self.login(ADMIN_ACCESS_USER) + + # Find the manifest digest for the prod tag in the complex repo. + tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod') + repository = ADMIN_ACCESS_USER + '/complex' + + # Check the existing labels on the complex repo, which should be empty + json = self.getJsonResponse(RepositoryManifestLabels, + params=dict(repository=repository, manifestref=tag_manifest.digest)) + + self.assertEquals(0, len(json['labels'])) + + # Add some labels to the manifest. + with assert_action_logged('manifest_label_add'): + label1 = self.postJsonResponse(RepositoryManifestLabels, + params=dict(repository=repository, + manifestref=tag_manifest.digest), + data=dict(key='hello', value='world', + media_type='text/plain'), + expected_code=201) + + + with assert_action_logged('manifest_label_add'): + label2 = self.postJsonResponse(RepositoryManifestLabels, + params=dict(repository=repository, + manifestref=tag_manifest.digest), + data=dict(key='hi', value='there', + media_type='text/plain'), + expected_code=201) + + with assert_action_logged('manifest_label_add'): + label3 = self.postJsonResponse(RepositoryManifestLabels, + params=dict(repository=repository, + manifestref=tag_manifest.digest), + data=dict(key='hello', value='someone', + media_type='application/json'), + expected_code=201) + + + # Ensure we have *3* labels + json = self.getJsonResponse(RepositoryManifestLabels, + params=dict(repository=repository, + manifestref=tag_manifest.digest)) + + self.assertEquals(3, len(json['labels'])) + + self.assertNotEquals(label2['label']['id'], label1['label']['id']) + self.assertNotEquals(label3['label']['id'], label1['label']['id']) + self.assertNotEquals(label2['label']['id'], label3['label']['id']) + + self.assertEquals('text/plain', label1['label']['media_type']) + self.assertEquals('text/plain', label2['label']['media_type']) + self.assertEquals('application/json', label3['label']['media_type']) + + # Delete a label. + with assert_action_logged('manifest_label_delete'): + self.deleteResponse(ManageRepositoryManifestLabel, + params=dict(repository=repository, + manifestref=tag_manifest.digest, + labelid=label1['label']['id'])) + + # Ensure the label is gone. + json = self.getJsonResponse(RepositoryManifestLabels, + params=dict(repository=repository, + manifestref=tag_manifest.digest)) + + self.assertEquals(2, len(json['labels'])) + + # Check filtering. + json = self.getJsonResponse(RepositoryManifestLabels, + params=dict(repository=repository, + manifestref=tag_manifest.digest, + filter='hello')) + + self.assertEquals(1, len(json['labels'])) + + + def test_prefixed_labels(self): + self.login(ADMIN_ACCESS_USER) + + # Find the manifest digest for the prod tag in the complex repo. + tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod') + repository = ADMIN_ACCESS_USER + '/complex' + + self.postJsonResponse(RepositoryManifestLabels, + params=dict(repository=repository, + manifestref=tag_manifest.digest), + data=dict(key='com.dockers.whatever', value='pants', + media_type='text/plain'), + expected_code=201) + + + self.postJsonResponse(RepositoryManifestLabels, + params=dict(repository=repository, + manifestref=tag_manifest.digest), + data=dict(key='my.cool.prefix.for.my.label', value='value', + media_type='text/plain'), + expected_code=201) + + + + def test_add_invalid_media_type(self): + self.login(ADMIN_ACCESS_USER) + + tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod') + repository = ADMIN_ACCESS_USER + '/complex' + + self.postResponse(RepositoryManifestLabels, + params=dict(repository=repository, + manifestref=tag_manifest.digest), + data=dict(key='hello', value='world', media_type='some/invalid'), + expected_code=400) + + + def test_add_invalid_key(self): + self.login(ADMIN_ACCESS_USER) + + tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod') + repository = ADMIN_ACCESS_USER + '/complex' + + # Try to add an empty label key. + self.postResponse(RepositoryManifestLabels, + params=dict(repository=repository, + manifestref=tag_manifest.digest), + data=dict(key='', value='world'), + expected_code=400) + + # Try to add an invalid label key. + self.postResponse(RepositoryManifestLabels, + params=dict(repository=repository, + manifestref=tag_manifest.digest), + data=dict(key='invalid___key', value='world'), + expected_code=400) + + # Try to add a label key in a reserved namespace. + self.postResponse(RepositoryManifestLabels, + params=dict(repository=repository, + manifestref=tag_manifest.digest), + data=dict(key='io.docker.whatever', value='world'), + expected_code=400) + + class TestSuperUserManagement(ApiTestCase): def test_get_user(self): self.login(ADMIN_ACCESS_USER) diff --git a/test/test_validation.py b/test/test_validation.py new file mode 100644 index 000000000..980e76b07 --- /dev/null +++ b/test/test_validation.py @@ -0,0 +1,44 @@ +import unittest +from util.validation import validate_label_key + +class TestLabelKeyValidation(unittest.TestCase): + def assertValidKey(self, key): + self.assertTrue(validate_label_key(key)) + + def assertInvalidKey(self, key): + self.assertFalse(validate_label_key(key)) + + def test_basic_keys(self): + self.assertValidKey('foo') + self.assertValidKey('bar') + + self.assertValidKey('foo1') + self.assertValidKey('bar2') + + self.assertValidKey('1') + self.assertValidKey('12') + self.assertValidKey('123') + self.assertValidKey('1234') + + self.assertValidKey('git-sha') + + self.assertValidKey('com.coreos.something') + self.assertValidKey('io.quay.git-sha') + + def test_invalid_keys(self): + self.assertInvalidKey('') + self.assertInvalidKey('git_sha') + + def test_must_start_with_alphanumeric(self): + self.assertInvalidKey('-125') + self.assertInvalidKey('-foo') + self.assertInvalidKey('foo-') + self.assertInvalidKey('123-') + + def test_no_double_dashesdots(self): + self.assertInvalidKey('foo--bar') + self.assertInvalidKey('foo..bar') + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/util/config/superusermanager.py b/util/config/superusermanager.py index adf26332d..2f587166e 100644 --- a/util/config/superusermanager.py +++ b/util/config/superusermanager.py @@ -1,5 +1,5 @@ from multiprocessing.sharedctypes import Array -from util.validation import MAX_LENGTH +from util.validation import MAX_USERNAME_LENGTH class SuperUserManager(object): """ In-memory helper class for quickly accessing (and updating) the valid @@ -11,7 +11,7 @@ class SuperUserManager(object): usernames = app.config.get('SUPER_USERS', []) usernames_str = ','.join(usernames) - self._max_length = len(usernames_str) + MAX_LENGTH + 1 + self._max_length = len(usernames_str) + MAX_USERNAME_LENGTH + 1 self._array = Array('c', self._max_length, lock=True) self._array.value = usernames_str diff --git a/util/label_validator.py b/util/label_validator.py new file mode 100644 index 000000000..dc7f67ae1 --- /dev/null +++ b/util/label_validator.py @@ -0,0 +1,20 @@ +class LabelValidator(object): + """ Helper class for validating that labels meet prefix requirements. """ + def __init__(self, app): + self.app = app + + overridden_prefixes = app.config.get('LABEL_KEY_RESERVED_PREFIXES', []) + for prefix in overridden_prefixes: + if not prefix.endswith('.'): + raise Exception('Prefix "%s" in LABEL_KEY_RESERVED_PREFIXES must end in a dot', prefix) + + default_prefixes = app.config.get('DEFAULT_LABEL_KEY_RESERVED_PREFIXES', []) + self.reserved_prefixed_set = set(default_prefixes + overridden_prefixes) + + def has_reserved_prefix(self, label_key): + """ Validates that the provided label key does not match any reserved prefixes. """ + for prefix in self.reserved_prefixed_set: + if label_key.startswith(prefix): + return True + + return False diff --git a/util/migrate/__init__.py b/util/migrate/__init__.py index 809bcaef8..9f72f7872 100644 --- a/util/migrate/__init__.py +++ b/util/migrate/__init__.py @@ -1,7 +1,7 @@ import logging -from sqlalchemy.types import TypeDecorator, Text -from sqlalchemy.dialects.mysql import TEXT as MySQLText, LONGTEXT +from sqlalchemy.types import TypeDecorator, Text, String +from sqlalchemy.dialects.mysql import TEXT as MySQLText, LONGTEXT, VARCHAR as MySQLString logger = logging.getLogger(__name__) @@ -20,3 +20,19 @@ class UTF8LongText(TypeDecorator): return dialect.type_descriptor(LONGTEXT(charset='utf8mb4', collation='utf8mb4_unicode_ci')) else: return dialect.type_descriptor(Text()) + + +class UTF8CharField(TypeDecorator): + """ Platform-independent UTF-8 Char type. + + Uses MySQL's VARCHAR with charset utf8mb4, otherwise uses String, because + other engines default to UTF-8. + """ + impl = String + + def load_dialect_impl(self, dialect): + if dialect.name == 'mysql': + return dialect.type_descriptor(MySQLString(charset='utf8mb4', collation='utf8mb4_unicode_ci', + length=self.impl.length)) + else: + return dialect.type_descriptor(String(length=self.impl.length)) diff --git a/util/validation.py b/util/validation.py index 2dc49cff0..1f6c5666c 100644 --- a/util/validation.py +++ b/util/validation.py @@ -1,5 +1,6 @@ import string import re +import json import anunidecode # Don't listen to pylint's lies. This import is required. @@ -9,14 +10,21 @@ INVALID_PASSWORD_MESSAGE = 'Invalid password, password must be at least ' + \ INVALID_USERNAME_CHARACTERS = r'[^a-z0-9_]' VALID_CHARACTERS = string.digits + string.lowercase -MIN_LENGTH = 4 -MAX_LENGTH = 30 +MIN_USERNAME_LENGTH = 4 +MAX_USERNAME_LENGTH = 30 + +VALID_LABEL_KEY_REGEX = r'^[a-z0-9](([a-z0-9]|[-.](?![.-]))*[a-z0-9])?$' + + +def validate_label_key(label_key): + if len(label_key) > 255: + return False + + return bool(re.match(VALID_LABEL_KEY_REGEX, label_key)) def validate_email(email_address): - if re.match(r'[^@]+@[^@]+\.[^@]+', email_address): - return True - return False + return bool(re.match(r'[^@]+@[^@]+\.[^@]+', email_address)) def validate_username(username): @@ -25,10 +33,10 @@ def validate_username(username): if not regex_match: return (False, 'Username must match expression [a-z0-9_]+') - length_match = (len(username) >= MIN_LENGTH and len(username) <= MAX_LENGTH) + length_match = (len(username) >= MIN_USERNAME_LENGTH and len(username) <= MAX_USERNAME_LENGTH) if not length_match: return (False, 'Username must be between %s and %s characters in length' % - (MIN_LENGTH, MAX_LENGTH)) + (MIN_USERNAME_LENGTH, MAX_USERNAME_LENGTH)) return (True, '') @@ -58,9 +66,20 @@ def generate_valid_usernames(input_username): if prefix.endswith('_'): prefix = prefix[0:len(prefix) - 1] - num_filler_chars = max(0, MIN_LENGTH - len(prefix)) + num_filler_chars = max(0, MIN_USERNAME_LENGTH - len(prefix)) - while num_filler_chars + len(prefix) <= MAX_LENGTH: + while num_filler_chars + len(prefix) <= MAX_USERNAME_LENGTH: for suffix in _gen_filler_chars(num_filler_chars): yield prefix + suffix num_filler_chars += 1 + + +def is_json(value): + if ((value.startswith('{') and value.endswith('}')) or + (value.startswith('[') and value.endswith(']'))): + try: + json.loads(value) + return True + except TypeError: + return False +