Basic labels support
Adds basic labels support to the registry code (V2), and the API. Note that this does not yet add any UI related support.
This commit is contained in:
Joseph Schorr
parent
427070b453
commit
608ffd9663
24 changed files with 907 additions and 36 deletions
2
app.py
2
app.py
|
|
@ -40,6 +40,7 @@ from util.config.superusermanager import SuperUserManager
|
|||
from util.secscan.api import SecurityScannerAPI
|
||||
from util.metrics.metricqueue import MetricQueue
|
||||
from util.metrics.prometheus import PrometheusPlugin
|
||||
from util.label_validator import LabelValidator
|
||||
|
||||
OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/'
|
||||
OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml'
|
||||
|
|
@ -183,6 +184,7 @@ userevents = UserEventsBuilderModule(app)
|
|||
superusers = SuperUserManager(app)
|
||||
signer = Signer(app, config_provider)
|
||||
instance_keys = InstanceKeys(app)
|
||||
label_validator = LabelValidator(app)
|
||||
|
||||
start_cloudwatch_sender(metric_queue, app)
|
||||
|
||||
|
|
|
|||
12
config.py
12
config.py
|
|
@ -357,3 +357,15 @@ class DefaultConfig(object):
|
|||
|
||||
# URL that specifies the location of the prometheus stats aggregator.
|
||||
PROMETHEUS_AGGREGATOR_URL = 'http://localhost:9092'
|
||||
|
||||
# Reverse DNS prefixes that are reserved for internal use on labels and should not be allowable
|
||||
# to be set via the API.
|
||||
DEFAULT_LABEL_KEY_RESERVED_PREFIXES = ['com.docker.', 'io.docker.', 'org.dockerproject.',
|
||||
'org.opencontainers.', 'io.cncf.',
|
||||
'io.kubernetes.', 'io.k8s.',
|
||||
'io.quay', 'com.coreos', 'com.tectonic',
|
||||
'internal', 'quay']
|
||||
|
||||
# Overridable list of reverse DNS prefixes that are reserved for internal use on labels.
|
||||
LABEL_KEY_RESERVED_PREFIXES = []
|
||||
|
||||
|
|
|
|||
|
|
@ -423,8 +423,7 @@ class Repository(BaseModel):
|
|||
# These models don't need to use transitive deletes, because the referenced objects
|
||||
# are cleaned up directly
|
||||
skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger, BlobUpload,
|
||||
Image, TagManifest, DerivedStorageForImage}
|
||||
|
||||
Image, TagManifest, TagManifestLabel, Label, DerivedStorageForImage}
|
||||
delete_instance_filtered(self, Repository, delete_nullable, skip_transitive_deletes)
|
||||
|
||||
|
||||
|
|
@ -920,5 +919,40 @@ class ServiceKey(BaseModel):
|
|||
approval = ForeignKeyField(ServiceKeyApproval, null=True)
|
||||
|
||||
|
||||
class MediaType(BaseModel):
|
||||
""" MediaType is an enumeration of the possible formats of various objects in the data model. """
|
||||
name = CharField(index=True, unique=True)
|
||||
|
||||
|
||||
class LabelSourceType(BaseModel):
|
||||
""" LabelSourceType is an enumeration of the possible sources for a label. """
|
||||
name = CharField(index=True, unique=True)
|
||||
mutable = BooleanField(default=False)
|
||||
|
||||
|
||||
class Label(BaseModel):
|
||||
""" Label represents user-facing metadata associated with another entry in the
|
||||
database (e.g. a Manifest). """
|
||||
uuid = CharField(default=uuid_generator, index=True, unique=True)
|
||||
key = CharField(index=True)
|
||||
value = TextField()
|
||||
media_type = ForeignKeyField(MediaType)
|
||||
source_type = ForeignKeyField(LabelSourceType)
|
||||
|
||||
|
||||
class TagManifestLabel(BaseModel):
|
||||
""" Mapping from a tag manifest to a label. """
|
||||
repository = ForeignKeyField(Repository, index=True)
|
||||
annotated = ForeignKeyField(TagManifest, index=True)
|
||||
label = ForeignKeyField(Label)
|
||||
|
||||
class Meta:
|
||||
database = db
|
||||
read_slaves = (read_slave,)
|
||||
indexes = (
|
||||
(('annotated', 'label'), True),
|
||||
)
|
||||
|
||||
|
||||
is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel
|
||||
all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,89 @@
|
|||
"""Add labels to the schema
|
||||
|
||||
Revision ID: c9b91bee7554
|
||||
Revises: 983247d75af3
|
||||
Create Date: 2016-08-22 15:40:25.226541
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'c9b91bee7554'
|
||||
down_revision = '983247d75af3'
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from util.migrate import UTF8LongText, UTF8CharField
|
||||
|
||||
def upgrade(tables):
|
||||
### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('labelsourcetype',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('mutable', sa.Boolean(), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_labelsourcetype'))
|
||||
)
|
||||
op.create_index('labelsourcetype_name', 'labelsourcetype', ['name'], unique=True)
|
||||
op.create_table('mediatype',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_mediatype'))
|
||||
)
|
||||
op.create_index('mediatype_name', 'mediatype', ['name'], unique=True)
|
||||
op.create_table('label',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('uuid', sa.String(length=255), nullable=False),
|
||||
sa.Column('key', UTF8CharField(length=255), nullable=False),
|
||||
sa.Column('value', UTF8LongText(), nullable=False),
|
||||
sa.Column('media_type_id', sa.Integer(), nullable=False),
|
||||
sa.Column('source_type_id', sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_label_media_type_id_mediatype')),
|
||||
sa.ForeignKeyConstraint(['source_type_id'], ['labelsourcetype.id'], name=op.f('fk_label_source_type_id_labelsourcetype')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_label'))
|
||||
)
|
||||
op.create_index('label_key', 'label', ['key'], unique=False)
|
||||
op.create_index('label_media_type_id', 'label', ['media_type_id'], unique=False)
|
||||
op.create_index('label_source_type_id', 'label', ['source_type_id'], unique=False)
|
||||
op.create_index('label_uuid', 'label', ['uuid'], unique=True)
|
||||
op.create_table('tagmanifestlabel',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('repository_id', sa.Integer(), nullable=False),
|
||||
sa.Column('annotated_id', sa.Integer(), nullable=False),
|
||||
sa.Column('label_id', sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['annotated_id'], ['tagmanifest.id'], name=op.f('fk_tagmanifestlabel_annotated_id_tagmanifest')),
|
||||
sa.ForeignKeyConstraint(['label_id'], ['label.id'], name=op.f('fk_tagmanifestlabel_label_id_label')),
|
||||
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_tagmanifestlabel_repository_id_repository')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_tagmanifestlabel'))
|
||||
)
|
||||
op.create_index('tagmanifestlabel_annotated_id', 'tagmanifestlabel', ['annotated_id'], unique=False)
|
||||
op.create_index('tagmanifestlabel_annotated_id_label_id', 'tagmanifestlabel', ['annotated_id', 'label_id'], unique=True)
|
||||
op.create_index('tagmanifestlabel_label_id', 'tagmanifestlabel', ['label_id'], unique=False)
|
||||
op.create_index('tagmanifestlabel_repository_id', 'tagmanifestlabel', ['repository_id'], unique=False)
|
||||
### end Alembic commands ###
|
||||
|
||||
op.bulk_insert(tables.logentrykind, [
|
||||
{'name':'manifest_label_add'},
|
||||
{'name':'manifest_label_delete'},
|
||||
])
|
||||
|
||||
op.bulk_insert(tables.mediatype, [
|
||||
{'name':'text/plain'},
|
||||
{'name':'application/json'},
|
||||
])
|
||||
|
||||
op.bulk_insert(tables.labelsourcetype, [
|
||||
{'name':'manifest', 'mutable': False},
|
||||
{'name':'api', 'mutable': True},
|
||||
{'name':'internal', 'mutable': False},
|
||||
])
|
||||
|
||||
|
||||
def downgrade(tables):
|
||||
op.execute(tables.logentrykind.delete().where(tables.logentrykind.c.name == op.inline_literal('manifest_label_add')))
|
||||
op.execute(tables.logentrykind.delete().where(tables.logentrykind.c.name == op.inline_literal('manifest_label_delete')))
|
||||
|
||||
### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('tagmanifestlabel')
|
||||
op.drop_table('label')
|
||||
op.drop_table('mediatype')
|
||||
op.drop_table('labelsourcetype')
|
||||
### end Alembic commands ###
|
||||
|
|
@ -5,9 +5,18 @@ class DataModelException(Exception):
|
|||
pass
|
||||
|
||||
|
||||
class InvalidLabelKeyException(DataModelException):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidMediaTypeException(DataModelException):
|
||||
pass
|
||||
|
||||
|
||||
class BlobDoesNotExist(DataModelException):
|
||||
pass
|
||||
|
||||
|
||||
class TorrentInfoDoesNotExist(DataModelException):
|
||||
pass
|
||||
|
||||
|
|
@ -105,7 +114,23 @@ config = Config()
|
|||
|
||||
# There MUST NOT be any circular dependencies between these subsections. If there are fix it by
|
||||
# moving the minimal number of things to _basequery
|
||||
# TODO document the methods and modules for each one of the submodules below.
|
||||
from data.model import (blob, build, image, log, notification, oauth, organization, permission,
|
||||
repository, service_keys, storage, tag, team, token, user, release,
|
||||
modelutil)
|
||||
from data.model import (
|
||||
blob,
|
||||
build,
|
||||
image,
|
||||
label,
|
||||
log,
|
||||
modelutil,
|
||||
notification,
|
||||
oauth,
|
||||
organization,
|
||||
permission,
|
||||
release,
|
||||
repository,
|
||||
service_keys,
|
||||
storage,
|
||||
tag,
|
||||
team,
|
||||
token,
|
||||
user,
|
||||
)
|
||||
|
|
|
|||
121
data/model/label.py
Normal file
121
data/model/label.py
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
import logging
|
||||
|
||||
from data.database import Label, TagManifestLabel, MediaType, LabelSourceType, db_transaction
|
||||
from data.model import InvalidLabelKeyException, InvalidMediaTypeException, DataModelException
|
||||
from data.model._basequery import prefix_search
|
||||
from util.validation import validate_label_key
|
||||
from util.validation import is_json
|
||||
from cachetools import lru_cache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_label_source_types():
|
||||
source_type_map = {}
|
||||
for kind in LabelSourceType.select():
|
||||
source_type_map[kind.id] = kind.name
|
||||
source_type_map[kind.name] = kind.id
|
||||
|
||||
return source_type_map
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_media_types():
|
||||
media_type_map = {}
|
||||
for kind in MediaType.select():
|
||||
media_type_map[kind.id] = kind.name
|
||||
media_type_map[kind.name] = kind.id
|
||||
|
||||
return media_type_map
|
||||
|
||||
|
||||
def _get_label_source_type_id(name):
|
||||
kinds = get_label_source_types()
|
||||
return kinds[name]
|
||||
|
||||
|
||||
def _get_media_type_id(name):
|
||||
kinds = get_media_types()
|
||||
return kinds[name]
|
||||
|
||||
|
||||
def create_manifest_label(tag_manifest, key, value, source_type_name, media_type_name=None):
|
||||
""" Creates a new manifest label on a specific tag manifest. """
|
||||
if not key:
|
||||
raise InvalidLabelKeyException()
|
||||
|
||||
# Note that we don't prevent invalid label names coming from the manifest to be stored, as Docker
|
||||
# does not currently prevent them from being put into said manifests.
|
||||
if not validate_label_key(key) and source_type_name != 'manifest':
|
||||
raise InvalidLabelKeyException()
|
||||
|
||||
# Find the matching media type. If none specified, we infer.
|
||||
if media_type_name is None:
|
||||
media_type_name = 'text/plain'
|
||||
if is_json(value):
|
||||
media_type_name = 'application/json'
|
||||
|
||||
media_type_id = _get_media_type_id(media_type_name)
|
||||
if media_type_id is None:
|
||||
raise InvalidMediaTypeException
|
||||
|
||||
source_type_id = _get_label_source_type_id(source_type_name)
|
||||
|
||||
with db_transaction():
|
||||
label = Label.create(key=key, value=value, source_type=source_type_id, media_type=media_type_id)
|
||||
TagManifestLabel.create(annotated=tag_manifest, label=label,
|
||||
repository=tag_manifest.tag.repository)
|
||||
|
||||
return label
|
||||
|
||||
|
||||
def list_manifest_labels(tag_manifest, prefix_filter=None):
|
||||
""" Lists all labels found on the given tag manifest. """
|
||||
query = (Label.select(Label, MediaType)
|
||||
.join(MediaType)
|
||||
.switch(Label)
|
||||
.join(LabelSourceType)
|
||||
.switch(Label)
|
||||
.join(TagManifestLabel)
|
||||
.where(TagManifestLabel.annotated == tag_manifest))
|
||||
|
||||
if prefix_filter is not None:
|
||||
query = query.where(prefix_search(Label.key, prefix_filter))
|
||||
|
||||
return query
|
||||
|
||||
|
||||
def get_manifest_label(label_uuid, tag_manifest):
|
||||
""" Retrieves the manifest label on the tag manifest with the given ID. """
|
||||
try:
|
||||
return (Label.select(Label, LabelSourceType)
|
||||
.join(LabelSourceType)
|
||||
.where(Label.uuid == label_uuid)
|
||||
.switch(Label)
|
||||
.join(TagManifestLabel)
|
||||
.where(TagManifestLabel.annotated == tag_manifest)
|
||||
.get())
|
||||
except Label.DoesNotExist:
|
||||
return None
|
||||
|
||||
|
||||
def delete_manifest_label(label_uuid, tag_manifest):
|
||||
""" Deletes the manifest label on the tag manifest with the given ID. """
|
||||
|
||||
# Find the label itself.
|
||||
label = get_manifest_label(label_uuid, tag_manifest)
|
||||
if label is None:
|
||||
return None
|
||||
|
||||
if not label.source_type.mutable:
|
||||
raise DataModelException('Cannot delete immutable label')
|
||||
|
||||
# Delete the mapping record and label.
|
||||
deleted_count = TagManifestLabel.delete().where(TagManifestLabel.label == label).execute()
|
||||
if deleted_count != 1:
|
||||
logger.warning('More than a single label deleted for matching label %s', label_uuid)
|
||||
|
||||
label.delete_instance(recursive=False)
|
||||
return label
|
||||
|
||||
|
|
@ -10,7 +10,8 @@ from data.model import (DataModelException, tag, db_transaction, storage, permis
|
|||
from data.database import (Repository, Namespace, RepositoryTag, Star, Image, User,
|
||||
Visibility, RepositoryPermission, RepositoryActionCount,
|
||||
Role, RepositoryAuthorizedEmail, TagManifest, DerivedStorageForImage,
|
||||
get_epoch_timestamp, db_random_func)
|
||||
Label, TagManifestLabel, db_for_update, get_epoch_timestamp,
|
||||
db_random_func)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -50,11 +51,24 @@ def _purge_all_repository_tags(namespace_name, repository_name):
|
|||
raise DataModelException('Invalid repository \'%s/%s\'' %
|
||||
(namespace_name, repository_name))
|
||||
|
||||
# Delete all manifests.
|
||||
# Finds all the tags to delete.
|
||||
repo_tags = list(RepositoryTag.select().where(RepositoryTag.repository == repo.id))
|
||||
if not repo_tags:
|
||||
return
|
||||
|
||||
# Find all labels to delete.
|
||||
labels = list(TagManifestLabel
|
||||
.select(TagManifestLabel.label)
|
||||
.where(TagManifestLabel.repository == repo))
|
||||
|
||||
# Delete all the mapping entries.
|
||||
TagManifestLabel.delete().where(TagManifestLabel.repository == repo).execute()
|
||||
|
||||
# Delete all the matching labels.
|
||||
if labels:
|
||||
Label.delete().where(Label.id << [label.id for label in labels]).execute()
|
||||
|
||||
# Delete all the manifests.
|
||||
TagManifest.delete().where(TagManifest.tag << repo_tags).execute()
|
||||
|
||||
# Delete all tags.
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@ from uuid import uuid4
|
|||
from data.model import (image, db_transaction, DataModelException, _basequery,
|
||||
InvalidManifestException)
|
||||
from data.database import (RepositoryTag, Repository, Image, ImageStorage, Namespace, TagManifest,
|
||||
RepositoryNotification, get_epoch_timestamp, db_for_update)
|
||||
RepositoryNotification, Label, TagManifestLabel, get_epoch_timestamp,
|
||||
db_for_update)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -150,6 +151,22 @@ def garbage_collect_tags(repo):
|
|||
|
||||
num_deleted_manifests = 0
|
||||
if len(manifests_to_delete) > 0:
|
||||
# Find the set of IDs for all the labels to delete.
|
||||
labels = list(TagManifestLabel
|
||||
.select(TagManifestLabel.id)
|
||||
.where(TagManifestLabel.annotated << manifests_to_delete))
|
||||
|
||||
if len(labels) > 0:
|
||||
# Delete the mapping entries.
|
||||
(TagManifestLabel
|
||||
.delete()
|
||||
.where(TagManifestLabel.annotated << manifests_to_delete)
|
||||
.execute())
|
||||
|
||||
# Delete the labels themselves.
|
||||
Label.delete().where(Label.id << [label.id for label in labels]).execute()
|
||||
|
||||
# Delete the tag manifests themselves.
|
||||
num_deleted_manifests = (TagManifest
|
||||
.delete()
|
||||
.where(TagManifest.id << manifests_to_delete)
|
||||
|
|
@ -234,15 +251,22 @@ def revert_tag(repo_obj, tag_name, docker_image_id):
|
|||
|
||||
def store_tag_manifest(namespace, repo_name, tag_name, docker_image_id, manifest_digest,
|
||||
manifest_data):
|
||||
""" Stores a tag manifest for a specific tag name in the database. Returns the TagManifest
|
||||
object, as well as a boolean indicating whether the TagManifest was created or updated.
|
||||
"""
|
||||
with db_transaction():
|
||||
tag = create_or_update_tag(namespace, repo_name, tag_name, docker_image_id)
|
||||
|
||||
try:
|
||||
manifest = TagManifest.get(digest=manifest_digest)
|
||||
if manifest.tag == tag:
|
||||
return manifest, False
|
||||
|
||||
manifest.tag = tag
|
||||
manifest.save()
|
||||
return manifest, True
|
||||
except TagManifest.DoesNotExist:
|
||||
return TagManifest.create(tag=tag, digest=manifest_digest, json_data=manifest_data)
|
||||
return TagManifest.create(tag=tag, digest=manifest_digest, json_data=manifest_data), True
|
||||
|
||||
|
||||
def get_active_tag(namespace, repo_name, tag_name):
|
||||
|
|
@ -282,7 +306,7 @@ def load_manifest_by_digest(namespace, repo_name, digest):
|
|||
|
||||
def _load_repo_manifests(namespace, repo_name):
|
||||
return _tag_alive(TagManifest
|
||||
.select(TagManifest, RepositoryTag)
|
||||
.select(TagManifest, RepositoryTag, Repository)
|
||||
.join(RepositoryTag)
|
||||
.join(Repository)
|
||||
.join(Namespace, on=(Namespace.id == Repository.namespace_user))
|
||||
|
|
|
|||
|
|
@ -364,6 +364,7 @@ import endpoints.api.discovery
|
|||
import endpoints.api.error
|
||||
import endpoints.api.image
|
||||
import endpoints.api.logs
|
||||
import endpoints.api.manifest
|
||||
import endpoints.api.organization
|
||||
import endpoints.api.permission
|
||||
import endpoints.api.prototype
|
||||
|
|
|
|||
159
endpoints/api/manifest.py
Normal file
159
endpoints/api/manifest.py
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
""" Manage the manifests of a repository. """
|
||||
|
||||
from app import label_validator
|
||||
from flask import request
|
||||
from endpoints.api import (resource, nickname, require_repo_read, require_repo_write,
|
||||
RepositoryParamResource, log_action, validate_json_request,
|
||||
path_param, parse_args, query_param, truthy_bool, abort, api)
|
||||
from endpoints.exception import NotFound
|
||||
from data import model
|
||||
|
||||
from digest import digest_tools
|
||||
|
||||
BASE_MANIFEST_ROUTE = '/v1/repository/<apirepopath:repository>/manifest/<regex("{0}"):manifestref>'
|
||||
MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN)
|
||||
ALLOWED_LABEL_MEDIA_TYPES = ['text/plain', 'application/json']
|
||||
|
||||
def label_view(label):
|
||||
view = {
|
||||
'id': label.uuid,
|
||||
'key': label.key,
|
||||
'value': label.value,
|
||||
'source_type': label.source_type.name,
|
||||
'media_type': label.media_type.name,
|
||||
}
|
||||
|
||||
return view
|
||||
|
||||
@resource(MANIFEST_DIGEST_ROUTE + '/labels')
|
||||
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
|
||||
@path_param('manifestref', 'The digest of the manifest')
|
||||
class RepositoryManifestLabels(RepositoryParamResource):
|
||||
""" Resource for listing the labels on a specific repository manifest. """
|
||||
schemas = {
|
||||
'AddLabel': {
|
||||
'type': 'object',
|
||||
'description': 'Adds a label to a manifest',
|
||||
'required': [
|
||||
'key',
|
||||
'value',
|
||||
'media_type',
|
||||
],
|
||||
'properties': {
|
||||
'key': {
|
||||
'type': 'string',
|
||||
'description': 'The key for the label',
|
||||
},
|
||||
'value': {
|
||||
'type': 'string',
|
||||
'description': 'The value for the label',
|
||||
},
|
||||
'media_type': {
|
||||
'type': ['string'],
|
||||
'description': 'The media type for this label',
|
||||
'enum': ALLOWED_LABEL_MEDIA_TYPES,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@require_repo_read
|
||||
@nickname('listManifestLabels')
|
||||
@parse_args()
|
||||
@query_param('filter', 'If specified, only labels matching the given prefix will be returned',
|
||||
type=str, default=None)
|
||||
def get(self, namespace, repository, manifestref, parsed_args):
|
||||
try:
|
||||
tag_manifest = model.tag.load_manifest_by_digest(namespace, repository, manifestref)
|
||||
except model.DataModelException:
|
||||
raise NotFound()
|
||||
|
||||
labels = model.label.list_manifest_labels(tag_manifest, prefix_filter=parsed_args['filter'])
|
||||
return {
|
||||
'labels': [label_view(label) for label in labels]
|
||||
}
|
||||
|
||||
@require_repo_write
|
||||
@nickname('addManifestLabel')
|
||||
@validate_json_request('AddLabel')
|
||||
def post(self, namespace, repository, manifestref):
|
||||
""" Adds a new label into the tag manifest. """
|
||||
try:
|
||||
tag_manifest = model.tag.load_manifest_by_digest(namespace, repository, manifestref)
|
||||
except model.DataModelException:
|
||||
raise NotFound()
|
||||
|
||||
label_data = request.get_json()
|
||||
|
||||
# Check for any reserved prefixes.
|
||||
if label_validator.has_reserved_prefix(label_data['key']):
|
||||
abort(400, message='Label has a reserved prefix')
|
||||
|
||||
label = model.label.create_manifest_label(tag_manifest, label_data['key'],
|
||||
label_data['value'], 'api',
|
||||
media_type_name=label_data['media_type'])
|
||||
metadata = {
|
||||
'id': label.uuid,
|
||||
'key': label_data['key'],
|
||||
'value': label_data['value'],
|
||||
'manifest_digest': manifestref,
|
||||
'media_type': label_data['media_type'],
|
||||
}
|
||||
|
||||
log_action('manifest_label_add', namespace, metadata, repo=tag_manifest.tag.repository)
|
||||
|
||||
resp = {'label': label_view(label)}
|
||||
repo_string = '%s/%s' % (namespace, repository)
|
||||
headers = {
|
||||
'Location': api.url_for(ManageRepositoryManifestLabel, repository=repo_string,
|
||||
manifestref=manifestref, labelid=label.uuid),
|
||||
}
|
||||
return resp, 201, headers
|
||||
|
||||
|
||||
@resource(MANIFEST_DIGEST_ROUTE + '/labels/<labelid>')
|
||||
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
|
||||
@path_param('manifestref', 'The digest of the manifest')
|
||||
@path_param('labelid', 'The ID of the label')
|
||||
class ManageRepositoryManifestLabel(RepositoryParamResource):
|
||||
""" Resource for managing the labels on a specific repository manifest. """
|
||||
@require_repo_read
|
||||
@nickname('getManifestLabel')
|
||||
@parse_args()
|
||||
def get(self, namespace, repository, manifestref, labelid):
|
||||
""" Retrieves the label with the specific ID under the manifest. """
|
||||
try:
|
||||
tag_manifest = model.tag.load_manifest_by_digest(namespace, repository, manifestref)
|
||||
except model.DataModelException:
|
||||
raise NotFound()
|
||||
|
||||
label = model.label.get_manifest_label(labelid, tag_manifest)
|
||||
if label is None:
|
||||
raise NotFound()
|
||||
|
||||
return label_view(label)
|
||||
|
||||
|
||||
@require_repo_write
|
||||
@nickname('deleteManifestLabel')
|
||||
def delete(self, namespace, repository, manifestref, labelid):
|
||||
""" Deletes an existing label from a manifest. """
|
||||
try:
|
||||
tag_manifest = model.tag.load_manifest_by_digest(namespace, repository, manifestref)
|
||||
except model.DataModelException:
|
||||
raise NotFound()
|
||||
|
||||
deleted = model.label.delete_manifest_label(labelid, tag_manifest)
|
||||
if deleted is None:
|
||||
raise NotFound()
|
||||
|
||||
metadata = {
|
||||
'id': labelid,
|
||||
'key': deleted.key,
|
||||
'value': deleted.value,
|
||||
'manifest_digest': manifestref
|
||||
}
|
||||
|
||||
log_action('manifest_label_delete', namespace, metadata, repo=tag_manifest.tag.repository)
|
||||
return 'Deleted', 204
|
||||
|
||||
|
|
@ -47,7 +47,7 @@ JWS_ALGORITHM = 'RS256'
|
|||
|
||||
ImageMetadata = namedtuple('ImageMetadata', ['digest', 'v1_metadata', 'v1_metadata_str'])
|
||||
ExtractedV1Metadata = namedtuple('ExtractedV1Metadata', ['docker_id', 'parent', 'created',
|
||||
'comment', 'command'])
|
||||
'comment', 'command', 'labels'])
|
||||
|
||||
|
||||
_SIGNATURES_KEY = 'signatures'
|
||||
|
|
@ -144,9 +144,10 @@ class SignedManifest(object):
|
|||
if not 'id' in v1_metadata:
|
||||
raise ManifestInvalid(detail={'message': 'invalid manifest v1 history'})
|
||||
|
||||
labels = v1_metadata.get('config', {}).get('Labels', {}) or {}
|
||||
extracted = ExtractedV1Metadata(v1_metadata['id'], v1_metadata.get('parent'),
|
||||
v1_metadata.get('created'), v1_metadata.get('comment'),
|
||||
command)
|
||||
command, labels)
|
||||
yield ImageMetadata(image_digest, extracted, metadata_string)
|
||||
|
||||
@property
|
||||
|
|
@ -450,8 +451,12 @@ def _write_manifest_itself(namespace_name, repo_name, manifest):
|
|||
# Store the manifest pointing to the tag.
|
||||
manifest_digest = manifest.digest
|
||||
leaf_layer_id = images_map[layers[-1].v1_metadata.docker_id].docker_image_id
|
||||
model.tag.store_tag_manifest(namespace_name, repo_name, tag_name, leaf_layer_id, manifest_digest,
|
||||
manifest.bytes)
|
||||
tag_manifest, manifest_created = model.tag.store_tag_manifest(namespace_name, repo_name, tag_name,
|
||||
leaf_layer_id, manifest_digest,
|
||||
manifest.bytes)
|
||||
if manifest_created:
|
||||
for key, value in layers[-1].v1_metadata.labels.iteritems():
|
||||
model.label.create_manifest_label(tag_manifest, key, value, 'manifest')
|
||||
|
||||
# Queue all blob manifests for replication.
|
||||
# TODO(jschorr): Find a way to optimize this insertion.
|
||||
|
|
|
|||
33
initdb.py
33
initdb.py
|
|
@ -12,9 +12,6 @@ from peewee import (SqliteDatabase, create_model_tables, drop_model_tables, save
|
|||
from itertools import count
|
||||
from uuid import UUID, uuid4
|
||||
from threading import Event
|
||||
from hashlib import sha256
|
||||
from Crypto.PublicKey import RSA
|
||||
from jwkest.jwk import RSAKey
|
||||
|
||||
from email.utils import formatdate
|
||||
from data.database import (db, all_models, Role, TeamRole, Visibility, LoginService,
|
||||
|
|
@ -22,7 +19,7 @@ from data.database import (db, all_models, Role, TeamRole, Visibility, LoginServ
|
|||
ImageStorageTransformation, ImageStorageSignatureKind,
|
||||
ExternalNotificationEvent, ExternalNotificationMethod, NotificationKind,
|
||||
QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode,
|
||||
ServiceKeyApprovalType)
|
||||
ServiceKeyApprovalType, MediaType, LabelSourceType)
|
||||
from data import model
|
||||
from data.queue import WorkQueue
|
||||
from app import app, storage as store, tf
|
||||
|
|
@ -345,6 +342,9 @@ def initialize_database():
|
|||
|
||||
LogEntryKind.create(name='take_ownership')
|
||||
|
||||
LogEntryKind.create(name='manifest_label_add')
|
||||
LogEntryKind.create(name='manifest_label_delete')
|
||||
|
||||
ImageStorageLocation.create(name='local_eu')
|
||||
ImageStorageLocation.create(name='local_us')
|
||||
|
||||
|
|
@ -389,6 +389,13 @@ def initialize_database():
|
|||
QuayRegion.create(name='us')
|
||||
QuayService.create(name='quay')
|
||||
|
||||
MediaType.create(name='text/plain')
|
||||
MediaType.create(name='application/json')
|
||||
|
||||
LabelSourceType.create(name='manifest')
|
||||
LabelSourceType.create(name='api', mutable=True)
|
||||
LabelSourceType.create(name='internal')
|
||||
|
||||
|
||||
def wipe_database():
|
||||
logger.debug('Wiping all data from the DB.')
|
||||
|
|
@ -474,6 +481,24 @@ def populate_database(minimal=False, with_storage=False):
|
|||
|
||||
simple_repo = __generate_repository(with_storage, new_user_1, 'simple', 'Simple repository.', False,
|
||||
[], (4, [], ['latest', 'prod']))
|
||||
|
||||
# Add some labels to the latest tag's manifest.
|
||||
tag_manifest = model.tag.load_tag_manifest(new_user_1.username, 'simple', 'latest')
|
||||
first_label = model.label.create_manifest_label(tag_manifest, 'foo', 'bar', 'manifest')
|
||||
model.label.create_manifest_label(tag_manifest, 'foo', 'baz', 'api')
|
||||
model.label.create_manifest_label(tag_manifest, 'anotherlabel', '1234', 'internal')
|
||||
|
||||
label_metadata = {
|
||||
'key': 'foo',
|
||||
'value': 'bar',
|
||||
'id': first_label.id,
|
||||
'manifest_digest': tag_manifest.digest
|
||||
}
|
||||
|
||||
model.log.log_action('manifest_label_add', new_user_1.username, performer=new_user_1,
|
||||
timestamp=datetime.now(), metadata=label_metadata,
|
||||
repository=tag_manifest.tag.repository)
|
||||
|
||||
model.blob.initiate_upload(new_user_1.username, simple_repo.name, str(uuid4()), 'local_us', {})
|
||||
model.notification.create_repo_notification(simple_repo, 'repo_push', 'quay_notification', {}, {})
|
||||
|
||||
|
|
|
|||
|
|
@ -230,6 +230,9 @@ angular.module('quay').directive('logsView', function () {
|
|||
}
|
||||
},
|
||||
|
||||
'manifest_label_add': 'Label {key} added to manifest {manifest_digest}',
|
||||
'manifest_label_delete': 'Label {key} deleted from manifest {manifest_digest}',
|
||||
|
||||
// Note: These are deprecated.
|
||||
'add_repo_webhook': 'Add webhook in repository {repo}',
|
||||
'delete_repo_webhook': 'Delete webhook in repository {repo}'
|
||||
|
|
@ -287,6 +290,8 @@ angular.module('quay').directive('logsView', function () {
|
|||
'service_key_extend': 'Extend Service Key Expiration',
|
||||
'service_key_rotate': 'Automatic rotation of Service Key',
|
||||
'take_ownership': 'Take Namespace Ownership',
|
||||
'manifest_label_add': 'Add Manifest Label',
|
||||
'manifest_label_delete': 'Delete Manifest Label',
|
||||
|
||||
// Note: these are deprecated.
|
||||
'add_repo_webhook': 'Add webhook',
|
||||
|
|
|
|||
|
|
@ -22,7 +22,8 @@ angular.module('quay').factory('StringBuilderService', ['$sce', 'UtilService', f
|
|||
'application_name': 'cloud',
|
||||
'image': 'archive',
|
||||
'original_image': 'archive',
|
||||
'client_id': 'chain'
|
||||
'client_id': 'chain',
|
||||
'manifest_digest': 'link'
|
||||
};
|
||||
|
||||
var filters = {
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -16,6 +16,7 @@ class assert_action_logged(object):
|
|||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
updated_count = self._get_log_count()
|
||||
error_msg = 'Missing new log entry of kind %s' % self.log_kind
|
||||
assert self.existing_count == (updated_count - 1), error_msg
|
||||
if exc_val is None:
|
||||
updated_count = self._get_log_count()
|
||||
error_msg = 'Missing new log entry of kind %s' % self.log_kind
|
||||
assert self.existing_count == (updated_count - 1), error_msg
|
||||
|
|
|
|||
|
|
@ -1129,6 +1129,55 @@ class V1RegistryTests(V1RegistryPullMixin, V1RegistryPushMixin, RegistryTestsMix
|
|||
class V2RegistryTests(V2RegistryPullMixin, V2RegistryPushMixin, RegistryTestsMixin,
|
||||
RegistryTestCaseMixin, LiveServerTestCase):
|
||||
""" Tests for V2 registry. """
|
||||
def test_label_invalid_manifest(self):
|
||||
images = [{
|
||||
'id': 'someid',
|
||||
'config': {'Labels': None},
|
||||
'contents': 'somecontent'
|
||||
}]
|
||||
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images=images)
|
||||
self.do_pull('devtable', 'newrepo', 'devtable', 'password')
|
||||
|
||||
def test_labels(self):
|
||||
# Push a new repo with the latest tag.
|
||||
images = [{
|
||||
'id': 'someid',
|
||||
'config': {'Labels': {'foo': 'bar', 'baz': 'meh', 'theoretically-invalid--label': 'foo'}},
|
||||
'contents': 'somecontent'
|
||||
}]
|
||||
|
||||
(_, digest) = self.do_push('devtable', 'newrepo', 'devtable', 'password', images=images)
|
||||
|
||||
self.conduct_api_login('devtable', 'password')
|
||||
labels = self.conduct('GET', '/api/v1/repository/devtable/newrepo/manifest/' + digest + '/labels').json()
|
||||
self.assertEquals(3, len(labels['labels']))
|
||||
|
||||
self.assertEquals('manifest', labels['labels'][0]['source_type'])
|
||||
self.assertEquals('manifest', labels['labels'][1]['source_type'])
|
||||
self.assertEquals('manifest', labels['labels'][2]['source_type'])
|
||||
|
||||
self.assertEquals('text/plain', labels['labels'][0]['media_type'])
|
||||
self.assertEquals('text/plain', labels['labels'][1]['media_type'])
|
||||
self.assertEquals('text/plain', labels['labels'][2]['media_type'])
|
||||
|
||||
def test_json_labels(self):
|
||||
# Push a new repo with the latest tag.
|
||||
images = [{
|
||||
'id': 'someid',
|
||||
'config': {'Labels': {'foo': 'bar', 'baz': '{"some": "json"}'}},
|
||||
'contents': 'somecontent'
|
||||
}]
|
||||
|
||||
(_, digest) = self.do_push('devtable', 'newrepo', 'devtable', 'password', images=images)
|
||||
|
||||
self.conduct_api_login('devtable', 'password')
|
||||
labels = self.conduct('GET', '/api/v1/repository/devtable/newrepo/manifest/' + digest + '/labels').json()
|
||||
self.assertEquals(2, len(labels['labels']))
|
||||
|
||||
self.assertEquals('text/plain', labels['labels'][0]['media_type'])
|
||||
self.assertEquals('application/json', labels['labels'][1]['media_type'])
|
||||
|
||||
def test_invalid_manifest_type(self):
|
||||
namespace = 'devtable'
|
||||
repository = 'somerepo'
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana
|
|||
SuperUserServiceKey, SuperUserServiceKeyApproval,
|
||||
SuperUserTakeOwnership)
|
||||
from endpoints.api.secscan import RepositoryImageSecurity
|
||||
from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel
|
||||
|
||||
|
||||
try:
|
||||
|
|
@ -4298,5 +4299,54 @@ class TestRepositoryImageSecurity(ApiTestCase):
|
|||
self._run_test('GET', 404, 'devtable', None)
|
||||
|
||||
|
||||
class TestRepositoryManifestLabels(ApiTestCase):
|
||||
def setUp(self):
|
||||
ApiTestCase.setUp(self)
|
||||
self._set_url(RepositoryManifestLabels, repository='devtable/simple', manifestref='sha256:abcd')
|
||||
|
||||
def test_get_anonymous(self):
|
||||
self._run_test('GET', 401, None, None)
|
||||
|
||||
def test_get_freshuser(self):
|
||||
self._run_test('GET', 403, 'freshuser', None)
|
||||
|
||||
def test_get_reader(self):
|
||||
self._run_test('GET', 403, 'reader', None)
|
||||
|
||||
def test_get_devtable(self):
|
||||
self._run_test('GET', 404, 'devtable', None)
|
||||
|
||||
def test_post_anonymous(self):
|
||||
self._run_test('POST', 401, None, dict(key='foo', value='bar', media_type='text/plain'))
|
||||
|
||||
def test_post_freshuser(self):
|
||||
self._run_test('POST', 403, 'freshuser', dict(key='foo', value='bar', media_type='text/plain'))
|
||||
|
||||
def test_post_reader(self):
|
||||
self._run_test('POST', 403, 'reader', dict(key='foo', value='bar', media_type='text/plain'))
|
||||
|
||||
def test_post_devtable(self):
|
||||
self._run_test('POST', 404, 'devtable', dict(key='foo', value='bar', media_type='text/plain'))
|
||||
|
||||
|
||||
class TestManageRepositoryManifestLabel(ApiTestCase):
|
||||
def setUp(self):
|
||||
ApiTestCase.setUp(self)
|
||||
self._set_url(ManageRepositoryManifestLabel, repository='devtable/simple',
|
||||
manifestref='sha256:abcd', labelid='someid')
|
||||
|
||||
def test_delete_anonymous(self):
|
||||
self._run_test('DELETE', 401, None, None)
|
||||
|
||||
def test_delete_freshuser(self):
|
||||
self._run_test('DELETE', 403, 'freshuser', None)
|
||||
|
||||
def test_delete_reader(self):
|
||||
self._run_test('DELETE', 403, 'reader', None)
|
||||
|
||||
def test_delete_devtable(self):
|
||||
self._run_test('DELETE', 404, 'devtable', None)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -65,6 +65,8 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana
|
|||
from endpoints.api.secscan import RepositoryImageSecurity
|
||||
from endpoints.api.suconfig import (SuperUserRegistryStatus, SuperUserConfig, SuperUserConfigFile,
|
||||
SuperUserCreateInitialSuperUser)
|
||||
from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel
|
||||
|
||||
|
||||
try:
|
||||
app.register_blueprint(api_bp, url_prefix='/api')
|
||||
|
|
@ -1765,6 +1767,15 @@ class TestDeleteRepository(ApiTestCase):
|
|||
RepositoryActionCount.create(repository=repository,
|
||||
date=datetime.datetime.now() - datetime.timedelta(days=5), count=6)
|
||||
|
||||
# Create some labels.
|
||||
tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod')
|
||||
model.label.create_manifest_label(tag_manifest, 'foo', 'bar', 'manifest')
|
||||
model.label.create_manifest_label(tag_manifest, 'foo', 'baz', 'manifest')
|
||||
model.label.create_manifest_label(tag_manifest, 'something', '{}', 'api',
|
||||
media_type_name='application/json')
|
||||
|
||||
model.label.create_manifest_label(tag_manifest, 'something', '{"some": "json"}', 'manifest')
|
||||
|
||||
# Delete the repository.
|
||||
with check_transitive_deletes():
|
||||
self.deleteResponse(Repository, params=dict(repository=self.COMPLEX_REPO))
|
||||
|
|
@ -3941,6 +3952,150 @@ class TestSuperUserKeyManagement(ApiTestCase):
|
|||
self.assertEquals('whazzup!?', json['approval']['notes'])
|
||||
|
||||
|
||||
class TestRepositoryManifestLabels(ApiTestCase):
|
||||
def test_basic_labels(self):
|
||||
self.login(ADMIN_ACCESS_USER)
|
||||
|
||||
# Find the manifest digest for the prod tag in the complex repo.
|
||||
tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod')
|
||||
repository = ADMIN_ACCESS_USER + '/complex'
|
||||
|
||||
# Check the existing labels on the complex repo, which should be empty
|
||||
json = self.getJsonResponse(RepositoryManifestLabels,
|
||||
params=dict(repository=repository, manifestref=tag_manifest.digest))
|
||||
|
||||
self.assertEquals(0, len(json['labels']))
|
||||
|
||||
# Add some labels to the manifest.
|
||||
with assert_action_logged('manifest_label_add'):
|
||||
label1 = self.postJsonResponse(RepositoryManifestLabels,
|
||||
params=dict(repository=repository,
|
||||
manifestref=tag_manifest.digest),
|
||||
data=dict(key='hello', value='world',
|
||||
media_type='text/plain'),
|
||||
expected_code=201)
|
||||
|
||||
|
||||
with assert_action_logged('manifest_label_add'):
|
||||
label2 = self.postJsonResponse(RepositoryManifestLabels,
|
||||
params=dict(repository=repository,
|
||||
manifestref=tag_manifest.digest),
|
||||
data=dict(key='hi', value='there',
|
||||
media_type='text/plain'),
|
||||
expected_code=201)
|
||||
|
||||
with assert_action_logged('manifest_label_add'):
|
||||
label3 = self.postJsonResponse(RepositoryManifestLabels,
|
||||
params=dict(repository=repository,
|
||||
manifestref=tag_manifest.digest),
|
||||
data=dict(key='hello', value='someone',
|
||||
media_type='application/json'),
|
||||
expected_code=201)
|
||||
|
||||
|
||||
# Ensure we have *3* labels
|
||||
json = self.getJsonResponse(RepositoryManifestLabels,
|
||||
params=dict(repository=repository,
|
||||
manifestref=tag_manifest.digest))
|
||||
|
||||
self.assertEquals(3, len(json['labels']))
|
||||
|
||||
self.assertNotEquals(label2['label']['id'], label1['label']['id'])
|
||||
self.assertNotEquals(label3['label']['id'], label1['label']['id'])
|
||||
self.assertNotEquals(label2['label']['id'], label3['label']['id'])
|
||||
|
||||
self.assertEquals('text/plain', label1['label']['media_type'])
|
||||
self.assertEquals('text/plain', label2['label']['media_type'])
|
||||
self.assertEquals('application/json', label3['label']['media_type'])
|
||||
|
||||
# Delete a label.
|
||||
with assert_action_logged('manifest_label_delete'):
|
||||
self.deleteResponse(ManageRepositoryManifestLabel,
|
||||
params=dict(repository=repository,
|
||||
manifestref=tag_manifest.digest,
|
||||
labelid=label1['label']['id']))
|
||||
|
||||
# Ensure the label is gone.
|
||||
json = self.getJsonResponse(RepositoryManifestLabels,
|
||||
params=dict(repository=repository,
|
||||
manifestref=tag_manifest.digest))
|
||||
|
||||
self.assertEquals(2, len(json['labels']))
|
||||
|
||||
# Check filtering.
|
||||
json = self.getJsonResponse(RepositoryManifestLabels,
|
||||
params=dict(repository=repository,
|
||||
manifestref=tag_manifest.digest,
|
||||
filter='hello'))
|
||||
|
||||
self.assertEquals(1, len(json['labels']))
|
||||
|
||||
|
||||
def test_prefixed_labels(self):
|
||||
self.login(ADMIN_ACCESS_USER)
|
||||
|
||||
# Find the manifest digest for the prod tag in the complex repo.
|
||||
tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod')
|
||||
repository = ADMIN_ACCESS_USER + '/complex'
|
||||
|
||||
self.postJsonResponse(RepositoryManifestLabels,
|
||||
params=dict(repository=repository,
|
||||
manifestref=tag_manifest.digest),
|
||||
data=dict(key='com.dockers.whatever', value='pants',
|
||||
media_type='text/plain'),
|
||||
expected_code=201)
|
||||
|
||||
|
||||
self.postJsonResponse(RepositoryManifestLabels,
|
||||
params=dict(repository=repository,
|
||||
manifestref=tag_manifest.digest),
|
||||
data=dict(key='my.cool.prefix.for.my.label', value='value',
|
||||
media_type='text/plain'),
|
||||
expected_code=201)
|
||||
|
||||
|
||||
|
||||
def test_add_invalid_media_type(self):
|
||||
self.login(ADMIN_ACCESS_USER)
|
||||
|
||||
tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod')
|
||||
repository = ADMIN_ACCESS_USER + '/complex'
|
||||
|
||||
self.postResponse(RepositoryManifestLabels,
|
||||
params=dict(repository=repository,
|
||||
manifestref=tag_manifest.digest),
|
||||
data=dict(key='hello', value='world', media_type='some/invalid'),
|
||||
expected_code=400)
|
||||
|
||||
|
||||
def test_add_invalid_key(self):
|
||||
self.login(ADMIN_ACCESS_USER)
|
||||
|
||||
tag_manifest = model.tag.load_tag_manifest(ADMIN_ACCESS_USER, 'complex', 'prod')
|
||||
repository = ADMIN_ACCESS_USER + '/complex'
|
||||
|
||||
# Try to add an empty label key.
|
||||
self.postResponse(RepositoryManifestLabels,
|
||||
params=dict(repository=repository,
|
||||
manifestref=tag_manifest.digest),
|
||||
data=dict(key='', value='world'),
|
||||
expected_code=400)
|
||||
|
||||
# Try to add an invalid label key.
|
||||
self.postResponse(RepositoryManifestLabels,
|
||||
params=dict(repository=repository,
|
||||
manifestref=tag_manifest.digest),
|
||||
data=dict(key='invalid___key', value='world'),
|
||||
expected_code=400)
|
||||
|
||||
# Try to add a label key in a reserved namespace.
|
||||
self.postResponse(RepositoryManifestLabels,
|
||||
params=dict(repository=repository,
|
||||
manifestref=tag_manifest.digest),
|
||||
data=dict(key='io.docker.whatever', value='world'),
|
||||
expected_code=400)
|
||||
|
||||
|
||||
class TestSuperUserManagement(ApiTestCase):
|
||||
def test_get_user(self):
|
||||
self.login(ADMIN_ACCESS_USER)
|
||||
|
|
|
|||
44
test/test_validation.py
Normal file
44
test/test_validation.py
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
import unittest
|
||||
from util.validation import validate_label_key
|
||||
|
||||
class TestLabelKeyValidation(unittest.TestCase):
|
||||
def assertValidKey(self, key):
|
||||
self.assertTrue(validate_label_key(key))
|
||||
|
||||
def assertInvalidKey(self, key):
|
||||
self.assertFalse(validate_label_key(key))
|
||||
|
||||
def test_basic_keys(self):
|
||||
self.assertValidKey('foo')
|
||||
self.assertValidKey('bar')
|
||||
|
||||
self.assertValidKey('foo1')
|
||||
self.assertValidKey('bar2')
|
||||
|
||||
self.assertValidKey('1')
|
||||
self.assertValidKey('12')
|
||||
self.assertValidKey('123')
|
||||
self.assertValidKey('1234')
|
||||
|
||||
self.assertValidKey('git-sha')
|
||||
|
||||
self.assertValidKey('com.coreos.something')
|
||||
self.assertValidKey('io.quay.git-sha')
|
||||
|
||||
def test_invalid_keys(self):
|
||||
self.assertInvalidKey('')
|
||||
self.assertInvalidKey('git_sha')
|
||||
|
||||
def test_must_start_with_alphanumeric(self):
|
||||
self.assertInvalidKey('-125')
|
||||
self.assertInvalidKey('-foo')
|
||||
self.assertInvalidKey('foo-')
|
||||
self.assertInvalidKey('123-')
|
||||
|
||||
def test_no_double_dashesdots(self):
|
||||
self.assertInvalidKey('foo--bar')
|
||||
self.assertInvalidKey('foo..bar')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
from multiprocessing.sharedctypes import Array
|
||||
from util.validation import MAX_LENGTH
|
||||
from util.validation import MAX_USERNAME_LENGTH
|
||||
|
||||
class SuperUserManager(object):
|
||||
""" In-memory helper class for quickly accessing (and updating) the valid
|
||||
|
|
@ -11,7 +11,7 @@ class SuperUserManager(object):
|
|||
usernames = app.config.get('SUPER_USERS', [])
|
||||
usernames_str = ','.join(usernames)
|
||||
|
||||
self._max_length = len(usernames_str) + MAX_LENGTH + 1
|
||||
self._max_length = len(usernames_str) + MAX_USERNAME_LENGTH + 1
|
||||
self._array = Array('c', self._max_length, lock=True)
|
||||
self._array.value = usernames_str
|
||||
|
||||
|
|
|
|||
20
util/label_validator.py
Normal file
20
util/label_validator.py
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
class LabelValidator(object):
|
||||
""" Helper class for validating that labels meet prefix requirements. """
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
overridden_prefixes = app.config.get('LABEL_KEY_RESERVED_PREFIXES', [])
|
||||
for prefix in overridden_prefixes:
|
||||
if not prefix.endswith('.'):
|
||||
raise Exception('Prefix "%s" in LABEL_KEY_RESERVED_PREFIXES must end in a dot', prefix)
|
||||
|
||||
default_prefixes = app.config.get('DEFAULT_LABEL_KEY_RESERVED_PREFIXES', [])
|
||||
self.reserved_prefixed_set = set(default_prefixes + overridden_prefixes)
|
||||
|
||||
def has_reserved_prefix(self, label_key):
|
||||
""" Validates that the provided label key does not match any reserved prefixes. """
|
||||
for prefix in self.reserved_prefixed_set:
|
||||
if label_key.startswith(prefix):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import logging
|
||||
|
||||
from sqlalchemy.types import TypeDecorator, Text
|
||||
from sqlalchemy.dialects.mysql import TEXT as MySQLText, LONGTEXT
|
||||
from sqlalchemy.types import TypeDecorator, Text, String
|
||||
from sqlalchemy.dialects.mysql import TEXT as MySQLText, LONGTEXT, VARCHAR as MySQLString
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -20,3 +20,19 @@ class UTF8LongText(TypeDecorator):
|
|||
return dialect.type_descriptor(LONGTEXT(charset='utf8mb4', collation='utf8mb4_unicode_ci'))
|
||||
else:
|
||||
return dialect.type_descriptor(Text())
|
||||
|
||||
|
||||
class UTF8CharField(TypeDecorator):
|
||||
""" Platform-independent UTF-8 Char type.
|
||||
|
||||
Uses MySQL's VARCHAR with charset utf8mb4, otherwise uses String, because
|
||||
other engines default to UTF-8.
|
||||
"""
|
||||
impl = String
|
||||
|
||||
def load_dialect_impl(self, dialect):
|
||||
if dialect.name == 'mysql':
|
||||
return dialect.type_descriptor(MySQLString(charset='utf8mb4', collation='utf8mb4_unicode_ci',
|
||||
length=self.impl.length))
|
||||
else:
|
||||
return dialect.type_descriptor(String(length=self.impl.length))
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import string
|
||||
import re
|
||||
import json
|
||||
|
||||
import anunidecode # Don't listen to pylint's lies. This import is required.
|
||||
|
||||
|
|
@ -9,14 +10,21 @@ INVALID_PASSWORD_MESSAGE = 'Invalid password, password must be at least ' + \
|
|||
INVALID_USERNAME_CHARACTERS = r'[^a-z0-9_]'
|
||||
VALID_CHARACTERS = string.digits + string.lowercase
|
||||
|
||||
MIN_LENGTH = 4
|
||||
MAX_LENGTH = 30
|
||||
MIN_USERNAME_LENGTH = 4
|
||||
MAX_USERNAME_LENGTH = 30
|
||||
|
||||
VALID_LABEL_KEY_REGEX = r'^[a-z0-9](([a-z0-9]|[-.](?![.-]))*[a-z0-9])?$'
|
||||
|
||||
|
||||
def validate_label_key(label_key):
|
||||
if len(label_key) > 255:
|
||||
return False
|
||||
|
||||
return bool(re.match(VALID_LABEL_KEY_REGEX, label_key))
|
||||
|
||||
|
||||
def validate_email(email_address):
|
||||
if re.match(r'[^@]+@[^@]+\.[^@]+', email_address):
|
||||
return True
|
||||
return False
|
||||
return bool(re.match(r'[^@]+@[^@]+\.[^@]+', email_address))
|
||||
|
||||
|
||||
def validate_username(username):
|
||||
|
|
@ -25,10 +33,10 @@ def validate_username(username):
|
|||
if not regex_match:
|
||||
return (False, 'Username must match expression [a-z0-9_]+')
|
||||
|
||||
length_match = (len(username) >= MIN_LENGTH and len(username) <= MAX_LENGTH)
|
||||
length_match = (len(username) >= MIN_USERNAME_LENGTH and len(username) <= MAX_USERNAME_LENGTH)
|
||||
if not length_match:
|
||||
return (False, 'Username must be between %s and %s characters in length' %
|
||||
(MIN_LENGTH, MAX_LENGTH))
|
||||
(MIN_USERNAME_LENGTH, MAX_USERNAME_LENGTH))
|
||||
|
||||
return (True, '')
|
||||
|
||||
|
|
@ -58,9 +66,20 @@ def generate_valid_usernames(input_username):
|
|||
if prefix.endswith('_'):
|
||||
prefix = prefix[0:len(prefix) - 1]
|
||||
|
||||
num_filler_chars = max(0, MIN_LENGTH - len(prefix))
|
||||
num_filler_chars = max(0, MIN_USERNAME_LENGTH - len(prefix))
|
||||
|
||||
while num_filler_chars + len(prefix) <= MAX_LENGTH:
|
||||
while num_filler_chars + len(prefix) <= MAX_USERNAME_LENGTH:
|
||||
for suffix in _gen_filler_chars(num_filler_chars):
|
||||
yield prefix + suffix
|
||||
num_filler_chars += 1
|
||||
|
||||
|
||||
def is_json(value):
|
||||
if ((value.startswith('{') and value.endswith('}')) or
|
||||
(value.startswith('[') and value.endswith(']'))):
|
||||
try:
|
||||
json.loads(value)
|
||||
return True
|
||||
except TypeError:
|
||||
return False
|
||||
|
||||
|
|
|
|||
Reference in a new issue