vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge remote-tracking branch 'upstream/v2-phase4' into python-registry-v2

Browse source
This commit is contained in:
Jake Moshenko 2015-10-22 16:59:28 -04:00
commit e7a6176594
105 changed files with 4439 additions and 2074 deletions
Download patch file Download diff file Expand all files Collapse all files

6
endpoints/api/__init__.py
View file

@ -49,10 +49,16 @@ class ApiException(Exception):
return rv
class ExternalServiceTimeout(ApiException):
def __init__(self, error_description, payload=None):
ApiException.__init__(self, 'external_service_timeout', 520, error_description, payload)
class InvalidRequest(ApiException):
def __init__(self, error_description, payload=None):
ApiException.__init__(self, 'invalid_request', 400, error_description, payload)
class InvalidResponse(ApiException):
def __init__(self, error_description, payload=None):
ApiException.__init__(self, 'invalid_response', 400, error_description, payload)

2
endpoints/api/build.py
View file

@ -9,12 +9,12 @@ from flask import request
from rfc3987 import parse as uri_parse
from app import app, userfiles as user_files, build_logs, log_archive, dockerfile_build_queue
from buildtrigger.basehandler import BuildTriggerHandler
from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource,
require_repo_read, require_repo_write, validate_json_request,
ApiResource, internal_only, format_date, api, Unauthorized, NotFound,
path_param, InvalidRequest, require_repo_admin)
from endpoints.building import start_build, PreparedBuild
from endpoints.trigger import BuildTriggerHandler
from data import database
from data import model
from auth.auth_context import get_authenticated_user

12
endpoints/api/image.py
View file

@ -12,10 +12,7 @@ from util.cache import cache_control_flask_restful
def image_view(image, image_map, include_ancestors=True):
# TODO: Remove this once we've migrated all storage data to the image records.
storage_props = image
if image.storage and image.storage.id:
storage_props = image.storage
command = image.command
def docker_id(aid):
if not aid or not aid in image_map:
@ -23,13 +20,12 @@ def image_view(image, image_map, include_ancestors=True):
return image_map[aid].docker_image_id
command = image.command or storage_props.command
image_data = {
'id': image.docker_image_id,
'created': format_date(image.created or storage_props.created),
'comment': image.comment or storage_props.comment,
'created': format_date(image.created),
'comment': image.comment,
'command': json.loads(command) if command else None,
'size': storage_props.image_size,
'size': image.storage.image_size,
'uploading': image.storage.uploading,
'sort_index': len(image.ancestors),
}

3
endpoints/api/logs.py
View file

@ -15,6 +15,7 @@ from auth import scopes
from app import avatar
LOGS_PER_PAGE = 50
MAX_PAGES = 20
def log_view(log, kinds):
view = {
@ -80,7 +81,7 @@ def _validate_logs_arguments(start_time, end_time, performer_name):
def get_logs(start_time, end_time, performer_name=None, repository=None, namespace=None, page=None):
(start_time, end_time, performer) = _validate_logs_arguments(start_time, end_time, performer_name)
page = page if page else 1
page = min(MAX_PAGES, page if page else 1)
kinds = model.log.get_log_entry_kinds()
logs = model.log.list_logs(start_time, end_time, performer=performer, repository=repository,
namespace=namespace, page=page, count=LOGS_PER_PAGE + 1)

15
endpoints/api/repository.py
View file

@ -23,6 +23,7 @@ from auth.permissions import (ModifyRepositoryPermission, AdministerRepositoryPe
CreateRepositoryPermission)
from auth.auth_context import get_authenticated_user
from auth import scopes
from util.names import REPOSITORY_NAME_REGEX
logger = logging.getLogger(__name__)
@ -104,6 +105,10 @@ class RepositoryList(ApiResource):
if visibility == 'private':
check_allowed_private_repos(namespace_name)
# Verify that the repository name is valid.
if not REPOSITORY_NAME_REGEX.match(repository_name):
raise InvalidRequest('Invalid repository name')
repo = model.repository.create_repository(namespace_name, repository_name, owner, visibility)
repo.description = req['description']
repo.save()
@ -141,6 +146,10 @@ class RepositoryList(ApiResource):
starred_repos = model.repository.get_user_starred_repositories(get_authenticated_user())
star_lookup = set([repo.id for repo in starred_repos])
# If the user asked for only public repositories, limit to only public repos.
if public and (not namespace and not starred):
username = None
# Find the matching repositories.
repositories = model.repository.get_visible_repositories(username=username,
limit=limit,
@ -172,6 +181,8 @@ class RepositoryList(ApiResource):
def get(self, args):
""" Fetch the list of repositories visible to the current user under a variety of situations.
"""
if not args['namespace'] and not args['starred'] and not args['public']:
raise InvalidRequest('namespace, starred or public are required for this API call')
repositories, star_lookup = self._load_repositories(args['namespace'], args['public'],
args['starred'], args['limit'],
@ -192,7 +203,7 @@ class RepositoryList(ApiResource):
'namespace': repo_obj.namespace_user.username,
'name': repo_obj.name,
'description': repo_obj.description,
'is_public': repo_obj.visibility.name == 'public'
'is_public': repo_obj.visibility_id == model.repository.get_public_repo_visibility().id,
}
repo_id = repo_obj.id
@ -243,7 +254,7 @@ class Repository(RepositoryParamResource):
tag_info = {
'name': tag.name,
'image_id': tag.image.docker_image_id,
'size': tag.image.storage.aggregate_size
'size': tag.image.aggregate_size
}
if tag.lifetime_start_ts > 0:

32
endpoints/api/search.py
View file

@ -95,38 +95,6 @@ class EntitySearch(ApiResource):
}
@resource('/v1/find/repository')
class FindRepositories(ApiResource):
""" Resource for finding repositories. """
@parse_args
@query_param('query', 'The prefix to use when querying for repositories.', type=str, default='')
@require_scope(scopes.READ_REPO)
@nickname('findRepos')
def get(self, args):
""" Get a list of repositories that match the specified prefix query. """
prefix = args['query']
def repo_view(repo):
return {
'namespace': repo.namespace_user.username,
'name': repo.name,
'description': repo.description
}
username = None
user = get_authenticated_user()
if user is not None:
username = user.username
matching = model.repository.get_matching_repositories(prefix, username)
return {
'repositories': [repo_view(repo) for repo in matching
if (repo.visibility.name == 'public' or
ReadRepositoryPermission(repo.namespace_user.username, repo.name).can())]
}
def search_entity_view(username, entity, get_short_name=None):
kind = 'user'
avatar_data = avatar.get_data_for_user(entity)

8
endpoints/api/tag.py
View file

@ -8,6 +8,7 @@ from endpoints.api import (resource, nickname, require_repo_read, require_repo_w
from endpoints.api.image import image_view
from data import model
from auth.auth_context import get_authenticated_user
from util.names import TAG_ERROR, TAG_REGEX
@resource('/v1/repository/<repopath:repository>/tag/')
@ -85,6 +86,10 @@ class RepositoryTag(RepositoryParamResource):
@validate_json_request('MoveTag')
def put(self, namespace, repository, tag):
""" Change which image a tag points to or create a new tag."""
if not TAG_REGEX.match(tag):
abort(400, TAG_ERROR)
image_id = request.get_json()['image']
image = model.image.get_repo_image(namespace, repository, image_id)
if not image:
@ -100,7 +105,6 @@ class RepositoryTag(RepositoryParamResource):
pass
model.tag.create_or_update_tag(namespace, repository, tag, image_id)
model.repository.garbage_collect_repository(namespace, repository)
username = get_authenticated_user().username
log_action('move_tag' if original_image_id else 'create_tag', namespace,
@ -115,7 +119,6 @@ class RepositoryTag(RepositoryParamResource):
def delete(self, namespace, repository, tag):
""" Delete the specified repository tag. """
model.tag.delete_tag(namespace, repository, tag)
model.repository.garbage_collect_repository(namespace, repository)
username = get_authenticated_user().username
log_action('delete_tag', namespace,
@ -188,7 +191,6 @@ class RevertTag(RepositoryParamResource):
# Revert the tag back to the previous image.
image_id = request.get_json()['image']
model.tag.revert_tag(tag_image.repository, tag, image_id)
model.repository.garbage_collect_repository(namespace, repository)
# Log the reversion.
username = get_authenticated_user().username

7
endpoints/api/trigger.py
View file

@ -8,15 +8,16 @@ from urllib import quote
from urlparse import urlunparse
from app import app
from buildtrigger.basehandler import BuildTriggerHandler
from buildtrigger.triggerutil import (TriggerDeactivationException,
TriggerActivationException, EmptyRepositoryException,
RepositoryReadException, TriggerStartException)
from endpoints.api import (RepositoryParamResource, nickname, resource, require_repo_admin,
log_action, request_error, query_param, parse_args, internal_only,
validate_json_request, api, Unauthorized, NotFound, InvalidRequest,
path_param)
from endpoints.api.build import build_status_view, trigger_view, RepositoryBuildStatus
from endpoints.building import start_build
from endpoints.trigger import (BuildTriggerHandler, TriggerDeactivationException,
TriggerActivationException, EmptyRepositoryException,
RepositoryReadException, TriggerStartException)
from data import model
from auth.permissions import (UserAdminPermission, AdministerOrganizationPermission,
ReadRepositoryPermission)

28
endpoints/api/user.py
View file

@ -62,16 +62,22 @@ def handle_invite_code(invite_code, user):
def user_view(user):
def org_view(o):
def org_view(o, user_admin=True):
admin_org = AdministerOrganizationPermission(o.username)
return {
org_response = {
'name': o.username,
'avatar': avatar.get_data_for_org(o),
'is_org_admin': admin_org.can(),
'can_create_repo': admin_org.can() or CreateRepositoryPermission(o.username).can(),
'preferred_namespace': not (o.stripe_id is None)
'can_create_repo': CreateRepositoryPermission(o.username).can(),
}
if user_admin:
org_response.update({
'is_org_admin': admin_org.can(),
'preferred_namespace': not (o.stripe_id is None),
})
return org_response
organizations = model.organization.get_user_organizations(user.username)
def login_view(login):
@ -91,23 +97,29 @@ def user_view(user):
user_response = {
'anonymous': False,
'username': user.username,
'avatar': avatar.get_data_for_user(user)
'avatar': avatar.get_data_for_user(user),
}
user_admin = UserAdminPermission(user.username)
if user_admin.can():
user_response.update({
'can_create_repo': True,
'is_me': True,
'verified': user.verified,
'email': user.email,
'organizations': [org_view(o) for o in organizations],
'logins': [login_view(login) for login in logins],
'can_create_repo': True,
'invoice_email': user.invoice_email,
'preferred_namespace': not (user.stripe_id is None),
'tag_expiration': user.removed_tag_expiration_s,
})
user_view_perm = UserReadPermission(user.username)
if user_view_perm.can():
user_response.update({
'organizations': [org_view(o, user_admin=user_admin.can()) for o in organizations],
})
if features.SUPER_USERS and SuperUserPermission().can():
user_response.update({
'super_user': user and user == get_authenticated_user() and SuperUserPermission().can()

3
endpoints/bitbuckettrigger.py
View file

@ -3,7 +3,8 @@ import logging
from flask import request, redirect, url_for, Blueprint
from flask.ext.login import current_user
from endpoints.trigger import BitbucketBuildTrigger, BuildTriggerHandler
from buildtrigger.basehandler import BuildTriggerHandler
from buildtrigger.bitbuckethandler import BitbucketBuildTrigger
from endpoints.common import route_show_if
from app import app
from data import model

2
endpoints/building.py
View file

@ -96,7 +96,7 @@ class PreparedBuild(object):
def get_display_name(sha):
return sha[0:7]
def tags_from_ref(self, ref, default_branch='master'):
def tags_from_ref(self, ref, default_branch=None):
branch = ref.split('/')[-1]
tags = {branch}

1589
endpoints/trigger.py
View file

File diff suppressed because it is too large Load diff

45
endpoints/v1/index.py
View file

@ -9,7 +9,7 @@ from data import model
from app import app, authentication, userevents, storage
from auth.auth import process_auth, generate_signed_token
from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token
from util.names import parse_repository_name
from util.names import parse_repository_name, REPOSITORY_NAME_REGEX
from auth.permissions import (ModifyRepositoryPermission, UserAdminPermission,
ReadRepositoryPermission, CreateRepositoryPermission,
repository_read_grant, repository_write_grant)
@ -173,6 +173,10 @@ def update_user(username):
@generate_headers(scope=GrantType.WRITE_REPOSITORY, add_grant_for_status=201)
@anon_allowed
def create_repository(namespace, repository):
# Verify that the repository name is valid.
if not REPOSITORY_NAME_REGEX.match(repository):
abort(400, message='Invalid repository name. Repository names cannot contain slashes.')
logger.debug('Looking up repository %s/%s', namespace, repository)
repo = model.repository.get_repository(namespace, repository)
@ -232,9 +236,6 @@ def update_images(namespace, repository):
# Make sure the repo actually exists.
abort(404, message='Unknown repository', issue='unknown-repo')
logger.debug('GCing repository')
model.repository.garbage_collect_repository(namespace, repository)
# Generate a job for each notification that has been added to this repo
logger.debug('Adding notifications for repository')
@ -292,16 +293,31 @@ def put_repository_auth(namespace, repository):
abort(501, 'Not Implemented', issue='not-implemented')
def conduct_repo_search(username, query, results):
""" Finds matching repositories. """
def can_read(repo):
if repo.is_public:
return True
return ReadRepositoryPermission(repo.namespace_user.username, repo.name).can()
only_public = username is None
matching_repos = model.repository.get_sorted_matching_repositories(query, only_public, can_read,
limit=5)
for repo in matching_repos:
results.append({
'name': repo.name,
'description': repo.description,
'is_public': repo.is_public,
'href': '/repository/' + repo.namespace_user.username + '/' + repo.name
})
@v1_bp.route('/search', methods=['GET'])
@process_auth
@anon_protect
def get_search():
def result_view(repo):
return {
"name": repo.namespace_user.username + '/' + repo.name,
"description": repo.description
}
query = request.args.get('q')
username = None
@ -309,14 +325,9 @@ def get_search():
if user is not None:
username = user.username
results = []
if query:
matching = model.repository.get_matching_repositories(query, username)
else:
matching = []
results = [result_view(repo) for repo in matching
if (repo.visibility.name == 'public' or
ReadRepositoryPermission(repo.namespace_user.username, repo.name).can())]
conduct_repo_search(username, query, results)
data = {
"query": query,

33
endpoints/v1/registry.py
View file

@ -193,11 +193,11 @@ def put_image_layer(namespace, repository, image_id):
repo_image = model.image.get_repo_image_extended(namespace, repository, image_id)
try:
logger.debug('Retrieving image data')
json_data = model.image.get_image_json(repo_image)
except (IOError, AttributeError):
uuid = repo_image.storage.uuid
json_data = repo_image.v1_json_metadata
except (AttributeError):
logger.exception('Exception when retrieving image data')
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
image_id=image_id)
abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id)
uuid = repo_image.storage.uuid
layer_path = store.v1_image_layer_path(uuid)
@ -241,15 +241,15 @@ def put_image_layer(namespace, repository, image_id):
logger.exception('Exception when writing image data')
abort(520, 'Image %(image_id)s could not be written. Please try again.', image_id=image_id)
# Save the size of the image.
model.image.set_image_size(image_id, namespace, repository, size_info.compressed_size,
size_info.uncompressed_size)
# Append the computed checksum.
csums = []
csums.append('sha256:{0}'.format(h.hexdigest()))
try:
# Save the size of the image.
model.image.set_image_size(image_id, namespace, repository, size_info.compressed_size,
size_info.uncompressed_size)
if requires_tarsum:
tmp.seek(0)
csums.append(checksums.compute_tarsum(tmp, json_data))
@ -315,7 +315,7 @@ def put_image_checksum(namespace, repository, image_id):
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
logger.debug('Looking up repo layer data')
if not model.image.has_image_json(repo_image):
if not repo_image.v1_json_metadata:
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
logger.debug('Marking image path')
@ -355,21 +355,17 @@ def get_image_json(namespace, repository, image_id, headers):
logger.debug('Looking up repo image')
repo_image = model.image.get_repo_image_extended(namespace, repository, image_id)
logger.debug('Looking up repo layer data')
try:
data = model.image.get_image_json(repo_image)
except (IOError, AttributeError):
if repo_image is None:
flask_abort(404)
logger.debug('Looking up repo layer size')
size = repo_image.storage.image_size
headers['Content-Type'] = 'application/json'
if size is not None:
# Note: X-Docker-Size is optional and we *can* end up with a NULL image_size,
# so handle this case rather than failing.
headers['X-Docker-Size'] = str(size)
response = make_response(data, 200)
response = make_response(repo_image.v1_json_metadata, 200)
response.headers.extend(headers)
return response
@ -472,7 +468,8 @@ def put_image_json(namespace, repository, image_id):
abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s',
issue='invalid-request', image_id=image_id, parent_id=parent_id)
if not image_is_uploading(repo_image) and model.image.has_image_json(repo_image):
logger.debug('Checking if image already exists')
if repo_image.v1_json_metadata and not image_is_uploading(repo_image):
exact_abort(409, 'Image already exists')
set_uploading_flag(repo_image, True)

7
endpoints/v1/tag.py
View file

@ -5,7 +5,7 @@ import json
from flask import abort, request, jsonify, make_response, session
from app import app
from util.names import parse_repository_name
from util.names import TAG_ERROR, TAG_REGEX, parse_repository_name
from auth.auth import process_auth
from auth.permissions import (ReadRepositoryPermission,
ModifyRepositoryPermission)
@ -60,6 +60,9 @@ def put_tag(namespace, repository, tag):
permission = ModifyRepositoryPermission(namespace, repository)
if permission.can():
if not TAG_REGEX.match(tag):
abort(400, TAG_ERROR)
docker_image_id = json.loads(request.data)
model.tag.create_or_update_tag(namespace, repository, tag, docker_image_id)
@ -83,8 +86,6 @@ def delete_tag(namespace, repository, tag):
if permission.can():
model.tag.delete_tag(namespace, repository, tag)
model.repository.garbage_collect_repository(namespace, repository)
return make_response('Deleted', 200)
abort(403)

16
endpoints/v2/manifest.py
View file

@ -383,10 +383,10 @@ def _generate_and_store_manifest(namespace, repo_name, tag_name):
builder = SignedManifestBuilder(namespace, repo_name, tag_name)
# Add the leaf layer
builder.add_layer(image.storage.checksum, __get_and_backfill_image_metadata(image))
builder.add_layer(image.storage.checksum, image.v1_json_metadata)
for parent in parents:
builder.add_layer(parent.storage.checksum, __get_and_backfill_image_metadata(parent))
builder.add_layer(parent.storage.checksum, parent.v1_json_metadata)
# Sign the manifest with our signing key.
manifest = builder.build(docker_v2_signing_key)
@ -394,15 +394,3 @@ def _generate_and_store_manifest(namespace, repo_name, tag_name):
manifest.digest, manifest.bytes)
return manifest_row
def __get_and_backfill_image_metadata(image):
image_metadata = image.v1_json_metadata
if image_metadata is None:
logger.warning('Loading metadata from storage for image id: %s', image.id)
image.v1_json_metadata = model.image.get_image_json(image)
logger.info('Saving backfilled metadata for image id: %s', image.id)
image.save()
return image_metadata

9
endpoints/verbs.py
View file

@ -27,7 +27,7 @@ def _open_stream(formatter, namespace, repository, tag, synthetic_image_id, imag
store = Storage(app)
def get_image_json(image):
return json.loads(model.image.get_image_json(image))
return json.loads(image.v1_json_metadata)
def get_next_image():
for current_image in image_list:
@ -113,7 +113,7 @@ def _verify_repo_verb(store, namespace, repository, tag, verb, checker=None):
abort(404)
# Lookup the tag's image and storage.
repo_image = model.image.get_repo_image_extended(namespace, repository, tag_image.docker_image_id)
repo_image = model.image.get_repo_image(namespace, repository, tag_image.docker_image_id)
if not repo_image:
abort(404)
@ -121,7 +121,8 @@ def _verify_repo_verb(store, namespace, repository, tag, verb, checker=None):
image_json = None
if checker is not None:
image_json = json.loads(model.image.get_image_json(repo_image))
image_json = json.loads(repo_image.v1_json_metadata)
if not checker(image_json):
logger.debug('Check mismatch on %s/%s:%s, verb %s', namespace, repository, tag, verb)
abort(404)
@ -187,7 +188,7 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=
# Load the image's JSON layer.
if not image_json:
image_json = json.loads(model.image.get_image_json(repo_image))
image_json = json.loads(repo_image.v1_json_metadata)
# Calculate a synthetic image ID.
synthetic_image_id = hashlib.sha256(tag_image.docker_image_id + ':' + verb).hexdigest()

8
endpoints/web.py
View file

@ -21,8 +21,12 @@ from util.cache import no_cache
from endpoints.common import common_login, render_page_template, route_show_if, param_required
from endpoints.decorators import anon_protect
from endpoints.csrf import csrf_protect, generate_csrf_token, verify_csrf
from endpoints.trigger import (CustomBuildTrigger, BitbucketBuildTrigger, TriggerProviderException,
BuildTriggerHandler)
from buildtrigger.customhandler import CustomBuildTrigger
from buildtrigger.bitbuckethandler import BitbucketBuildTrigger
from buildtrigger.triggerutil import TriggerProviderException
from buildtrigger.basehandler import BuildTriggerHandler
from util.names import parse_repository_name, parse_repository_name_and_tag
from util.useremails import send_email_changed
from util.systemlogs import build_logs_archive

5
endpoints/webhooks.py
View file

@ -9,8 +9,9 @@ from auth.permissions import ModifyRepositoryPermission
from util.invoice import renderInvoiceToHtml
from util.useremails import send_invoice_email, send_subscription_change, send_payment_failed
from util.http import abort
from endpoints.trigger import (BuildTriggerHandler, ValidationRequestException,
SkipRequestException, InvalidPayloadException)
from buildtrigger.basehandler import BuildTriggerHandler
from buildtrigger.triggerutil import (ValidationRequestException, SkipRequestException,
InvalidPayloadException)
from endpoints.building import start_build