Merge remote-tracking branch 'upstream/phase4-11-07-2015' into python-registry-v2
This commit is contained in:
commit
c2fcf8bead
177 changed files with 4354 additions and 1462 deletions
|
@ -93,6 +93,11 @@ class NotFound(ApiException):
|
|||
ApiException.__init__(self, None, 404, 'Not Found', payload)
|
||||
|
||||
|
||||
class DownstreamIssue(ApiException):
|
||||
def __init__(self, payload=None):
|
||||
ApiException.__init__(self, None, 520, 'Downstream Issue', payload)
|
||||
|
||||
|
||||
@api_bp.app_errorhandler(ApiException)
|
||||
@crossdomain(origin='*', headers=['Authorization', 'Content-Type'])
|
||||
def handle_api_error(error):
|
||||
|
@ -105,14 +110,6 @@ def handle_api_error(error):
|
|||
return response
|
||||
|
||||
|
||||
@api_bp.app_errorhandler(model.TooManyLoginAttemptsException)
|
||||
@crossdomain(origin='*', headers=['Authorization', 'Content-Type'])
|
||||
def handle_too_many_login_attempts(error):
|
||||
response = make_response('Too many login attempts', 429)
|
||||
response.headers['Retry-After'] = int(error.retry_after)
|
||||
return response
|
||||
|
||||
|
||||
def resource(*urls, **kwargs):
|
||||
def wrapper(api_resource):
|
||||
if not api_resource:
|
||||
|
@ -426,4 +423,5 @@ import endpoints.api.tag
|
|||
import endpoints.api.team
|
||||
import endpoints.api.trigger
|
||||
import endpoints.api.user
|
||||
import endpoints.api.secscan
|
||||
|
||||
|
|
|
@ -241,10 +241,10 @@ def swagger_route_data(include_internal=False, compact=False):
|
|||
],
|
||||
'info': {
|
||||
'version': 'v1',
|
||||
'title': 'Quay.io Frontend',
|
||||
'title': 'Quay Frontend',
|
||||
'description': ('This API allows you to perform many of the operations required to work '
|
||||
'with Quay.io repositories, users, and organizations. You can find out more '
|
||||
'at <a href="https://quay.io">Quay.io</a>.'),
|
||||
'with Quay repositories, users, and organizations. You can find out more '
|
||||
'at <a href="https://quay.io">Quay</a>.'),
|
||||
'termsOfService': 'https://quay.io/tos',
|
||||
'contact': {
|
||||
'email': 'support@quay.io'
|
||||
|
|
|
@ -57,6 +57,10 @@ class RepositoryNotificationList(RepositoryParamResource):
|
|||
'type': 'object',
|
||||
'description': 'JSON config information for the specific method of notification'
|
||||
},
|
||||
'eventConfig': {
|
||||
'type': 'object',
|
||||
'description': 'JSON config information for the specific event of notification',
|
||||
},
|
||||
'title': {
|
||||
'type': 'string',
|
||||
'description': 'The human-readable title of the notification',
|
||||
|
@ -84,6 +88,7 @@ class RepositoryNotificationList(RepositoryParamResource):
|
|||
|
||||
new_notification = model.notification.create_repo_notification(repo, parsed['event'],
|
||||
parsed['method'], parsed['config'],
|
||||
parsed['eventConfig'],
|
||||
parsed.get('title', None))
|
||||
|
||||
resp = notification_view(new_notification)
|
||||
|
|
103
endpoints/api/secscan.py
Normal file
103
endpoints/api/secscan.py
Normal file
|
@ -0,0 +1,103 @@
|
|||
""" List and manage repository vulnerabilities and other sec information. """
|
||||
|
||||
import logging
|
||||
import features
|
||||
import json
|
||||
import requests
|
||||
|
||||
from app import secscan_endpoint
|
||||
from data import model
|
||||
from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_param,
|
||||
RepositoryParamResource, resource, nickname, show_if, parse_args,
|
||||
query_param)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _call_security_api(relative_url, *args, **kwargs):
|
||||
""" Issues an HTTP call to the sec API at the given relative URL. """
|
||||
try:
|
||||
response = secscan_endpoint.call_api(relative_url, *args, **kwargs)
|
||||
except requests.exceptions.Timeout:
|
||||
raise DownstreamIssue(payload=dict(message='API call timed out'))
|
||||
except requests.exceptions.ConnectionError:
|
||||
raise DownstreamIssue(payload=dict(message='Could not connect to downstream service'))
|
||||
|
||||
if response.status_code == 404:
|
||||
raise NotFound()
|
||||
|
||||
try:
|
||||
response_data = json.loads(response.text)
|
||||
except ValueError:
|
||||
raise DownstreamIssue(payload=dict(message='Non-json response from downstream service'))
|
||||
|
||||
if response.status_code / 100 != 2:
|
||||
logger.warning('Got %s status code to call: %s', response.status_code, response.text)
|
||||
raise DownstreamIssue(payload=dict(message=response_data['Message']))
|
||||
|
||||
return response_data
|
||||
|
||||
|
||||
@show_if(features.SECURITY_SCANNER)
|
||||
@resource('/v1/repository/<repopath:repository>/tag/<tag>/vulnerabilities')
|
||||
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
|
||||
@path_param('tag', 'The name of the tag')
|
||||
class RepositoryTagVulnerabilities(RepositoryParamResource):
|
||||
""" Operations for managing the vulnerabilities in a repository tag. """
|
||||
|
||||
@require_repo_read
|
||||
@nickname('getRepoTagVulnerabilities')
|
||||
@parse_args
|
||||
@query_param('minimumPriority', 'Minimum vulnerability priority', type=str,
|
||||
default='Low')
|
||||
def get(self, args, namespace, repository, tag):
|
||||
""" Fetches the vulnerabilities (if any) for a repository tag. """
|
||||
try:
|
||||
tag_image = model.tag.get_tag_image(namespace, repository, tag)
|
||||
except model.DataModelException:
|
||||
raise NotFound()
|
||||
|
||||
if not tag_image.security_indexed:
|
||||
logger.debug('Image %s for tag %s under repository %s/%s not security indexed',
|
||||
tag_image.docker_image_id, tag, namespace, repository)
|
||||
return {
|
||||
'security_indexed': False
|
||||
}
|
||||
|
||||
data = _call_security_api('layers/%s/vulnerabilities', tag_image.docker_image_id,
|
||||
minimumPriority=args.minimumPriority)
|
||||
|
||||
return {
|
||||
'security_indexed': True,
|
||||
'data': data,
|
||||
}
|
||||
|
||||
|
||||
@show_if(features.SECURITY_SCANNER)
|
||||
@resource('/v1/repository/<repopath:repository>/image/<imageid>/packages')
|
||||
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
|
||||
@path_param('imageid', 'The image ID')
|
||||
class RepositoryImagePackages(RepositoryParamResource):
|
||||
""" Operations for listing the packages added/removed in an image. """
|
||||
|
||||
@require_repo_read
|
||||
@nickname('getRepoImagePackages')
|
||||
def get(self, namespace, repository, imageid):
|
||||
""" Fetches the packages added/removed in the given repo image. """
|
||||
repo_image = model.image.get_repo_image(namespace, repository, imageid)
|
||||
if repo_image is None:
|
||||
raise NotFound()
|
||||
|
||||
if not repo_image.security_indexed:
|
||||
return {
|
||||
'security_indexed': False
|
||||
}
|
||||
|
||||
data = _call_security_api('layers/%s/packages/diff', repo_image.docker_image_id)
|
||||
|
||||
return {
|
||||
'security_indexed': True,
|
||||
'data': data,
|
||||
}
|
||||
|
|
@ -9,7 +9,7 @@ from flask import request
|
|||
|
||||
import features
|
||||
|
||||
from app import app, avatar, superusers, authentication
|
||||
from app import app, avatar, superusers, authentication, config_provider
|
||||
from endpoints.api import (ApiResource, nickname, resource, validate_json_request,
|
||||
internal_only, require_scope, show_if, parse_args,
|
||||
query_param, abort, require_fresh_login, path_param, verify_not_prod)
|
||||
|
@ -131,6 +131,7 @@ class SuperUserLogs(ApiResource):
|
|||
def org_view(org):
|
||||
return {
|
||||
'name': org.username,
|
||||
'email': org.email,
|
||||
'avatar': avatar.get_data_for_org(org),
|
||||
}
|
||||
|
||||
|
@ -236,6 +237,10 @@ class SuperUserList(ApiResource):
|
|||
@require_scope(scopes.SUPERUSER)
|
||||
def post(self):
|
||||
""" Creates a new user. """
|
||||
# Ensure that we are using database auth.
|
||||
if app.config['AUTHENTICATION_TYPE'] != 'Database':
|
||||
abort(400)
|
||||
|
||||
user_information = request.get_json()
|
||||
if SuperUserPermission().can():
|
||||
username = user_information['username']
|
||||
|
@ -274,6 +279,10 @@ class SuperUserSendRecoveryEmail(ApiResource):
|
|||
@nickname('sendInstallUserRecoveryEmail')
|
||||
@require_scope(scopes.SUPERUSER)
|
||||
def post(self, username):
|
||||
# Ensure that we are using database auth.
|
||||
if app.config['AUTHENTICATION_TYPE'] != 'Database':
|
||||
abort(400)
|
||||
|
||||
if SuperUserPermission().can():
|
||||
user = model.user.get_nonrobot_user(username)
|
||||
if not user:
|
||||
|
@ -370,9 +379,17 @@ class SuperUserManagement(ApiResource):
|
|||
|
||||
user_data = request.get_json()
|
||||
if 'password' in user_data:
|
||||
# Ensure that we are using database auth.
|
||||
if app.config['AUTHENTICATION_TYPE'] != 'Database':
|
||||
abort(400)
|
||||
|
||||
model.user.change_password(user, user_data['password'])
|
||||
|
||||
if 'email' in user_data:
|
||||
# Ensure that we are using database auth.
|
||||
if app.config['AUTHENTICATION_TYPE'] != 'Database':
|
||||
abort(400)
|
||||
|
||||
model.user.update_email(user, user_data['email'], auto_verify=True)
|
||||
|
||||
if 'enabled' in user_data:
|
||||
|
@ -380,6 +397,18 @@ class SuperUserManagement(ApiResource):
|
|||
user.enabled = bool(user_data['enabled'])
|
||||
user.save()
|
||||
|
||||
if 'superuser' in user_data:
|
||||
config_object = config_provider.get_config()
|
||||
superusers_set = set(config_object['SUPER_USERS'])
|
||||
|
||||
if user_data['superuser']:
|
||||
superusers_set.add(username)
|
||||
elif username in superusers_set:
|
||||
superusers_set.remove(username)
|
||||
|
||||
config_object['SUPER_USERS'] = list(superusers_set)
|
||||
config_provider.save_config(config_object)
|
||||
|
||||
return user_view(user, password=user_data.get('password'))
|
||||
|
||||
abort(403)
|
||||
|
|
|
@ -4,7 +4,7 @@ from flask import request, abort
|
|||
|
||||
from endpoints.api import (resource, nickname, require_repo_read, require_repo_write,
|
||||
RepositoryParamResource, log_action, NotFound, validate_json_request,
|
||||
path_param, parse_args, query_param)
|
||||
path_param, parse_args, query_param, truthy_bool)
|
||||
from endpoints.api.image import image_view
|
||||
from data import model
|
||||
from auth.auth_context import get_authenticated_user
|
||||
|
@ -135,7 +135,10 @@ class RepositoryTagImages(RepositoryParamResource):
|
|||
""" Resource for listing the images in a specific repository tag. """
|
||||
@require_repo_read
|
||||
@nickname('listTagImages')
|
||||
def get(self, namespace, repository, tag):
|
||||
@parse_args
|
||||
@query_param('owned', 'If specified, only images wholely owned by this tag are returned.',
|
||||
type=truthy_bool, default=False)
|
||||
def get(self, args, namespace, repository, tag):
|
||||
""" List the images for the specified repository tag. """
|
||||
try:
|
||||
tag_image = model.tag.get_tag_image(namespace, repository, tag)
|
||||
|
@ -144,15 +147,37 @@ class RepositoryTagImages(RepositoryParamResource):
|
|||
|
||||
parent_images = model.image.get_parent_images(namespace, repository, tag_image)
|
||||
image_map = {}
|
||||
|
||||
image_map[str(tag_image.id)] = tag_image
|
||||
|
||||
for image in parent_images:
|
||||
image_map[str(image.id)] = image
|
||||
|
||||
image_map_all = dict(image_map)
|
||||
|
||||
parents = list(parent_images)
|
||||
parents.reverse()
|
||||
all_images = [tag_image] + parents
|
||||
|
||||
# Filter the images returned to those not found in the ancestry of any of the other tags in
|
||||
# the repository.
|
||||
if args['owned']:
|
||||
all_tags = model.tag.list_repository_tags(namespace, repository)
|
||||
for current_tag in all_tags:
|
||||
if current_tag.name == tag:
|
||||
continue
|
||||
|
||||
# Remove the tag's image ID.
|
||||
tag_image_id = str(current_tag.image_id)
|
||||
image_map.pop(tag_image_id, None)
|
||||
|
||||
# Remove any ancestors:
|
||||
for ancestor_id in current_tag.image.ancestors.split('/'):
|
||||
image_map.pop(ancestor_id, None)
|
||||
|
||||
return {
|
||||
'images': [image_view(image, image_map) for image in all_images]
|
||||
'images': [image_view(image, image_map_all) for image in all_images
|
||||
if not args['owned'] or (str(image.id) in image_map)]
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -5,6 +5,7 @@ from flask import make_response
|
|||
from app import app
|
||||
from util.useremails import CannotSendEmailException
|
||||
from util.config.provider.baseprovider import CannotWriteConfigException
|
||||
from flask.ext.restful.utils.cors import crossdomain
|
||||
from data import model
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
@ -25,4 +26,13 @@ def handle_configexception(ex):
|
|||
'Please make sure the mounted volume is not read-only and restart ' +
|
||||
'the setup process. \n\nIssue: %s' % ex)
|
||||
|
||||
return make_response(json.dumps({'message': message}), 400)
|
||||
return make_response(json.dumps({'message': message}), 400)
|
||||
|
||||
@app.errorhandler(model.TooManyLoginAttemptsException)
|
||||
@crossdomain(origin='*', headers=['Authorization', 'Content-Type'])
|
||||
def handle_too_many_login_attempts(error):
|
||||
msg = 'Too many login attempts. \nPlease reset your Quay password and try again.'
|
||||
response = make_response(msg, 429)
|
||||
response.headers['Retry-After'] = int(error.retry_after)
|
||||
return response
|
||||
|
||||
|
|
|
@ -84,6 +84,40 @@ def _build_summary(event_data):
|
|||
return summary
|
||||
|
||||
|
||||
class VulnerabilityFoundEvent(NotificationEvent):
|
||||
@classmethod
|
||||
def event_name(cls):
|
||||
return 'vulnerability_found'
|
||||
|
||||
def get_level(self, event_data, notification_data):
|
||||
priority = event_data['vulnerability']['priority']
|
||||
if priority == 'Defcon1' or priority == 'Critical':
|
||||
return 'error'
|
||||
|
||||
if priority == 'Medium' or priority == 'High':
|
||||
return 'warning'
|
||||
|
||||
return 'info'
|
||||
|
||||
def get_sample_data(self, repository):
|
||||
return build_event_data(repository, {
|
||||
'tags': ['latest', 'prod'],
|
||||
'image': 'some-image-id',
|
||||
'vulnerability': {
|
||||
'id': 'CVE-FAKE-CVE',
|
||||
'description': 'A futurist vulnerability',
|
||||
'link': 'https://security-tracker.debian.org/tracker/CVE-FAKE-CVE',
|
||||
'priority': 'Critical',
|
||||
},
|
||||
})
|
||||
|
||||
def get_summary(self, event_data, notification_data):
|
||||
msg = '%s vulnerability detected in repository %s in tags %s'
|
||||
return msg % (event_data['vulnerability']['priority'],
|
||||
event_data['repository'],
|
||||
', '.join(event_data['tags']))
|
||||
|
||||
|
||||
class BuildQueueEvent(NotificationEvent):
|
||||
@classmethod
|
||||
def event_name(cls):
|
||||
|
|
|
@ -152,7 +152,7 @@ def get_image_layer(namespace, repository, image_id, headers):
|
|||
image_id=image_id)
|
||||
|
||||
try:
|
||||
path = store.blob_path(repo_image.storage.checksum)
|
||||
path = store.blob_path(repo_image.storage.content_checksum)
|
||||
if not repo_image.storage.cas_path:
|
||||
path = store.v1_image_layer_path(repo_image.storage.uuid)
|
||||
logger.info('Serving legacy v1 image from path: %s', path)
|
||||
|
@ -233,6 +233,10 @@ def put_image_layer(namespace, repository, image_id):
|
|||
h, sum_hndlr = checksums.simple_checksum_handler(json_data)
|
||||
sr.add_handler(sum_hndlr)
|
||||
|
||||
# Add a handler which computes the content checksum only
|
||||
ch, content_sum_hndlr = checksums.content_checksum_handler()
|
||||
sr.add_handler(content_sum_hndlr)
|
||||
|
||||
# Stream write the data to storage.
|
||||
with database.CloseForLongOperation(app.config):
|
||||
try:
|
||||
|
@ -258,13 +262,14 @@ def put_image_layer(namespace, repository, image_id):
|
|||
except (IOError, checksums.TarError) as exc:
|
||||
logger.debug('put_image_layer: Error when computing tarsum %s', exc)
|
||||
|
||||
if repo_image.storage.checksum is None:
|
||||
if repo_image.v1_checksum is None:
|
||||
# We don't have a checksum stored yet, that's fine skipping the check.
|
||||
# Not removing the mark though, image is not downloadable yet.
|
||||
session['checksum'] = csums
|
||||
session['content_checksum'] = 'sha256:{0}'.format(ch.hexdigest())
|
||||
return make_response('true', 200)
|
||||
|
||||
checksum = repo_image.storage.checksum
|
||||
checksum = repo_image.v1_checksum
|
||||
|
||||
# We check if the checksums provided matches one the one we computed
|
||||
if checksum not in csums:
|
||||
|
@ -323,8 +328,9 @@ def put_image_checksum(namespace, repository, image_id):
|
|||
abort(409, 'Cannot set checksum for image %(image_id)s',
|
||||
issue='image-write-error', image_id=image_id)
|
||||
|
||||
logger.debug('Storing image checksum')
|
||||
err = store_checksum(repo_image.storage, checksum)
|
||||
logger.debug('Storing image and content checksums')
|
||||
content_checksum = session.get('content_checksum', None)
|
||||
err = store_checksum(repo_image, checksum, content_checksum)
|
||||
if err:
|
||||
abort(400, err)
|
||||
|
||||
|
@ -398,14 +404,17 @@ def get_image_ancestry(namespace, repository, image_id, headers):
|
|||
return response
|
||||
|
||||
|
||||
def store_checksum(image_storage, checksum):
|
||||
def store_checksum(image_with_storage, checksum, content_checksum):
|
||||
checksum_parts = checksum.split(':')
|
||||
if len(checksum_parts) != 2:
|
||||
return 'Invalid checksum format'
|
||||
|
||||
# We store the checksum
|
||||
image_storage.checksum = checksum
|
||||
image_storage.save()
|
||||
image_with_storage.storage.content_checksum = content_checksum
|
||||
image_with_storage.storage.save()
|
||||
|
||||
image_with_storage.v1_checksum = checksum
|
||||
image_with_storage.save()
|
||||
|
||||
|
||||
@v1_bp.route('/images/<image_id>/json', methods=['PUT'])
|
||||
|
@ -519,7 +528,7 @@ def process_image_changes(namespace, repository, image_id):
|
|||
parent_trie.frombytes(parent_trie_bytes)
|
||||
|
||||
# Read in the file entries from the layer tar file
|
||||
layer_path = store.blob_path(repo_image.storage.checksum)
|
||||
layer_path = store.blob_path(repo_image.storage.content_checksum)
|
||||
if not repo_image.storage.cas_path:
|
||||
logger.info('Processing diffs for newly stored v1 image at %s', layer_path)
|
||||
layer_path = store.v1_image_layer_path(uuid)
|
||||
|
|
|
@ -301,8 +301,8 @@ def _write_manifest(namespace, repo_name, manifest):
|
|||
|
||||
# Lookup the storages associated with each blob in the manifest.
|
||||
checksums = [str(mdata.digest) for mdata in manifest.layers]
|
||||
storage_query = model.storage.lookup_repo_storages_by_checksum(repo, checksums)
|
||||
storage_map = {storage.checksum: storage for storage in storage_query}
|
||||
storage_query = model.storage.lookup_repo_storages_by_content_checksum(repo, checksums)
|
||||
storage_map = {storage.content_checksum: storage for storage in storage_query}
|
||||
|
||||
# Synthesize the V1 metadata for each layer.
|
||||
manifest_digest = manifest.digest
|
||||
|
@ -387,10 +387,10 @@ def _generate_and_store_manifest(namespace, repo_name, tag_name):
|
|||
builder = SignedManifestBuilder(namespace, repo_name, tag_name)
|
||||
|
||||
# Add the leaf layer
|
||||
builder.add_layer(image.storage.checksum, image.v1_json_metadata)
|
||||
builder.add_layer(image.storage.content_checksum, image.v1_json_metadata)
|
||||
|
||||
for parent in parents:
|
||||
builder.add_layer(parent.storage.checksum, parent.v1_json_metadata)
|
||||
builder.add_layer(parent.storage.content_checksum, parent.v1_json_metadata)
|
||||
|
||||
# Sign the manifest with our signing key.
|
||||
manifest = builder.build(docker_v2_signing_key)
|
||||
|
|
|
@ -31,6 +31,10 @@ def _open_stream(formatter, namespace, repository, tag, synthetic_image_id, imag
|
|||
image_list = list(model.image.get_parent_images(namespace, repository, repo_image))
|
||||
image_list.append(repo_image)
|
||||
|
||||
# Note: The image list ordering must be from top-level image, downward, so we reverse the order
|
||||
# here.
|
||||
image_list.reverse()
|
||||
|
||||
def get_next_image():
|
||||
for current_image in image_list:
|
||||
yield current_image
|
||||
|
@ -42,7 +46,8 @@ def _open_stream(formatter, namespace, repository, tag, synthetic_image_id, imag
|
|||
current_image_path)
|
||||
|
||||
current_image_id = current_image_entry.id
|
||||
logger.debug('Returning image layer %s: %s', current_image_id, current_image_path)
|
||||
logger.debug('Returning image layer %s (%s): %s', current_image_id,
|
||||
current_image_entry.docker_image_id, current_image_path)
|
||||
yield current_image_stream
|
||||
|
||||
stream = formatter.build_stream(namespace, repository, tag, synthetic_image_id, image_json,
|
||||
|
|
|
@ -58,6 +58,12 @@ def index(path, **kwargs):
|
|||
def internal_error_display():
|
||||
return render_page_template('500.html')
|
||||
|
||||
@web.errorhandler(404)
|
||||
@web.route('/404', methods=['GET'])
|
||||
def not_found_error_display(e = None):
|
||||
resp = render_page_template('404.html')
|
||||
resp.status_code = 404
|
||||
return resp
|
||||
|
||||
@web.route('/organization/<path:path>', methods=['GET'])
|
||||
@no_cache
|
||||
|
@ -393,7 +399,6 @@ def confirm_recovery():
|
|||
|
||||
@web.route('/repository/<path:repository>/status', methods=['GET'])
|
||||
@parse_repository_name
|
||||
@no_cache
|
||||
@anon_protect
|
||||
def build_status_badge(namespace, repository):
|
||||
token = request.args.get('token', None)
|
||||
|
@ -417,8 +422,13 @@ def build_status_badge(namespace, repository):
|
|||
else:
|
||||
status_name = 'none'
|
||||
|
||||
if request.headers.get('If-None-Match') == status_name:
|
||||
return Response(status=304)
|
||||
|
||||
response = make_response(STATUS_TAGS[status_name])
|
||||
response.content_type = 'image/svg+xml'
|
||||
response.headers['Cache-Control'] = 'no-cache'
|
||||
response.headers['ETag'] = status_name
|
||||
return response
|
||||
|
||||
|
||||
|
|
Reference in a new issue