Merge branch 'master' into bees
This commit is contained in:
Joseph Schorr
commit
ccc16fd6f4
245 changed files with 243608 additions and 1263 deletions
|
|
@ -28,8 +28,8 @@ api_bp = Blueprint('api', __name__)
|
|||
api = Api()
|
||||
api.init_app(api_bp)
|
||||
api.decorators = [csrf_protect,
|
||||
process_oauth,
|
||||
crossdomain(origin='*', headers=['Authorization', 'Content-Type'])]
|
||||
crossdomain(origin='*', headers=['Authorization', 'Content-Type']),
|
||||
process_oauth]
|
||||
|
||||
|
||||
class ApiException(Exception):
|
||||
|
|
@ -91,6 +91,7 @@ def handle_api_error(error):
|
|||
if error.error_type is not None:
|
||||
response.headers['WWW-Authenticate'] = ('Bearer error="%s" error_description="%s"' %
|
||||
(error.error_type, error.error_description))
|
||||
|
||||
return response
|
||||
|
||||
|
||||
|
|
@ -195,6 +196,7 @@ def query_param(name, help_str, type=reqparse.text_type, default=None,
|
|||
'default': default,
|
||||
'choices': choices,
|
||||
'required': required,
|
||||
'location': ('args')
|
||||
})
|
||||
return func
|
||||
return add_param
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ class RepositoryImage(RepositoryParamResource):
|
|||
@nickname('getImage')
|
||||
def get(self, namespace, repository, image_id):
|
||||
""" Get the information available for the specified image. """
|
||||
image = model.get_repo_image(namespace, repository, image_id)
|
||||
image = model.get_repo_image_extended(namespace, repository, image_id)
|
||||
if not image:
|
||||
raise NotFound()
|
||||
|
||||
|
|
@ -99,7 +99,7 @@ class RepositoryImageChanges(RepositoryParamResource):
|
|||
@nickname('getImageChanges')
|
||||
def get(self, namespace, repository, image_id):
|
||||
""" Get the list of changes for the specified image. """
|
||||
image = model.get_repo_image(namespace, repository, image_id)
|
||||
image = model.get_repo_image_extended(namespace, repository, image_id)
|
||||
|
||||
if not image:
|
||||
raise NotFound()
|
||||
|
|
|
|||
|
|
@ -52,6 +52,25 @@ def user_view(user):
|
|||
'super_user': user.username in app.config['SUPER_USERS']
|
||||
}
|
||||
|
||||
@resource('/v1/superuser/usage/')
|
||||
@internal_only
|
||||
@show_if(features.SUPER_USERS)
|
||||
class UsageInformation(ApiResource):
|
||||
""" Resource for returning the usage information for enterprise customers. """
|
||||
@require_fresh_login
|
||||
@nickname('getSystemUsage')
|
||||
def get(self):
|
||||
""" Returns the number of repository handles currently held. """
|
||||
if SuperUserPermission().can():
|
||||
return {
|
||||
'usage': model.get_repository_usage(),
|
||||
'allowed': app.config.get('MAXIMUM_REPOSITORY_USAGE', 20)
|
||||
}
|
||||
|
||||
abort(403)
|
||||
|
||||
|
||||
|
||||
@resource('/v1/superuser/users/')
|
||||
@internal_only
|
||||
@show_if(features.SUPER_USERS)
|
||||
|
|
|
|||
|
|
@ -326,7 +326,7 @@ class BuildTriggerAnalyze(RepositoryParamResource):
|
|||
if not found_repository:
|
||||
return {
|
||||
'status': 'error',
|
||||
'message': 'Repository "%s" was not found' % (base_image)
|
||||
'message': 'Repository "%s" referenced by the Dockerfile was not found' % (base_image)
|
||||
}
|
||||
|
||||
# If the repository is private and the user cannot see that repo, then
|
||||
|
|
@ -335,7 +335,7 @@ class BuildTriggerAnalyze(RepositoryParamResource):
|
|||
if found_repository.visibility.name != 'public' and not can_read:
|
||||
return {
|
||||
'status': 'error',
|
||||
'message': 'Repository "%s" was not found' % (base_image)
|
||||
'message': 'Repository "%s" referenced by the Dockerfile was not found' % (base_image)
|
||||
}
|
||||
|
||||
# Check to see if the repository is public. If not, we suggest the
|
||||
|
|
@ -463,18 +463,18 @@ class BuildTriggerFieldValues(RepositoryParamResource):
|
|||
""" Custom verb to fetch a values list for a particular field name. """
|
||||
@require_repo_admin
|
||||
@nickname('listTriggerFieldValues')
|
||||
def get(self, namespace, repository, trigger_uuid, field_name):
|
||||
def post(self, namespace, repository, trigger_uuid, field_name):
|
||||
""" List the field values for a custom run field. """
|
||||
try:
|
||||
trigger = model.get_build_trigger(namespace, repository, trigger_uuid)
|
||||
except model.InvalidBuildTriggerException:
|
||||
raise NotFound()
|
||||
|
||||
config = request.get_json() or json.loads(trigger.config)
|
||||
user_permission = UserAdminPermission(trigger.connected_user.username)
|
||||
if user_permission.can():
|
||||
trigger_handler = BuildTriggerBase.get_trigger_for_service(trigger.service.name)
|
||||
values = trigger_handler.list_field_values(trigger.auth_token, json.loads(trigger.config),
|
||||
field_name)
|
||||
values = trigger_handler.list_field_values(trigger.auth_token, config, field_name)
|
||||
|
||||
if values is None:
|
||||
raise NotFound()
|
||||
|
|
|
|||
|
|
@ -120,6 +120,10 @@ class User(ApiResource):
|
|||
'type': 'string',
|
||||
'description': 'The user\'s email address',
|
||||
},
|
||||
'invite_code': {
|
||||
'type': 'string',
|
||||
'description': 'The optional invite code'
|
||||
}
|
||||
}
|
||||
},
|
||||
'UpdateUser': {
|
||||
|
|
@ -239,15 +243,12 @@ class User(ApiResource):
|
|||
|
||||
@show_if(features.USER_CREATION)
|
||||
@nickname('createNewUser')
|
||||
@parse_args
|
||||
@query_param('inviteCode', 'Invitation code given for creating the user.', type=str,
|
||||
default='')
|
||||
@internal_only
|
||||
@validate_json_request('NewUser')
|
||||
def post(self, args):
|
||||
def post(self):
|
||||
""" Create a new user. """
|
||||
user_data = request.get_json()
|
||||
invite_code = args['inviteCode']
|
||||
invite_code = user_data.get('invite_code', '')
|
||||
|
||||
existing_user = model.get_user(user_data['username'])
|
||||
if existing_user:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
import logging
|
||||
import requests
|
||||
|
||||
from flask import request, redirect, url_for, Blueprint
|
||||
from flask.ext.login import current_user
|
||||
|
||||
from endpoints.common import render_page_template, common_login, route_show_if
|
||||
from app import app, analytics, get_app_url
|
||||
from app import app, analytics, get_app_url, github_login, google_login, github_trigger
|
||||
from data import model
|
||||
from util.names import parse_repository_name
|
||||
from util.validation import generate_valid_usernames
|
||||
|
|
@ -29,20 +30,16 @@ def render_ologin_error(service_name,
|
|||
service_url=get_app_url(),
|
||||
user_creation=features.USER_CREATION)
|
||||
|
||||
def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_encode=False,
|
||||
redirect_suffix=''):
|
||||
def exchange_code_for_token(code, service, form_encode=False, redirect_suffix=''):
|
||||
code = request.args.get('code')
|
||||
id_config = service_name + '_LOGIN_CLIENT_ID' if for_login else service_name + '_CLIENT_ID'
|
||||
secret_config = service_name + '_LOGIN_CLIENT_SECRET' if for_login else service_name + '_CLIENT_SECRET'
|
||||
|
||||
payload = {
|
||||
'client_id': app.config[id_config],
|
||||
'client_secret': app.config[secret_config],
|
||||
'client_id': service.client_id(),
|
||||
'client_secret': service.client_secret(),
|
||||
'code': code,
|
||||
'grant_type': 'authorization_code',
|
||||
'redirect_uri': '%s://%s/oauth2/%s/callback%s' % (app.config['PREFERRED_URL_SCHEME'],
|
||||
app.config['SERVER_HOSTNAME'],
|
||||
service_name.lower(),
|
||||
service.service_name().lower(),
|
||||
redirect_suffix)
|
||||
}
|
||||
|
||||
|
|
@ -50,12 +47,11 @@ def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_en
|
|||
'Accept': 'application/json'
|
||||
}
|
||||
|
||||
token_url = service.token_endpoint()
|
||||
if form_encode:
|
||||
get_access_token = client.post(app.config[service_name + '_TOKEN_URL'],
|
||||
data=payload, headers=headers)
|
||||
get_access_token = client.post(token_url, data=payload, headers=headers)
|
||||
else:
|
||||
get_access_token = client.post(app.config[service_name + '_TOKEN_URL'],
|
||||
params=payload, headers=headers)
|
||||
get_access_token = client.post(token_url, params=payload, headers=headers)
|
||||
|
||||
json_data = get_access_token.json()
|
||||
if not json_data:
|
||||
|
|
@ -65,25 +61,20 @@ def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_en
|
|||
return token
|
||||
|
||||
|
||||
def get_github_user(token):
|
||||
token_param = {
|
||||
'access_token': token,
|
||||
}
|
||||
get_user = client.get(app.config['GITHUB_USER_URL'], params=token_param)
|
||||
|
||||
return get_user.json()
|
||||
|
||||
|
||||
def get_google_user(token):
|
||||
def get_user(service, token):
|
||||
token_param = {
|
||||
'access_token': token,
|
||||
'alt': 'json',
|
||||
}
|
||||
get_user = client.get(service.user_endpoint(), params=token_param)
|
||||
if get_user.status_code != requests.codes.ok:
|
||||
return {}
|
||||
|
||||
get_user = client.get(app.config['GOOGLE_USER_URL'], params=token_param)
|
||||
return get_user.json()
|
||||
|
||||
def conduct_oauth_login(service_name, user_id, username, email, metadata={}):
|
||||
|
||||
def conduct_oauth_login(service, user_id, username, email, metadata={}):
|
||||
service_name = service.service_name()
|
||||
to_login = model.verify_federated_login(service_name.lower(), user_id)
|
||||
if not to_login:
|
||||
# See if we can create a new user.
|
||||
|
|
@ -93,8 +84,15 @@ def conduct_oauth_login(service_name, user_id, username, email, metadata={}):
|
|||
|
||||
# Try to create the user
|
||||
try:
|
||||
valid = next(generate_valid_usernames(username))
|
||||
to_login = model.create_federated_user(valid, email, service_name.lower(),
|
||||
new_username = None
|
||||
for valid in generate_valid_usernames(username):
|
||||
if model.get_user_or_org(valid):
|
||||
continue
|
||||
|
||||
new_username = valid
|
||||
break
|
||||
|
||||
to_login = model.create_federated_user(new_username, email, service_name.lower(),
|
||||
user_id, set_password_notification=True,
|
||||
metadata=metadata)
|
||||
|
||||
|
|
@ -106,7 +104,15 @@ def conduct_oauth_login(service_name, user_id, username, email, metadata={}):
|
|||
logger.debug('Aliasing with state: %s' % state)
|
||||
analytics.alias(to_login.username, state)
|
||||
|
||||
except model.DataModelException, ex:
|
||||
except model.InvalidEmailAddressException as ieex:
|
||||
message = "The e-mail address %s is already associated " % (email, )
|
||||
message = message + "with an existing %s account." % (app.config['REGISTRY_TITLE_SHORT'], )
|
||||
message = message + "\nPlease log in with your username and password and "
|
||||
message = message + "associate your %s account to use it in the future." % (service_name, )
|
||||
|
||||
return render_ologin_error(service_name, message)
|
||||
|
||||
except model.DataModelException as ex:
|
||||
return render_ologin_error(service_name, ex.message)
|
||||
|
||||
if common_login(to_login):
|
||||
|
|
@ -130,8 +136,8 @@ def google_oauth_callback():
|
|||
if error:
|
||||
return render_ologin_error('Google', error)
|
||||
|
||||
token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE', form_encode=True)
|
||||
user_data = get_google_user(token)
|
||||
token = exchange_code_for_token(request.args.get('code'), google_login, form_encode=True)
|
||||
user_data = get_user(google_login, token)
|
||||
if not user_data or not user_data.get('id', None) or not user_data.get('email', None):
|
||||
return render_ologin_error('Google')
|
||||
|
||||
|
|
@ -140,7 +146,7 @@ def google_oauth_callback():
|
|||
'service_username': user_data['email']
|
||||
}
|
||||
|
||||
return conduct_oauth_login('Google', user_data['id'], username, user_data['email'],
|
||||
return conduct_oauth_login(google_login, user_data['id'], username, user_data['email'],
|
||||
metadata=metadata)
|
||||
|
||||
|
||||
|
|
@ -151,8 +157,8 @@ def github_oauth_callback():
|
|||
if error:
|
||||
return render_ologin_error('GitHub', error)
|
||||
|
||||
token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB')
|
||||
user_data = get_github_user(token)
|
||||
token = exchange_code_for_token(request.args.get('code'), github_login)
|
||||
user_data = get_user(github_login, token)
|
||||
if not user_data or not 'login' in user_data:
|
||||
return render_ologin_error('GitHub')
|
||||
|
||||
|
|
@ -166,7 +172,7 @@ def github_oauth_callback():
|
|||
token_param = {
|
||||
'access_token': token,
|
||||
}
|
||||
get_email = client.get(app.config['GITHUB_USER_EMAILS'], params=token_param,
|
||||
get_email = client.get(github_login.email_endpoint(), params=token_param,
|
||||
headers=v3_media_type)
|
||||
|
||||
# We will accept any email, but we prefer the primary
|
||||
|
|
@ -180,17 +186,17 @@ def github_oauth_callback():
|
|||
'service_username': username
|
||||
}
|
||||
|
||||
return conduct_oauth_login('github', github_id, username, found_email, metadata=metadata)
|
||||
return conduct_oauth_login(github_login, github_id, username, found_email, metadata=metadata)
|
||||
|
||||
|
||||
@callback.route('/google/callback/attach', methods=['GET'])
|
||||
@route_show_if(features.GOOGLE_LOGIN)
|
||||
@require_session_login
|
||||
def google_oauth_attach():
|
||||
token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE',
|
||||
token = exchange_code_for_token(request.args.get('code'), google_login,
|
||||
redirect_suffix='/attach', form_encode=True)
|
||||
|
||||
user_data = get_google_user(token)
|
||||
user_data = get_user(google_login, token)
|
||||
if not user_data or not user_data.get('id', None):
|
||||
return render_ologin_error('Google')
|
||||
|
||||
|
|
@ -216,8 +222,8 @@ def google_oauth_attach():
|
|||
@route_show_if(features.GITHUB_LOGIN)
|
||||
@require_session_login
|
||||
def github_oauth_attach():
|
||||
token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB')
|
||||
user_data = get_github_user(token)
|
||||
token = exchange_code_for_token(request.args.get('code'), github_login)
|
||||
user_data = get_user(github_login, token)
|
||||
if not user_data:
|
||||
return render_ologin_error('GitHub')
|
||||
|
||||
|
|
@ -247,8 +253,7 @@ def github_oauth_attach():
|
|||
def attach_github_build_trigger(namespace, repository):
|
||||
permission = AdministerRepositoryPermission(namespace, repository)
|
||||
if permission.can():
|
||||
token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB',
|
||||
for_login=False)
|
||||
token = exchange_code_for_token(request.args.get('code'), github_trigger)
|
||||
repo = model.get_repository(namespace, repository)
|
||||
if not repo:
|
||||
msg = 'Invalid repository: %s/%s' % (namespace, repository)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,9 @@ from flask.ext.principal import identity_changed
|
|||
from random import SystemRandom
|
||||
|
||||
from data import model
|
||||
from app import app, login_manager, dockerfile_build_queue, notification_queue
|
||||
from data.database import db
|
||||
from app import app, login_manager, dockerfile_build_queue, notification_queue, oauth_apps
|
||||
|
||||
from auth.permissions import QuayDeferredPermissionUser
|
||||
from auth import scopes
|
||||
from endpoints.api.discovery import swagger_route_data
|
||||
|
|
@ -19,6 +21,7 @@ from functools import wraps
|
|||
from config import getFrontendVisibleConfig
|
||||
from external_libraries import get_external_javascript, get_external_css
|
||||
from endpoints.notificationhelper import spawn_notification
|
||||
from util.useremails import CannotSendEmailException
|
||||
|
||||
import features
|
||||
|
||||
|
|
@ -128,6 +131,10 @@ def handle_dme(ex):
|
|||
logger.exception(ex)
|
||||
return make_response(json.dumps({'message': ex.message}), 400)
|
||||
|
||||
@app.errorhandler(CannotSendEmailException)
|
||||
def handle_emailexception(ex):
|
||||
message = 'Could not send email. Please contact an administrator and report this problem.'
|
||||
return make_response(json.dumps({'message': message}), 400)
|
||||
|
||||
def random_string():
|
||||
random = SystemRandom()
|
||||
|
|
@ -170,6 +177,17 @@ def render_page_template(name, **kwargs):
|
|||
external_styles = get_external_css(local=not app.config.get('USE_CDN', True))
|
||||
external_scripts = get_external_javascript(local=not app.config.get('USE_CDN', True))
|
||||
|
||||
def get_oauth_config():
|
||||
oauth_config = {}
|
||||
for oauth_app in oauth_apps:
|
||||
oauth_config[oauth_app.key_name] = oauth_app.get_public_config()
|
||||
|
||||
return oauth_config
|
||||
|
||||
contact_href = None
|
||||
if len(app.config.get('CONTACT_INFO', [])) == 1:
|
||||
contact_href = app.config['CONTACT_INFO'][0]
|
||||
|
||||
resp = make_response(render_template(name, route_data=json.dumps(get_route_data()),
|
||||
external_styles=external_styles,
|
||||
external_scripts=external_scripts,
|
||||
|
|
@ -179,6 +197,7 @@ def render_page_template(name, **kwargs):
|
|||
library_scripts=library_scripts,
|
||||
feature_set=json.dumps(features.get_features()),
|
||||
config_set=json.dumps(getFrontendVisibleConfig(app.config)),
|
||||
oauth_set=json.dumps(get_oauth_config()),
|
||||
mixpanel_key=app.config.get('MIXPANEL_KEY', ''),
|
||||
google_analytics_key=app.config.get('GOOGLE_ANALYTICS_KEY', ''),
|
||||
sentry_public_dsn=app.config.get('SENTRY_PUBLIC_DSN', ''),
|
||||
|
|
@ -186,6 +205,7 @@ def render_page_template(name, **kwargs):
|
|||
show_chat=features.OLARK_CHAT,
|
||||
cache_buster=cache_buster,
|
||||
has_billing=features.BILLING,
|
||||
contact_href=contact_href,
|
||||
**kwargs))
|
||||
|
||||
resp.headers['X-FRAME-OPTIONS'] = 'DENY'
|
||||
|
|
@ -217,16 +237,17 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
|
|||
'build_subdir': subdir
|
||||
}
|
||||
|
||||
build_request = model.create_repository_build(repository, token, job_config,
|
||||
dockerfile_id, build_name,
|
||||
trigger, pull_robot_name=pull_robot_name)
|
||||
with app.config['DB_TRANSACTION_FACTORY'](db):
|
||||
build_request = model.create_repository_build(repository, token, job_config,
|
||||
dockerfile_id, build_name,
|
||||
trigger, pull_robot_name=pull_robot_name)
|
||||
|
||||
dockerfile_build_queue.put([repository.namespace_user.username, repository.name], json.dumps({
|
||||
'build_uuid': build_request.uuid,
|
||||
'namespace': repository.namespace_user.username,
|
||||
'repository': repository.name,
|
||||
'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None
|
||||
}), retries_remaining=1)
|
||||
dockerfile_build_queue.put([repository.namespace_user.username, repository.name], json.dumps({
|
||||
'build_uuid': build_request.uuid,
|
||||
'namespace': repository.namespace_user.username,
|
||||
'repository': repository.name,
|
||||
'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None
|
||||
}), retries_remaining=1)
|
||||
|
||||
# Add the build to the repo's log.
|
||||
metadata = {
|
||||
|
|
@ -260,5 +281,4 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
|
|||
spawn_notification(repository, 'build_queued', event_data,
|
||||
subpage='build?current=%s' % build_request.uuid,
|
||||
pathargs=['build', build_request.uuid])
|
||||
return build_request
|
||||
|
||||
return build_request
|
||||
|
|
@ -8,7 +8,7 @@ from collections import OrderedDict
|
|||
|
||||
from data import model
|
||||
from data.model import oauth
|
||||
from app import analytics, app, authentication, userevents, storage
|
||||
from app import app, authentication, userevents, storage
|
||||
from auth.auth import process_auth
|
||||
from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token
|
||||
from util.names import parse_repository_name
|
||||
|
|
@ -17,6 +17,7 @@ from auth.permissions import (ModifyRepositoryPermission, UserAdminPermission,
|
|||
ReadRepositoryPermission, CreateRepositoryPermission)
|
||||
|
||||
from util.http import abort
|
||||
from endpoints.trackhelper import track_and_log
|
||||
from endpoints.notificationhelper import spawn_notification
|
||||
|
||||
import features
|
||||
|
|
@ -70,7 +71,7 @@ def create_user():
|
|||
abort(400, 'User creation is disabled. Please speak to your administrator.')
|
||||
|
||||
user_data = request.get_json()
|
||||
if not 'username' in user_data:
|
||||
if not user_data or not 'username' in user_data:
|
||||
abort(400, 'Missing username')
|
||||
|
||||
username = user_data['username']
|
||||
|
|
@ -222,13 +223,20 @@ def create_repository(namespace, repository):
|
|||
repo = model.create_repository(namespace, repository,
|
||||
get_authenticated_user())
|
||||
|
||||
profile.debug('Determining added images')
|
||||
added_images = OrderedDict([(desc['id'], desc)
|
||||
for desc in image_descriptions])
|
||||
profile.debug('Determining already added images')
|
||||
added_images = OrderedDict([(desc['id'], desc) for desc in image_descriptions])
|
||||
new_repo_images = dict(added_images)
|
||||
|
||||
for existing in model.get_repository_images(namespace, repository):
|
||||
if existing.docker_image_id in new_repo_images:
|
||||
# Optimization: Lookup any existing images in the repository with matching docker IDs and
|
||||
# remove them from the added dict, so we don't need to look them up one-by-one.
|
||||
def chunks(l, n):
|
||||
for i in xrange(0, len(l), n):
|
||||
yield l[i:i+n]
|
||||
|
||||
# Note: We do this in chunks in an effort to not hit the SQL query size limit.
|
||||
for chunk in chunks(new_repo_images.keys(), 50):
|
||||
existing_images = model.lookup_repository_images(namespace, repository, chunk)
|
||||
for existing in existing_images:
|
||||
added_images.pop(existing.docker_image_id)
|
||||
|
||||
profile.debug('Creating/Linking necessary images')
|
||||
|
|
@ -240,49 +248,8 @@ def create_repository(namespace, repository):
|
|||
|
||||
|
||||
profile.debug('Created images')
|
||||
response = make_response('Created', 201)
|
||||
|
||||
extra_params = {
|
||||
'repository': '%s/%s' % (namespace, repository),
|
||||
}
|
||||
|
||||
metadata = {
|
||||
'repo': repository,
|
||||
'namespace': namespace
|
||||
}
|
||||
|
||||
if get_validated_oauth_token():
|
||||
analytics.track(username, 'push_repo', extra_params)
|
||||
|
||||
oauth_token = get_validated_oauth_token()
|
||||
metadata['oauth_token_id'] = oauth_token.id
|
||||
metadata['oauth_token_application_id'] = oauth_token.application.client_id
|
||||
metadata['oauth_token_application'] = oauth_token.application.name
|
||||
elif get_authenticated_user():
|
||||
username = get_authenticated_user().username
|
||||
|
||||
analytics.track(username, 'push_repo', extra_params)
|
||||
metadata['username'] = username
|
||||
|
||||
# Mark that the user has started pushing the repo.
|
||||
user_data = {
|
||||
'action': 'push_repo',
|
||||
'repository': repository,
|
||||
'namespace': namespace
|
||||
}
|
||||
|
||||
event = userevents.get_event(username)
|
||||
event.publish_event_data('docker-cli', user_data)
|
||||
|
||||
elif get_validated_token():
|
||||
analytics.track(get_validated_token().code, 'push_repo', extra_params)
|
||||
metadata['token'] = get_validated_token().friendly_name
|
||||
metadata['token_code'] = get_validated_token().code
|
||||
|
||||
model.log_action('push_repo', namespace, performer=get_authenticated_user(),
|
||||
ip=request.remote_addr, metadata=metadata, repository=repo)
|
||||
|
||||
return response
|
||||
track_and_log('push_repo', repo)
|
||||
return make_response('Created', 201)
|
||||
|
||||
|
||||
@index.route('/repositories/<path:repository>/images', methods=['PUT'])
|
||||
|
|
@ -299,13 +266,6 @@ def update_images(namespace, repository):
|
|||
# Make sure the repo actually exists.
|
||||
abort(404, message='Unknown repository', issue='unknown-repo')
|
||||
|
||||
profile.debug('Parsing image data')
|
||||
image_with_checksums = json.loads(request.data.decode('utf8'))
|
||||
|
||||
updated_tags = {}
|
||||
for image in image_with_checksums:
|
||||
updated_tags[image['Tag']] = image['id']
|
||||
|
||||
if get_authenticated_user():
|
||||
profile.debug('Publishing push event')
|
||||
username = get_authenticated_user().username
|
||||
|
|
@ -326,12 +286,11 @@ def update_images(namespace, repository):
|
|||
# Generate a job for each notification that has been added to this repo
|
||||
profile.debug('Adding notifications for repository')
|
||||
|
||||
updated_tags = session.get('pushed_tags', {})
|
||||
event_data = {
|
||||
'updated_tags': updated_tags,
|
||||
'pushed_image_count': len(image_with_checksums),
|
||||
'pruned_image_count': num_removed
|
||||
}
|
||||
|
||||
spawn_notification(repo, 'repo_push', event_data)
|
||||
return make_response('Updated', 204)
|
||||
|
||||
|
|
@ -368,38 +327,7 @@ def get_repository_images(namespace, repository):
|
|||
resp = make_response(json.dumps(all_images), 200)
|
||||
resp.mimetype = 'application/json'
|
||||
|
||||
metadata = {
|
||||
'repo': repository,
|
||||
'namespace': namespace,
|
||||
}
|
||||
|
||||
profile.debug('Logging the pull to Mixpanel and the log system')
|
||||
if get_validated_oauth_token():
|
||||
oauth_token = get_validated_oauth_token()
|
||||
metadata['oauth_token_id'] = oauth_token.id
|
||||
metadata['oauth_token_application_id'] = oauth_token.application.client_id
|
||||
metadata['oauth_token_application'] = oauth_token.application.name
|
||||
elif get_authenticated_user():
|
||||
metadata['username'] = get_authenticated_user().username
|
||||
elif get_validated_token():
|
||||
metadata['token'] = get_validated_token().friendly_name
|
||||
metadata['token_code'] = get_validated_token().code
|
||||
else:
|
||||
metadata['public'] = True
|
||||
|
||||
pull_username = 'anonymous'
|
||||
if get_authenticated_user():
|
||||
pull_username = get_authenticated_user().username
|
||||
|
||||
extra_params = {
|
||||
'repository': '%s/%s' % (namespace, repository),
|
||||
}
|
||||
|
||||
analytics.track(pull_username, 'pull_repo', extra_params)
|
||||
model.log_action('pull_repo', namespace,
|
||||
performer=get_authenticated_user(),
|
||||
ip=request.remote_addr, metadata=metadata,
|
||||
repository=repo)
|
||||
track_and_log('pull_repo', repo)
|
||||
return resp
|
||||
|
||||
abort(403)
|
||||
|
|
@ -458,6 +386,7 @@ def get_search():
|
|||
@index.route('/_ping')
|
||||
@index.route('/_ping')
|
||||
def ping():
|
||||
# NOTE: any changes made here must also be reflected in the nginx config
|
||||
response = make_response('true', 200)
|
||||
response.headers['X-Docker-Registry-Version'] = '0.6.0'
|
||||
response.headers['X-Docker-Registry-Standalone'] = '0'
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
import logging
|
||||
|
||||
from notificationhelper import build_event_data
|
||||
from util.jinjautil import get_template_env
|
||||
|
||||
template_env = get_template_env("events")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class InvalidNotificationEventException(Exception):
|
||||
|
|
@ -14,7 +16,7 @@ class NotificationEvent(object):
|
|||
def get_level(self, event_data, notification_data):
|
||||
"""
|
||||
Returns a 'level' representing the severity of the event.
|
||||
Valid values are: 'info', 'warning', 'error', 'primary'
|
||||
Valid values are: 'info', 'warning', 'error', 'primary', 'success'
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
|
@ -28,7 +30,10 @@ class NotificationEvent(object):
|
|||
"""
|
||||
Returns a human readable HTML message for the given notification data.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
return template_env.get_template(self.event_name() + '.html').render({
|
||||
'event_data': event_data,
|
||||
'notification_data': notification_data
|
||||
})
|
||||
|
||||
def get_sample_data(self, repository=None):
|
||||
"""
|
||||
|
|
@ -59,32 +64,14 @@ class RepoPushEvent(NotificationEvent):
|
|||
return 'repo_push'
|
||||
|
||||
def get_level(self, event_data, notification_data):
|
||||
return 'info'
|
||||
return 'primary'
|
||||
|
||||
def get_summary(self, event_data, notification_data):
|
||||
return 'Repository %s updated' % (event_data['repository'])
|
||||
|
||||
def get_message(self, event_data, notification_data):
|
||||
if not event_data.get('updated_tags', {}).keys():
|
||||
html = """
|
||||
Repository <a href="%s">%s</a> has been updated via a push.
|
||||
""" % (event_data['homepage'],
|
||||
event_data['repository'])
|
||||
else:
|
||||
html = """
|
||||
Repository <a href="%s">%s</a> has been updated via a push.
|
||||
<br><br>
|
||||
Tags Updated: %s
|
||||
""" % (event_data['homepage'],
|
||||
event_data['repository'],
|
||||
', '.join(event_data['updated_tags'].keys()))
|
||||
|
||||
return html
|
||||
|
||||
def get_sample_data(self, repository):
|
||||
return build_event_data(repository, {
|
||||
'updated_tags': {'latest': 'someimageid', 'foo': 'anotherimage'},
|
||||
'pushed_image_count': 10,
|
||||
'pruned_image_count': 3
|
||||
})
|
||||
|
||||
|
|
@ -109,26 +96,7 @@ class BuildQueueEvent(NotificationEvent):
|
|||
}, subpage='/build?current=%s' % build_uuid)
|
||||
|
||||
def get_summary(self, event_data, notification_data):
|
||||
return 'Build queued for repository %s' % (event_data['repository'])
|
||||
|
||||
def get_message(self, event_data, notification_data):
|
||||
is_manual = event_data['is_manual']
|
||||
if is_manual:
|
||||
html = """
|
||||
A <a href="%s">new build</a> has been manually queued to start on repository %s.
|
||||
<br><br>
|
||||
Build ID: %s
|
||||
""" % (event_data['homepage'], event_data['repository'], event_data['build_id'])
|
||||
else:
|
||||
html = """
|
||||
A <a href="%s">new build</a> has been queued via a %s trigger to start on repository %s.
|
||||
<br><br>
|
||||
Build ID: %s
|
||||
""" % (event_data['homepage'], event_data['trigger_kind'],
|
||||
event_data['repository'], event_data['build_id'])
|
||||
|
||||
return html
|
||||
|
||||
return 'Build queued for repository %s' % (event_data['repository'])
|
||||
|
||||
|
||||
class BuildStartEvent(NotificationEvent):
|
||||
|
|
@ -152,15 +120,6 @@ class BuildStartEvent(NotificationEvent):
|
|||
def get_summary(self, event_data, notification_data):
|
||||
return 'Build started for repository %s' % (event_data['repository'])
|
||||
|
||||
def get_message(self, event_data, notification_data):
|
||||
html = """
|
||||
A <a href="%s">new build</a> has started on repository %s.
|
||||
<br><br>
|
||||
Build ID: %s
|
||||
""" % (event_data['homepage'], event_data['repository'], event_data['build_id'])
|
||||
|
||||
return html
|
||||
|
||||
|
||||
class BuildSuccessEvent(NotificationEvent):
|
||||
@classmethod
|
||||
|
|
@ -168,7 +127,7 @@ class BuildSuccessEvent(NotificationEvent):
|
|||
return 'build_success'
|
||||
|
||||
def get_level(self, event_data, notification_data):
|
||||
return 'primary'
|
||||
return 'success'
|
||||
|
||||
def get_sample_data(self, repository):
|
||||
build_uuid = 'fake-build-id'
|
||||
|
|
@ -183,15 +142,6 @@ class BuildSuccessEvent(NotificationEvent):
|
|||
def get_summary(self, event_data, notification_data):
|
||||
return 'Build succeeded for repository %s' % (event_data['repository'])
|
||||
|
||||
def get_message(self, event_data, notification_data):
|
||||
html = """
|
||||
A <a href="%s">build</a> has finished on repository %s.
|
||||
<br><br>
|
||||
Build ID: %s
|
||||
""" % (event_data['homepage'], event_data['repository'], event_data['build_id'])
|
||||
|
||||
return html
|
||||
|
||||
|
||||
class BuildFailureEvent(NotificationEvent):
|
||||
@classmethod
|
||||
|
|
@ -215,13 +165,3 @@ class BuildFailureEvent(NotificationEvent):
|
|||
def get_summary(self, event_data, notification_data):
|
||||
return 'Build failure for repository %s' % (event_data['repository'])
|
||||
|
||||
def get_message(self, event_data, notification_data):
|
||||
html = """
|
||||
A <a href="%s">build</a> has failed on repository %s.
|
||||
<br><br>
|
||||
Reason: %s<br>
|
||||
Build ID: %s<br>
|
||||
""" % (event_data['homepage'], event_data['repository'],
|
||||
event_data['error_message'], event_data['build_id'])
|
||||
|
||||
return html
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from app import app, notification_queue
|
||||
from data import model
|
||||
from auth.auth_context import get_authenticated_user, get_validated_oauth_token
|
||||
|
||||
import json
|
||||
|
||||
|
|
@ -27,21 +28,37 @@ def build_event_data(repo, extra_data={}, subpage=None):
|
|||
event_data.update(extra_data)
|
||||
return event_data
|
||||
|
||||
def build_notification_data(notification, event_data):
|
||||
def build_notification_data(notification, event_data, performer_data=None):
|
||||
if not performer_data:
|
||||
performer_data = {}
|
||||
|
||||
oauth_token = get_validated_oauth_token()
|
||||
if oauth_token:
|
||||
performer_data['oauth_token_id'] = oauth_token.id
|
||||
performer_data['oauth_token_application_id'] = oauth_token.application.client_id
|
||||
performer_data['oauth_token_application'] = oauth_token.application.name
|
||||
|
||||
performer_user = get_authenticated_user()
|
||||
if performer_user:
|
||||
performer_data['entity_id'] = performer_user.id
|
||||
performer_data['entity_name'] = performer_user.username
|
||||
|
||||
return {
|
||||
'notification_uuid': notification.uuid,
|
||||
'repository_namespace': notification.repository.namespace_user.username,
|
||||
'repository_name': notification.repository.name,
|
||||
'event_data': event_data
|
||||
'event_data': event_data,
|
||||
'performer_data': performer_data
|
||||
}
|
||||
|
||||
|
||||
def spawn_notification(repo, event_name, extra_data={}, subpage=None, pathargs=[]):
|
||||
def spawn_notification(repo, event_name, extra_data={}, subpage=None, pathargs=[],
|
||||
performer_data=None):
|
||||
event_data = build_event_data(repo, extra_data=extra_data, subpage=subpage)
|
||||
|
||||
notifications = model.list_repo_notifications(repo.namespace_user.username, repo.name,
|
||||
event_name=event_name)
|
||||
for notification in notifications:
|
||||
notification_data = build_notification_data(notification, event_data)
|
||||
for notification in list(notifications):
|
||||
notification_data = build_notification_data(notification, event_data, performer_data)
|
||||
path = [repo.namespace_user.username, repo.name, event_name] + pathargs
|
||||
notification_queue.put(path, json.dumps(notification_data))
|
||||
|
|
|
|||
|
|
@ -211,7 +211,7 @@ class FlowdockMethod(NotificationMethod):
|
|||
if not token:
|
||||
return
|
||||
|
||||
owner = model.get_user(notification.repository.namespace_user.username)
|
||||
owner = model.get_user_or_org(notification.repository.namespace_user.username)
|
||||
if not owner:
|
||||
# Something went wrong.
|
||||
return
|
||||
|
|
@ -267,7 +267,7 @@ class HipchatMethod(NotificationMethod):
|
|||
if not token or not room_id:
|
||||
return
|
||||
|
||||
owner = model.get_user(notification.repository.namespace_user.username)
|
||||
owner = model.get_user_or_org(notification.repository.namespace_user.username)
|
||||
if not owner:
|
||||
# Something went wrong.
|
||||
return
|
||||
|
|
@ -279,6 +279,7 @@ class HipchatMethod(NotificationMethod):
|
|||
'info': 'gray',
|
||||
'warning': 'yellow',
|
||||
'error': 'red',
|
||||
'success': 'green',
|
||||
'primary': 'purple'
|
||||
}.get(level, 'gray')
|
||||
|
||||
|
|
@ -303,6 +304,56 @@ class HipchatMethod(NotificationMethod):
|
|||
raise NotificationMethodPerformException(ex.message)
|
||||
|
||||
|
||||
from HTMLParser import HTMLParser
|
||||
|
||||
class SlackAdjuster(HTMLParser):
|
||||
def __init__(self):
|
||||
self.reset()
|
||||
self.result = []
|
||||
|
||||
def handle_data(self, d):
|
||||
self.result.append(d)
|
||||
|
||||
def get_attr(self, attrs, name):
|
||||
for attr in attrs:
|
||||
if attr[0] == name:
|
||||
return attr[1]
|
||||
|
||||
return ''
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == 'a':
|
||||
self.result.append('<%s|' % (self.get_attr(attrs, 'href'), ))
|
||||
|
||||
if tag == 'i':
|
||||
self.result.append('_')
|
||||
|
||||
if tag == 'b' or tag == 'strong':
|
||||
self.result.append('*')
|
||||
|
||||
if tag == 'img':
|
||||
self.result.append(self.get_attr(attrs, 'alt'))
|
||||
self.result.append(' ')
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag == 'a':
|
||||
self.result.append('>')
|
||||
|
||||
if tag == 'b' or tag == 'strong':
|
||||
self.result.append('*')
|
||||
|
||||
if tag == 'i':
|
||||
self.result.append('_')
|
||||
|
||||
def get_data(self):
|
||||
return ''.join(self.result)
|
||||
|
||||
def adjust_tags(html):
|
||||
s = SlackAdjuster()
|
||||
s.feed(html)
|
||||
return s.get_data()
|
||||
|
||||
|
||||
class SlackMethod(NotificationMethod):
|
||||
""" Method for sending notifications to Slack via the API:
|
||||
https://api.slack.com/docs/attachments
|
||||
|
|
@ -318,12 +369,11 @@ class SlackMethod(NotificationMethod):
|
|||
if not config_data.get('subdomain', '').isalnum():
|
||||
raise CannotValidateNotificationMethodException('Missing Slack Subdomain Name')
|
||||
|
||||
def formatForSlack(self, message):
|
||||
def format_for_slack(self, message):
|
||||
message = message.replace('\n', '')
|
||||
message = re.sub(r'\s+', ' ', message)
|
||||
message = message.replace('<br>', '\n')
|
||||
message = re.sub(r'<a href="(.+)">(.+)</a>', '<\\1|\\2>', message)
|
||||
return message
|
||||
return adjust_tags(message)
|
||||
|
||||
def perform(self, notification, event_handler, notification_data):
|
||||
config_data = json.loads(notification.config_json)
|
||||
|
|
@ -334,7 +384,7 @@ class SlackMethod(NotificationMethod):
|
|||
if not token or not subdomain:
|
||||
return
|
||||
|
||||
owner = model.get_user(notification.repository.namespace_user.username)
|
||||
owner = model.get_user_or_org(notification.repository.namespace_user.username)
|
||||
if not owner:
|
||||
# Something went wrong.
|
||||
return
|
||||
|
|
@ -346,6 +396,7 @@ class SlackMethod(NotificationMethod):
|
|||
'info': '#ffffff',
|
||||
'warning': 'warning',
|
||||
'error': 'danger',
|
||||
'success': 'good',
|
||||
'primary': 'good'
|
||||
}.get(level, '#ffffff')
|
||||
|
||||
|
|
@ -359,8 +410,9 @@ class SlackMethod(NotificationMethod):
|
|||
'attachments': [
|
||||
{
|
||||
'fallback': summary,
|
||||
'text': self.formatForSlack(message),
|
||||
'color': color
|
||||
'text': self.format_for_slack(message),
|
||||
'color': color,
|
||||
'mrkdwn_in': ["text"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,13 +7,13 @@ from functools import wraps
|
|||
from datetime import datetime
|
||||
from time import time
|
||||
|
||||
from app import storage as store, image_diff_queue
|
||||
from app import storage as store, image_diff_queue, app
|
||||
from auth.auth import process_auth, extract_namespace_repo_from_session
|
||||
from util import checksums, changes
|
||||
from util.http import abort, exact_abort
|
||||
from auth.permissions import (ReadRepositoryPermission,
|
||||
ModifyRepositoryPermission)
|
||||
from data import model
|
||||
from data import model, database
|
||||
from util import gzipstream
|
||||
|
||||
|
||||
|
|
@ -59,7 +59,7 @@ def require_completion(f):
|
|||
@wraps(f)
|
||||
def wrapper(namespace, repository, *args, **kwargs):
|
||||
image_id = kwargs['image_id']
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
|
||||
if image_is_uploading(repo_image):
|
||||
abort(400, 'Image %(image_id)s is being uploaded, retry later',
|
||||
issue='upload-in-progress', image_id=kwargs['image_id'])
|
||||
|
|
@ -103,7 +103,7 @@ def head_image_layer(namespace, repository, image_id, headers):
|
|||
profile.debug('Checking repo permissions')
|
||||
if permission.can() or model.repository_is_public(namespace, repository):
|
||||
profile.debug('Looking up repo image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
|
||||
if not repo_image:
|
||||
profile.debug('Image not found')
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
|
|
@ -136,7 +136,7 @@ def get_image_layer(namespace, repository, image_id, headers):
|
|||
profile.debug('Checking repo permissions')
|
||||
if permission.can() or model.repository_is_public(namespace, repository):
|
||||
profile.debug('Looking up repo image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
|
||||
|
||||
profile.debug('Looking up the layer path')
|
||||
try:
|
||||
|
|
@ -151,6 +151,10 @@ def get_image_layer(namespace, repository, image_id, headers):
|
|||
return resp
|
||||
|
||||
profile.debug('Streaming layer data')
|
||||
|
||||
# Close the database handle here for this process before we send the long download.
|
||||
database.close_db_filter(None)
|
||||
|
||||
return Response(store.stream_read(repo_image.storage.locations, path), headers=headers)
|
||||
except (IOError, AttributeError):
|
||||
profile.debug('Image not found')
|
||||
|
|
@ -170,7 +174,7 @@ def put_image_layer(namespace, repository, image_id):
|
|||
abort(403)
|
||||
|
||||
profile.debug('Retrieving image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
|
||||
try:
|
||||
profile.debug('Retrieving image data')
|
||||
uuid = repo_image.storage.uuid
|
||||
|
|
@ -197,12 +201,15 @@ def put_image_layer(namespace, repository, image_id):
|
|||
# Create a socket reader to read the input stream containing the layer data.
|
||||
sr = SocketReader(input_stream)
|
||||
|
||||
# Add a handler that store the data in storage.
|
||||
tmp, store_hndlr = store.temp_store_handler()
|
||||
sr.add_handler(store_hndlr)
|
||||
# Add a handler that copies the data into a temp file. This is used to calculate the tarsum,
|
||||
# which is only needed for older versions of Docker.
|
||||
requires_tarsum = session.get('checksum_format') == 'tarsum'
|
||||
if requires_tarsum:
|
||||
tmp, tmp_hndlr = store.temp_store_handler()
|
||||
sr.add_handler(tmp_hndlr)
|
||||
|
||||
# Add a handler to compute the uncompressed size of the layer.
|
||||
uncompressed_size_info, size_hndlr = gzipstream.calculate_size_handler()
|
||||
# Add a handler to compute the compressed and uncompressed sizes of the layer.
|
||||
size_info, size_hndlr = gzipstream.calculate_size_handler()
|
||||
sr.add_handler(size_hndlr)
|
||||
|
||||
# Add a handler which computes the checksum.
|
||||
|
|
@ -210,21 +217,23 @@ def put_image_layer(namespace, repository, image_id):
|
|||
sr.add_handler(sum_hndlr)
|
||||
|
||||
# Stream write the data to storage.
|
||||
store.stream_write(repo_image.storage.locations, layer_path, sr)
|
||||
with database.CloseForLongOperation(app.config):
|
||||
store.stream_write(repo_image.storage.locations, layer_path, sr)
|
||||
|
||||
# Append the computed checksum.
|
||||
csums = []
|
||||
csums.append('sha256:{0}'.format(h.hexdigest()))
|
||||
|
||||
try:
|
||||
image_size = tmp.tell()
|
||||
|
||||
# Save the size of the image.
|
||||
model.set_image_size(image_id, namespace, repository, image_size, uncompressed_size_info.size)
|
||||
model.set_image_size(image_id, namespace, repository, size_info.compressed_size,
|
||||
size_info.uncompressed_size)
|
||||
|
||||
if requires_tarsum:
|
||||
tmp.seek(0)
|
||||
csums.append(checksums.compute_tarsum(tmp, json_data))
|
||||
tmp.close()
|
||||
|
||||
tmp.seek(0)
|
||||
csums.append(checksums.compute_tarsum(tmp, json_data))
|
||||
tmp.close()
|
||||
except (IOError, checksums.TarError) as e:
|
||||
logger.debug('put_image_layer: Error when computing tarsum '
|
||||
'{0}'.format(e))
|
||||
|
|
@ -267,7 +276,19 @@ def put_image_checksum(namespace, repository, image_id):
|
|||
if not permission.can():
|
||||
abort(403)
|
||||
|
||||
checksum = request.headers.get('X-Docker-Checksum')
|
||||
# Docker Version < 0.10 (tarsum+sha):
|
||||
old_checksum = request.headers.get('X-Docker-Checksum')
|
||||
|
||||
# Docker Version >= 0.10 (sha):
|
||||
new_checksum = request.headers.get('X-Docker-Checksum-Payload')
|
||||
|
||||
# Store whether we need to calculate the tarsum.
|
||||
if new_checksum:
|
||||
session['checksum_format'] = 'sha256'
|
||||
else:
|
||||
session['checksum_format'] = 'tarsum'
|
||||
|
||||
checksum = new_checksum or old_checksum
|
||||
if not checksum:
|
||||
abort(400, "Missing checksum for image %(image_id)s", issue='missing-checksum',
|
||||
image_id=image_id)
|
||||
|
|
@ -277,7 +298,10 @@ def put_image_checksum(namespace, repository, image_id):
|
|||
issue='missing-checksum-cookie', image_id=image_id)
|
||||
|
||||
profile.debug('Looking up repo image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
|
||||
if not repo_image or not repo_image.storage:
|
||||
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
|
||||
|
||||
uuid = repo_image.storage.uuid
|
||||
|
||||
profile.debug('Looking up repo layer data')
|
||||
|
|
@ -329,7 +353,7 @@ def get_image_json(namespace, repository, image_id, headers):
|
|||
abort(403)
|
||||
|
||||
profile.debug('Looking up repo image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
|
||||
|
||||
profile.debug('Looking up repo layer data')
|
||||
try:
|
||||
|
|
@ -360,7 +384,7 @@ def get_image_ancestry(namespace, repository, image_id, headers):
|
|||
abort(403)
|
||||
|
||||
profile.debug('Looking up repo image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
|
||||
|
||||
profile.debug('Looking up image data')
|
||||
try:
|
||||
|
|
@ -424,7 +448,7 @@ def put_image_json(namespace, repository, image_id):
|
|||
issue='invalid-request', image_id=image_id)
|
||||
|
||||
profile.debug('Looking up repo image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
|
||||
if not repo_image:
|
||||
profile.debug('Image not found')
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
|
|
@ -441,7 +465,7 @@ def put_image_json(namespace, repository, image_id):
|
|||
parent_image = None
|
||||
if parent_id:
|
||||
profile.debug('Looking up parent image')
|
||||
parent_image = model.get_repo_image(namespace, repository, parent_id)
|
||||
parent_image = model.get_repo_image_extended(namespace, repository, parent_id)
|
||||
|
||||
parent_uuid = parent_image and parent_image.storage.uuid
|
||||
parent_locations = parent_image and parent_image.storage.locations
|
||||
|
|
@ -494,7 +518,7 @@ def put_image_json(namespace, repository, image_id):
|
|||
def process_image_changes(namespace, repository, image_id):
|
||||
logger.debug('Generating diffs for image: %s' % image_id)
|
||||
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
|
||||
if not repo_image:
|
||||
logger.warning('No image for id: %s', image_id)
|
||||
return None, None
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
import logging
|
||||
import json
|
||||
|
||||
from flask import abort, request, jsonify, make_response, Blueprint
|
||||
from flask import abort, request, jsonify, make_response, Blueprint, session
|
||||
|
||||
from app import app
|
||||
from util.names import parse_repository_name
|
||||
|
|
@ -59,6 +59,12 @@ def put_tag(namespace, repository, tag):
|
|||
docker_image_id = json.loads(request.data)
|
||||
model.create_or_update_tag(namespace, repository, tag, docker_image_id)
|
||||
|
||||
# Store the updated tag.
|
||||
if not 'pushed_tags' in session:
|
||||
session['pushed_tags'] = {}
|
||||
|
||||
session['pushed_tags'][tag] = docker_image_id
|
||||
|
||||
return make_response('Created', 200)
|
||||
|
||||
abort(403)
|
||||
|
|
|
|||
62
endpoints/trackhelper.py
Normal file
62
endpoints/trackhelper.py
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
import logging
|
||||
|
||||
from app import analytics, app, userevents
|
||||
from data import model
|
||||
from flask import request
|
||||
from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
profile = logging.getLogger('application.profiler')
|
||||
|
||||
def track_and_log(event_name, repo, **kwargs):
|
||||
repository = repo.name
|
||||
namespace = repo.namespace_user.username
|
||||
metadata = {
|
||||
'repo': repository,
|
||||
'namespace': namespace,
|
||||
}
|
||||
metadata.update(kwargs)
|
||||
|
||||
analytics_id = 'anonymous'
|
||||
|
||||
profile.debug('Logging the %s to Mixpanel and the log system', event_name)
|
||||
if get_validated_oauth_token():
|
||||
oauth_token = get_validated_oauth_token()
|
||||
metadata['oauth_token_id'] = oauth_token.id
|
||||
metadata['oauth_token_application_id'] = oauth_token.application.client_id
|
||||
metadata['oauth_token_application'] = oauth_token.application.name
|
||||
analytics_id = 'oauth:' + oauth_token.id
|
||||
elif get_authenticated_user():
|
||||
metadata['username'] = get_authenticated_user().username
|
||||
analytics_id = get_authenticated_user().username
|
||||
elif get_validated_token():
|
||||
metadata['token'] = get_validated_token().friendly_name
|
||||
metadata['token_code'] = get_validated_token().code
|
||||
analytics_id = 'token:' + get_validated_token().code
|
||||
else:
|
||||
metadata['public'] = True
|
||||
analytics_id = 'anonymous'
|
||||
|
||||
extra_params = {
|
||||
'repository': '%s/%s' % (namespace, repository),
|
||||
}
|
||||
|
||||
# Publish the user event (if applicable)
|
||||
if get_authenticated_user():
|
||||
user_event_data = {
|
||||
'action': event_name,
|
||||
'repository': repository,
|
||||
'namespace': namespace
|
||||
}
|
||||
|
||||
event = userevents.get_event(get_authenticated_user().username)
|
||||
event.publish_event_data('docker-cli', user_event_data)
|
||||
|
||||
# Save the action to mixpanel.
|
||||
analytics.track(analytics_id, event_name, extra_params)
|
||||
|
||||
# Log the action to the database.
|
||||
model.log_action(event_name, namespace,
|
||||
performer=get_authenticated_user(),
|
||||
ip=request.remote_addr, metadata=metadata,
|
||||
repository=repo)
|
||||
|
|
@ -3,11 +3,13 @@ import io
|
|||
import os.path
|
||||
import tarfile
|
||||
import base64
|
||||
import re
|
||||
|
||||
from github import Github, UnknownObjectException, GithubException
|
||||
from tempfile import SpooledTemporaryFile
|
||||
|
||||
from app import app, userfiles as user_files
|
||||
from app import app, userfiles as user_files, github_trigger
|
||||
from util.tarfileappender import TarfileAppender
|
||||
|
||||
|
||||
client = app.config['HTTPCLIENT']
|
||||
|
|
@ -148,8 +150,8 @@ def raise_unsupported():
|
|||
class GithubBuildTrigger(BuildTrigger):
|
||||
@staticmethod
|
||||
def _get_client(auth_token):
|
||||
return Github(auth_token, client_id=app.config['GITHUB_CLIENT_ID'],
|
||||
client_secret=app.config['GITHUB_CLIENT_SECRET'])
|
||||
return Github(auth_token, client_id=github_trigger.client_id(),
|
||||
client_secret=github_trigger.client_secret())
|
||||
|
||||
@classmethod
|
||||
def service_name(cls):
|
||||
|
|
@ -229,13 +231,36 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
|
||||
return repos_by_org
|
||||
|
||||
def matches_ref(self, ref, regex):
|
||||
match_string = ref.split('/', 1)[1]
|
||||
if not regex:
|
||||
return False
|
||||
|
||||
m = regex.match(match_string)
|
||||
if not m:
|
||||
return False
|
||||
|
||||
return len(m.group(0)) == len(match_string)
|
||||
|
||||
def list_build_subdirs(self, auth_token, config):
|
||||
gh_client = self._get_client(auth_token)
|
||||
source = config['build_source']
|
||||
|
||||
try:
|
||||
try:
|
||||
repo = gh_client.get_repo(source)
|
||||
default_commit = repo.get_branch(repo.default_branch or 'master').commit
|
||||
|
||||
# Find the first matching branch.
|
||||
branches = None
|
||||
if 'branchtag_regex' in config:
|
||||
try:
|
||||
regex = re.compile(config['branchtag_regex'])
|
||||
branches = [branch.name for branch in repo.get_branches()
|
||||
if self.matches_ref('refs/heads/' + branch.name, regex)]
|
||||
except:
|
||||
pass
|
||||
|
||||
branches = branches or [repo.default_branch or 'master']
|
||||
default_commit = repo.get_branch(branches[0]).commit
|
||||
commit_tree = repo.get_git_tree(default_commit.sha, recursive=True)
|
||||
|
||||
return [os.path.dirname(elem.path) for elem in commit_tree.tree
|
||||
|
|
@ -301,10 +326,17 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
with tarfile.open(fileobj=tarball) as archive:
|
||||
tarball_subdir = archive.getnames()[0]
|
||||
|
||||
# Seek to position 0 to make boto multipart happy
|
||||
# Seek to position 0 to make tarfile happy.
|
||||
tarball.seek(0)
|
||||
|
||||
dockerfile_id = user_files.store_file(tarball, TARBALL_MIME)
|
||||
entries = {
|
||||
tarball_subdir + '/.git/HEAD': commit_sha,
|
||||
tarball_subdir + '/.git/objects/': None,
|
||||
tarball_subdir + '/.git/refs/': None
|
||||
}
|
||||
|
||||
appender = TarfileAppender(tarball, entries).get_stream()
|
||||
dockerfile_id = user_files.store_file(appender, TARBALL_MIME)
|
||||
|
||||
logger.debug('Successfully prepared job')
|
||||
|
||||
|
|
@ -330,7 +362,7 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
payload = request.get_json()
|
||||
if not payload or payload.get('head_commit') is None:
|
||||
raise SkipRequestException()
|
||||
|
||||
|
||||
if 'zen' in payload:
|
||||
raise ValidationRequestException()
|
||||
|
||||
|
|
@ -339,6 +371,15 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
commit_sha = payload['head_commit']['id']
|
||||
commit_message = payload['head_commit'].get('message', '')
|
||||
|
||||
if 'branchtag_regex' in config:
|
||||
try:
|
||||
regex = re.compile(config['branchtag_regex'])
|
||||
except:
|
||||
regex = re.compile('.*')
|
||||
|
||||
if not self.matches_ref(ref, regex):
|
||||
raise SkipRequestException()
|
||||
|
||||
if should_skip_commit(commit_message):
|
||||
raise SkipRequestException()
|
||||
|
||||
|
|
@ -362,22 +403,36 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
|
||||
gh_client = self._get_client(auth_token)
|
||||
repo = gh_client.get_repo(source)
|
||||
master = repo.get_branch(repo.default_branch)
|
||||
master_sha = master.commit.sha
|
||||
short_sha = GithubBuildTrigger.get_display_name(master_sha)
|
||||
ref = 'refs/heads/%s' % (run_parameters.get('branch_name') or repo.default_branch)
|
||||
branch_name = run_parameters.get('branch_name') or repo.default_branch
|
||||
branch = repo.get_branch(branch_name)
|
||||
branch_sha = branch.commit.sha
|
||||
short_sha = GithubBuildTrigger.get_display_name(branch_sha)
|
||||
ref = 'refs/heads/%s' % (branch_name)
|
||||
|
||||
return self._prepare_build(config, repo, master_sha, short_sha, ref)
|
||||
return self._prepare_build(config, repo, branch_sha, short_sha, ref)
|
||||
except GithubException as ghe:
|
||||
raise TriggerStartException(ghe.data['message'])
|
||||
|
||||
|
||||
def list_field_values(self, auth_token, config, field_name):
|
||||
if field_name == 'refs':
|
||||
branches = self.list_field_values(auth_token, config, 'branch_name')
|
||||
tags = self.list_field_values(auth_token, config, 'tag_name')
|
||||
|
||||
return ([{'kind': 'branch', 'name': b} for b in branches] +
|
||||
[{'kind': 'tag', 'name': tag} for tag in tags])
|
||||
|
||||
if field_name == 'tag_name':
|
||||
gh_client = self._get_client(auth_token)
|
||||
source = config['build_source']
|
||||
repo = gh_client.get_repo(source)
|
||||
return [tag.name for tag in repo.get_tags()]
|
||||
|
||||
if field_name == 'branch_name':
|
||||
gh_client = self._get_client(auth_token)
|
||||
source = config['build_source']
|
||||
repo = gh_client.get_repo(source)
|
||||
branches = [branch['name'] for branch in repo.get_branches()]
|
||||
branches = [branch.name for branch in repo.get_branches()]
|
||||
|
||||
if not repo.default_branch in branches:
|
||||
branches.insert(0, repo.default_branch)
|
||||
|
|
|
|||
152
endpoints/verbs.py
Normal file
152
endpoints/verbs.py
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
import logging
|
||||
import json
|
||||
import hashlib
|
||||
|
||||
from flask import redirect, Blueprint, abort, send_file, request
|
||||
|
||||
from app import app
|
||||
from auth.auth import process_auth
|
||||
from auth.auth_context import get_authenticated_user
|
||||
from auth.permissions import ReadRepositoryPermission
|
||||
from data import model
|
||||
from data import database
|
||||
from endpoints.trackhelper import track_and_log
|
||||
from storage import Storage
|
||||
|
||||
from util.queuefile import QueueFile
|
||||
from util.queueprocess import QueueProcess
|
||||
from util.gzipwrap import GzipWrap
|
||||
from util.dockerloadformat import build_docker_load_stream
|
||||
|
||||
verbs = Blueprint('verbs', __name__)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def _open_stream(namespace, repository, tag, synthetic_image_id, image_json, image_id_list):
|
||||
store = Storage(app)
|
||||
|
||||
# For performance reasons, we load the full image list here, cache it, then disconnect from
|
||||
# the database.
|
||||
with database.UseThenDisconnect(app.config):
|
||||
image_list = list(model.get_matching_repository_images(namespace, repository, image_id_list))
|
||||
|
||||
image_list.sort(key=lambda image: image_id_list.index(image.docker_image_id))
|
||||
|
||||
def get_next_image():
|
||||
for current_image in image_list:
|
||||
yield current_image
|
||||
|
||||
def get_next_layer():
|
||||
for current_image_entry in image_list:
|
||||
current_image_path = store.image_layer_path(current_image_entry.storage.uuid)
|
||||
current_image_stream = store.stream_read_file(current_image_entry.storage.locations,
|
||||
current_image_path)
|
||||
|
||||
current_image_id = current_image_entry.id
|
||||
logger.debug('Returning image layer %s: %s' % (current_image_id, current_image_path))
|
||||
yield current_image_stream
|
||||
|
||||
stream = build_docker_load_stream(namespace, repository, tag, synthetic_image_id, image_json,
|
||||
get_next_image, get_next_layer)
|
||||
|
||||
return stream.read
|
||||
|
||||
|
||||
def _write_synthetic_image_to_storage(linked_storage_uuid, linked_locations, queue_file):
|
||||
store = Storage(app)
|
||||
|
||||
def handle_exception(ex):
|
||||
logger.debug('Exception when building squashed image %s: %s', linked_storage_uuid, ex)
|
||||
|
||||
with database.UseThenDisconnect(app.config):
|
||||
model.delete_derived_storage_by_uuid(linked_storage_uuid)
|
||||
|
||||
queue_file.add_exception_handler(handle_exception)
|
||||
|
||||
image_path = store.image_layer_path(linked_storage_uuid)
|
||||
store.stream_write(linked_locations, image_path, queue_file)
|
||||
queue_file.close()
|
||||
|
||||
if not queue_file.raised_exception:
|
||||
with database.UseThenDisconnect(app.config):
|
||||
done_uploading = model.get_storage_by_uuid(linked_storage_uuid)
|
||||
done_uploading.uploading = False
|
||||
done_uploading.save()
|
||||
|
||||
|
||||
@verbs.route('/squash/<namespace>/<repository>/<tag>', methods=['GET'])
|
||||
@process_auth
|
||||
def get_squashed_tag(namespace, repository, tag):
|
||||
permission = ReadRepositoryPermission(namespace, repository)
|
||||
if permission.can() or model.repository_is_public(namespace, repository):
|
||||
# Lookup the requested tag.
|
||||
try:
|
||||
tag_image = model.get_tag_image(namespace, repository, tag)
|
||||
except model.DataModelException:
|
||||
abort(404)
|
||||
|
||||
# Lookup the tag's image and storage.
|
||||
repo_image = model.get_repo_image_extended(namespace, repository, tag_image.docker_image_id)
|
||||
if not repo_image:
|
||||
abort(404)
|
||||
|
||||
# Log the action.
|
||||
track_and_log('repo_verb', repo_image.repository, tag=tag, verb='squash')
|
||||
|
||||
store = Storage(app)
|
||||
derived = model.find_or_create_derived_storage(repo_image.storage, 'squash',
|
||||
store.preferred_locations[0])
|
||||
if not derived.uploading:
|
||||
logger.debug('Derived image %s exists in storage', derived.uuid)
|
||||
derived_layer_path = store.image_layer_path(derived.uuid)
|
||||
download_url = store.get_direct_download_url(derived.locations, derived_layer_path)
|
||||
if download_url:
|
||||
logger.debug('Redirecting to download URL for derived image %s', derived.uuid)
|
||||
return redirect(download_url)
|
||||
|
||||
# Close the database handle here for this process before we send the long download.
|
||||
database.close_db_filter(None)
|
||||
|
||||
logger.debug('Sending cached derived image %s', derived.uuid)
|
||||
return send_file(store.stream_read_file(derived.locations, derived_layer_path))
|
||||
|
||||
# Load the ancestry for the image.
|
||||
logger.debug('Building and returning derived image %s', derived.uuid)
|
||||
uuid = repo_image.storage.uuid
|
||||
ancestry_data = store.get_content(repo_image.storage.locations, store.image_ancestry_path(uuid))
|
||||
full_image_list = json.loads(ancestry_data)
|
||||
|
||||
# Load the image's JSON layer.
|
||||
image_json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
|
||||
image_json = json.loads(image_json_data)
|
||||
|
||||
# Calculate a synthetic image ID.
|
||||
synthetic_image_id = hashlib.sha256(tag_image.docker_image_id + ':squash').hexdigest()
|
||||
|
||||
# Create a queue process to generate the data. The queue files will read from the process
|
||||
# and send the results to the client and storage.
|
||||
def _cleanup():
|
||||
# Close any existing DB connection once the process has exited.
|
||||
database.close_db_filter(None)
|
||||
|
||||
args = (namespace, repository, tag, synthetic_image_id, image_json, full_image_list)
|
||||
queue_process = QueueProcess(_open_stream,
|
||||
8 * 1024, 10 * 1024 * 1024, # 8K/10M chunk/max
|
||||
args, finished=_cleanup)
|
||||
|
||||
client_queue_file = QueueFile(queue_process.create_queue(), 'client')
|
||||
storage_queue_file = QueueFile(queue_process.create_queue(), 'storage')
|
||||
|
||||
# Start building.
|
||||
queue_process.run()
|
||||
|
||||
# Start the storage saving.
|
||||
storage_args = (derived.uuid, derived.locations, storage_queue_file)
|
||||
QueueProcess.run_process(_write_synthetic_image_to_storage, storage_args, finished=_cleanup)
|
||||
|
||||
# Close the database handle here for this process before we send the long download.
|
||||
database.close_db_filter(None)
|
||||
|
||||
# Return the client's data.
|
||||
return send_file(client_queue_file)
|
||||
|
||||
abort(403)
|
||||
|
|
@ -5,6 +5,7 @@ from flask import (abort, redirect, request, url_for, make_response, Response,
|
|||
Blueprint, send_from_directory, jsonify)
|
||||
from flask.ext.login import current_user
|
||||
from urlparse import urlparse
|
||||
from health.healthcheck import HealthCheck
|
||||
|
||||
from data import model
|
||||
from data.model.oauth import DatabaseAuthorizationProvider
|
||||
|
|
@ -151,6 +152,20 @@ def v1():
|
|||
return index('')
|
||||
|
||||
|
||||
@web.route('/health', methods=['GET'])
|
||||
@no_cache
|
||||
def health():
|
||||
db_healthy = model.check_health()
|
||||
buildlogs_healthy = build_logs.check_health()
|
||||
|
||||
check = HealthCheck.get_check(app.config['HEALTH_CHECKER'][0], app.config['HEALTH_CHECKER'][1])
|
||||
(data, is_healthy) = check.conduct_healthcheck(db_healthy, buildlogs_healthy)
|
||||
|
||||
response = jsonify(dict(data = data, is_healthy = is_healthy))
|
||||
response.status_code = 200 if is_healthy else 503
|
||||
return response
|
||||
|
||||
|
||||
@web.route('/status', methods=['GET'])
|
||||
@no_cache
|
||||
def status():
|
||||
|
|
@ -160,6 +175,7 @@ def status():
|
|||
response = jsonify({
|
||||
'db_healthy': db_healthy,
|
||||
'buildlogs_healthy': buildlogs_healthy,
|
||||
'is_testing': app.config['TESTING'],
|
||||
})
|
||||
response.status_code = 200 if db_healthy and buildlogs_healthy else 503
|
||||
|
||||
|
|
|
|||
Reference in a new issue