vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'master' into nomenclature

Browse source 
Conflicts:
	test/data/test.db
This commit is contained in:
Jake Moshenko 2014-11-17 17:59:59 -05:00
commit f4681f2c18
60 changed files with 1716 additions and 496 deletions
Download patch file Download diff file Expand all files Collapse all files

4
endpoints/api/image.py
View file

@ -73,7 +73,7 @@ class RepositoryImage(RepositoryParamResource):
@nickname('getImage')
def get(self, namespace, repository, image_id):
""" Get the information available for the specified image. """
image = model.get_repo_image(namespace, repository, image_id)
image = model.get_repo_image_extended(namespace, repository, image_id)
if not image:
raise NotFound()
@ -94,7 +94,7 @@ class RepositoryImageChanges(RepositoryParamResource):
@nickname('getImageChanges')
def get(self, namespace, repository, image_id):
""" Get the list of changes for the specified image. """
image = model.get_repo_image(namespace, repository, image_id)
image = model.get_repo_image_extended(namespace, repository, image_id)
if not image:
raise NotFound()

19
endpoints/api/superuser.py
View file

@ -52,6 +52,25 @@ def user_view(user):
'super_user': user.username in app.config['SUPER_USERS']
}
@resource('/v1/superuser/usage/')
@internal_only
@show_if(features.SUPER_USERS)
class UsageInformation(ApiResource):
""" Resource for returning the usage information for enterprise customers. """
@require_fresh_login
@nickname('getSystemUsage')
def get(self):
""" Returns the number of repository handles currently held. """
if SuperUserPermission().can():
return {
'usage': model.get_repository_usage(),
'allowed': app.config.get('MAXIMUM_REPOSITORY_USAGE', 20)
}
abort(403)
@resource('/v1/superuser/users/')
@internal_only
@show_if(features.SUPER_USERS)

77
endpoints/callbacks.py
View file

@ -1,10 +1,11 @@
import logging
import requests
from flask import request, redirect, url_for, Blueprint
from flask.ext.login import current_user
from endpoints.common import render_page_template, common_login, route_show_if
from app import app, analytics, get_app_url
from app import app, analytics, get_app_url, github_login, google_login, github_trigger
from data import model
from util.names import parse_repository_name
from util.validation import generate_valid_usernames
@ -29,20 +30,16 @@ def render_ologin_error(service_name,
service_url=get_app_url(),
user_creation=features.USER_CREATION)
def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_encode=False,
redirect_suffix=''):
def exchange_code_for_token(code, service, form_encode=False, redirect_suffix=''):
code = request.args.get('code')
id_config = service_name + '_LOGIN_CLIENT_ID' if for_login else service_name + '_CLIENT_ID'
secret_config = service_name + '_LOGIN_CLIENT_SECRET' if for_login else service_name + '_CLIENT_SECRET'
payload = {
'client_id': app.config[id_config],
'client_secret': app.config[secret_config],
'client_id': service.client_id(),
'client_secret': service.client_secret(),
'code': code,
'grant_type': 'authorization_code',
'redirect_uri': '%s://%s/oauth2/%s/callback%s' % (app.config['PREFERRED_URL_SCHEME'],
app.config['SERVER_HOSTNAME'],
service_name.lower(),
service.service_name().lower(),
redirect_suffix)
}
@ -50,12 +47,11 @@ def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_en
'Accept': 'application/json'
}
token_url = service.token_endpoint()
if form_encode:
get_access_token = client.post(app.config[service_name + '_TOKEN_URL'],
data=payload, headers=headers)
get_access_token = client.post(token_url, data=payload, headers=headers)
else:
get_access_token = client.post(app.config[service_name + '_TOKEN_URL'],
params=payload, headers=headers)
get_access_token = client.post(token_url, params=payload, headers=headers)
json_data = get_access_token.json()
if not json_data:
@ -65,25 +61,20 @@ def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_en
return token
def get_github_user(token):
token_param = {
'access_token': token,
}
get_user = client.get(app.config['GITHUB_USER_URL'], params=token_param)
return get_user.json()
def get_google_user(token):
def get_user(service, token):
token_param = {
'access_token': token,
'alt': 'json',
}
get_user = client.get(service.user_endpoint(), params=token_param)
if get_user.status_code != requests.codes.ok:
return {}
get_user = client.get(app.config['GOOGLE_USER_URL'], params=token_param)
return get_user.json()
def conduct_oauth_login(service_name, user_id, username, email, metadata={}):
def conduct_oauth_login(service, user_id, username, email, metadata={}):
service_name = service.service_name()
to_login = model.verify_federated_login(service_name.lower(), user_id)
if not to_login:
# See if we can create a new user.
@ -93,8 +84,15 @@ def conduct_oauth_login(service_name, user_id, username, email, metadata={}):
# Try to create the user
try:
valid = next(generate_valid_usernames(username))
to_login = model.create_federated_user(valid, email, service_name.lower(),
new_username = None
for valid in generate_valid_usernames(username):
if model.get_user_or_org(valid):
continue
new_username = valid
break
to_login = model.create_federated_user(new_username, email, service_name.lower(),
user_id, set_password_notification=True,
metadata=metadata)
@ -138,8 +136,8 @@ def google_oauth_callback():
if error:
return render_ologin_error('Google', error)
token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE', form_encode=True)
user_data = get_google_user(token)
token = exchange_code_for_token(request.args.get('code'), google_login, form_encode=True)
user_data = get_user(google_login, token)
if not user_data or not user_data.get('id', None) or not user_data.get('email', None):
return render_ologin_error('Google')
@ -148,7 +146,7 @@ def google_oauth_callback():
'service_username': user_data['email']
}
return conduct_oauth_login('Google', user_data['id'], username, user_data['email'],
return conduct_oauth_login(google_login, user_data['id'], username, user_data['email'],
metadata=metadata)
@ -159,8 +157,8 @@ def github_oauth_callback():
if error:
return render_ologin_error('GitHub', error)
token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB')
user_data = get_github_user(token)
token = exchange_code_for_token(request.args.get('code'), github_login)
user_data = get_user(github_login, token)
if not user_data or not 'login' in user_data:
return render_ologin_error('GitHub')
@ -174,7 +172,7 @@ def github_oauth_callback():
token_param = {
'access_token': token,
}
get_email = client.get(app.config['GITHUB_USER_EMAILS'], params=token_param,
get_email = client.get(github_login.email_endpoint(), params=token_param,
headers=v3_media_type)
# We will accept any email, but we prefer the primary
@ -188,17 +186,17 @@ def github_oauth_callback():
'service_username': username
}
return conduct_oauth_login('github', github_id, username, found_email, metadata=metadata)
return conduct_oauth_login(github_login, github_id, username, found_email, metadata=metadata)
@callback.route('/google/callback/attach', methods=['GET'])
@route_show_if(features.GOOGLE_LOGIN)
@require_session_login
def google_oauth_attach():
token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE',
token = exchange_code_for_token(request.args.get('code'), google_login,
redirect_suffix='/attach', form_encode=True)
user_data = get_google_user(token)
user_data = get_user(google_login, token)
if not user_data or not user_data.get('id', None):
return render_ologin_error('Google')
@ -224,8 +222,8 @@ def google_oauth_attach():
@route_show_if(features.GITHUB_LOGIN)
@require_session_login
def github_oauth_attach():
token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB')
user_data = get_github_user(token)
token = exchange_code_for_token(request.args.get('code'), github_login)
user_data = get_user(github_login, token)
if not user_data:
return render_ologin_error('GitHub')
@ -255,8 +253,7 @@ def github_oauth_attach():
def attach_github_build_trigger(namespace, repository):
permission = AdministerRepositoryPermission(namespace, repository)
if permission.can():
token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB',
for_login=False)
token = exchange_code_for_token(request.args.get('code'), github_trigger)
repo = model.get_repository(namespace, repository)
if not repo:
msg = 'Invalid repository: %s/%s' % (namespace, repository)

19
endpoints/common.py
View file

@ -11,7 +11,8 @@ from random import SystemRandom
from data import model
from data.database import db
from app import app, login_manager, dockerfile_build_queue, notification_queue
from app import app, login_manager, dockerfile_build_queue, notification_queue, oauth_apps
from auth.permissions import QuayDeferredPermissionUser
from auth import scopes
from endpoints.api.discovery import swagger_route_data
@ -20,6 +21,7 @@ from functools import wraps
from config import getFrontendVisibleConfig
from external_libraries import get_external_javascript, get_external_css
from endpoints.notificationhelper import spawn_notification
from util.useremails import CannotSendEmailException
import features
@ -129,6 +131,10 @@ def handle_dme(ex):
logger.exception(ex)
return make_response(json.dumps({'message': ex.message}), 400)
@app.errorhandler(CannotSendEmailException)
def handle_emailexception(ex):
message = 'Could not send email. Please contact an administrator and report this problem.'
return make_response(json.dumps({'message': message}), 400)
def random_string():
random = SystemRandom()
@ -171,6 +177,13 @@ def render_page_template(name, **kwargs):
external_styles = get_external_css(local=not app.config.get('USE_CDN', True))
external_scripts = get_external_javascript(local=not app.config.get('USE_CDN', True))
def get_oauth_config():
oauth_config = {}
for oauth_app in oauth_apps:
oauth_config[oauth_app.key_name] = oauth_app.get_public_config()
return oauth_config
contact_href = None
if len(app.config.get('CONTACT_INFO', [])) == 1:
contact_href = app.config['CONTACT_INFO'][0]
@ -184,6 +197,7 @@ def render_page_template(name, **kwargs):
library_scripts=library_scripts,
feature_set=json.dumps(features.get_features()),
config_set=json.dumps(getFrontendVisibleConfig(app.config)),
oauth_set=json.dumps(get_oauth_config()),
mixpanel_key=app.config.get('MIXPANEL_KEY', ''),
google_analytics_key=app.config.get('GOOGLE_ANALYTICS_KEY', ''),
sentry_public_dsn=app.config.get('SENTRY_PUBLIC_DSN', ''),
@ -265,5 +279,4 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
spawn_notification(repository, 'build_queued', event_data,
subpage='build?current=%s' % build_request.uuid,
pathargs=['build', build_request.uuid])
return build_request
return build_request

98
endpoints/index.py
View file

@ -8,7 +8,7 @@ from collections import OrderedDict
from data import model
from data.model import oauth
from app import analytics, app, authentication, userevents, storage
from app import app, authentication, userevents, storage
from auth.auth import process_auth
from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token
from util.names import parse_repository_name
@ -17,6 +17,7 @@ from auth.permissions import (ModifyRepositoryPermission, UserAdminPermission,
ReadRepositoryPermission, CreateRepositoryPermission)
from util.http import abort
from endpoints.trackhelper import track_and_log
from endpoints.notificationhelper import spawn_notification
import features
@ -222,13 +223,20 @@ def create_repository(namespace, repository):
repo = model.create_repository(namespace, repository,
get_authenticated_user())
profile.debug('Determining added images')
added_images = OrderedDict([(desc['id'], desc)
for desc in image_descriptions])
profile.debug('Determining already added images')
added_images = OrderedDict([(desc['id'], desc) for desc in image_descriptions])
new_repo_images = dict(added_images)
for existing in model.get_repository_images(namespace, repository):
if existing.docker_image_id in new_repo_images:
# Optimization: Lookup any existing images in the repository with matching docker IDs and
# remove them from the added dict, so we don't need to look them up one-by-one.
def chunks(l, n):
for i in xrange(0, len(l), n):
yield l[i:i+n]
# Note: We do this in chunks in an effort to not hit the SQL query size limit.
for chunk in chunks(new_repo_images.keys(), 50):
existing_images = model.lookup_repository_images(namespace, repository, chunk)
for existing in existing_images:
added_images.pop(existing.docker_image_id)
profile.debug('Creating/Linking necessary images')
@ -240,49 +248,8 @@ def create_repository(namespace, repository):
profile.debug('Created images')
response = make_response('Created', 201)
extra_params = {
'repository': '%s/%s' % (namespace, repository),
}
metadata = {
'repo': repository,
'namespace': namespace
}
if get_validated_oauth_token():
analytics.track(username, 'push_repo', extra_params)
oauth_token = get_validated_oauth_token()
metadata['oauth_token_id'] = oauth_token.id
metadata['oauth_token_application_id'] = oauth_token.application.client_id
metadata['oauth_token_application'] = oauth_token.application.name
elif get_authenticated_user():
username = get_authenticated_user().username
analytics.track(username, 'push_repo', extra_params)
metadata['username'] = username
# Mark that the user has started pushing the repo.
user_data = {
'action': 'push_repo',
'repository': repository,
'namespace': namespace
}
event = userevents.get_event(username)
event.publish_event_data('docker-cli', user_data)
elif get_validated_token():
analytics.track(get_validated_token().code, 'push_repo', extra_params)
metadata['token'] = get_validated_token().friendly_name
metadata['token_code'] = get_validated_token().code
model.log_action('push_repo', namespace, performer=get_authenticated_user(),
ip=request.remote_addr, metadata=metadata, repository=repo)
return response
track_and_log('push_repo', repo)
return make_response('Created', 201)
@index.route('/repositories/<path:repository>/images', methods=['PUT'])
@ -360,38 +327,7 @@ def get_repository_images(namespace, repository):
resp = make_response(json.dumps(all_images), 200)
resp.mimetype = 'application/json'
metadata = {
'repo': repository,
'namespace': namespace,
}
profile.debug('Logging the pull to Mixpanel and the log system')
if get_validated_oauth_token():
oauth_token = get_validated_oauth_token()
metadata['oauth_token_id'] = oauth_token.id
metadata['oauth_token_application_id'] = oauth_token.application.client_id
metadata['oauth_token_application'] = oauth_token.application.name
elif get_authenticated_user():
metadata['username'] = get_authenticated_user().username
elif get_validated_token():
metadata['token'] = get_validated_token().friendly_name
metadata['token_code'] = get_validated_token().code
else:
metadata['public'] = True
pull_username = 'anonymous'
if get_authenticated_user():
pull_username = get_authenticated_user().username
extra_params = {
'repository': '%s/%s' % (namespace, repository),
}
analytics.track(pull_username, 'pull_repo', extra_params)
model.log_action('pull_repo', namespace,
performer=get_authenticated_user(),
ip=request.remote_addr, metadata=metadata,
repository=repo)
track_and_log('pull_repo', repo)
return resp
abort(403)

31
endpoints/registry.py
View file

@ -7,13 +7,13 @@ from functools import wraps
from datetime import datetime
from time import time
from app import storage as store, image_diff_queue
from app import storage as store, image_diff_queue, app
from auth.auth import process_auth, extract_namespace_repo_from_session
from util import checksums, changes
from util.http import abort, exact_abort
from auth.permissions import (ReadRepositoryPermission,
ModifyRepositoryPermission)
from data import model
from data import model, database
from util import gzipstream
@ -59,7 +59,7 @@ def require_completion(f):
@wraps(f)
def wrapper(namespace, repository, *args, **kwargs):
image_id = kwargs['image_id']
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
if image_is_uploading(repo_image):
abort(400, 'Image %(image_id)s is being uploaded, retry later',
issue='upload-in-progress', image_id=kwargs['image_id'])
@ -103,7 +103,7 @@ def head_image_layer(namespace, repository, image_id, headers):
profile.debug('Checking repo permissions')
if permission.can() or model.repository_is_public(namespace, repository):
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
if not repo_image:
profile.debug('Image not found')
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
@ -136,7 +136,7 @@ def get_image_layer(namespace, repository, image_id, headers):
profile.debug('Checking repo permissions')
if permission.can() or model.repository_is_public(namespace, repository):
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
profile.debug('Looking up the layer path')
try:
@ -151,6 +151,10 @@ def get_image_layer(namespace, repository, image_id, headers):
return resp
profile.debug('Streaming layer data')
# Close the database handle here for this process before we send the long download.
database.close_db_filter(None)
return Response(store.stream_read(repo_image.storage.locations, path), headers=headers)
except (IOError, AttributeError):
profile.debug('Image not found')
@ -170,7 +174,7 @@ def put_image_layer(namespace, repository, image_id):
abort(403)
profile.debug('Retrieving image')
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
try:
profile.debug('Retrieving image data')
uuid = repo_image.storage.uuid
@ -213,7 +217,8 @@ def put_image_layer(namespace, repository, image_id):
sr.add_handler(sum_hndlr)
# Stream write the data to storage.
store.stream_write(repo_image.storage.locations, layer_path, sr)
with database.CloseForLongOperation(app.config):
store.stream_write(repo_image.storage.locations, layer_path, sr)
# Append the computed checksum.
csums = []
@ -294,7 +299,7 @@ def put_image_checksum(namespace, repository, image_id):
issue='missing-checksum-cookie', image_id=image_id)
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
if not repo_image or not repo_image.storage:
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
@ -350,7 +355,7 @@ def get_image_json(namespace, repository, image_id, headers):
abort(403)
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
profile.debug('Looking up repo layer data')
try:
@ -381,7 +386,7 @@ def get_image_ancestry(namespace, repository, image_id, headers):
abort(403)
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
profile.debug('Looking up image data')
try:
@ -445,7 +450,7 @@ def put_image_json(namespace, repository, image_id):
issue='invalid-request', image_id=image_id)
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
if not repo_image:
profile.debug('Image not found')
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
@ -462,7 +467,7 @@ def put_image_json(namespace, repository, image_id):
parent_image = None
if parent_id:
profile.debug('Looking up parent image')
parent_image = model.get_repo_image(namespace, repository, parent_id)
parent_image = model.get_repo_image_extended(namespace, repository, parent_id)
parent_uuid = parent_image and parent_image.storage.uuid
parent_locations = parent_image and parent_image.storage.locations
@ -515,7 +520,7 @@ def put_image_json(namespace, repository, image_id):
def process_image_changes(namespace, repository, image_id):
logger.debug('Generating diffs for image: %s' % image_id)
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
if not repo_image:
logger.warning('No image for id: %s', image_id)
return None, None

62
endpoints/trackhelper.py Normal file
View file

@ -0,0 +1,62 @@
import logging
from app import analytics, app, userevents
from data import model
from flask import request
from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token
logger = logging.getLogger(__name__)
profile = logging.getLogger('application.profiler')
def track_and_log(event_name, repo, **kwargs):
repository = repo.name
namespace = repo.namespace_user.username
metadata = {
'repo': repository,
'namespace': namespace,
}
metadata.update(kwargs)
analytics_id = 'anonymous'
profile.debug('Logging the %s to Mixpanel and the log system', event_name)
if get_validated_oauth_token():
oauth_token = get_validated_oauth_token()
metadata['oauth_token_id'] = oauth_token.id
metadata['oauth_token_application_id'] = oauth_token.application.client_id
metadata['oauth_token_application'] = oauth_token.application.name
analytics_id = 'oauth:' + oauth_token.id
elif get_authenticated_user():
metadata['username'] = get_authenticated_user().username
analytics_id = get_authenticated_user().username
elif get_validated_token():
metadata['token'] = get_validated_token().friendly_name
metadata['token_code'] = get_validated_token().code
analytics_id = 'token:' + get_validated_token().code
else:
metadata['public'] = True
analytics_id = 'anonymous'
extra_params = {
'repository': '%s/%s' % (namespace, repository),
}
# Publish the user event (if applicable)
if get_authenticated_user():
user_event_data = {
'action': event_name,
'repository': repository,
'namespace': namespace
}
event = userevents.get_event(get_authenticated_user().username)
event.publish_event_data('docker-cli', user_event_data)
# Save the action to mixpanel.
analytics.track(analytics_id, event_name, extra_params)
# Log the action to the database.
model.log_action(event_name, namespace,
performer=get_authenticated_user(),
ip=request.remote_addr, metadata=metadata,
repository=repo)

50
endpoints/trigger.py
View file

@ -8,7 +8,7 @@ import re
from github import Github, UnknownObjectException, GithubException
from tempfile import SpooledTemporaryFile
from app import app, userfiles as user_files
from app import app, userfiles as user_files, github_trigger
from util.tarfileappender import TarfileAppender
@ -150,8 +150,8 @@ def raise_unsupported():
class GithubBuildTrigger(BuildTrigger):
@staticmethod
def _get_client(auth_token):
return Github(auth_token, client_id=app.config['GITHUB_CLIENT_ID'],
client_secret=app.config['GITHUB_CLIENT_SECRET'])
return Github(auth_token, client_id=github_trigger.client_id(),
client_secret=github_trigger.client_secret())
@classmethod
def service_name(cls):
@ -231,15 +231,16 @@ class GithubBuildTrigger(BuildTrigger):
return repos_by_org
def matches_branch(self, branch_name, regex):
def matches_ref(self, ref, regex):
match_string = ref.split('/', 1)[1]
if not regex:
return False
m = regex.match(branch_name)
m = regex.match(match_string)
if not m:
return False
return len(m.group(0)) == len(branch_name)
return len(m.group(0)) == len(match_string)
def list_build_subdirs(self, auth_token, config):
gh_client = self._get_client(auth_token)
@ -250,11 +251,11 @@ class GithubBuildTrigger(BuildTrigger):
# Find the first matching branch.
branches = None
if 'branch_regex' in config:
if 'branchtag_regex' in config:
try:
regex = re.compile(config['branch_regex'])
regex = re.compile(config['branchtag_regex'])
branches = [branch.name for branch in repo.get_branches()
if self.matches_branch(branch.name, regex)]
if self.matches_ref('refs/heads/' + branch.name, regex)]
except:
pass
@ -370,14 +371,13 @@ class GithubBuildTrigger(BuildTrigger):
commit_sha = payload['head_commit']['id']
commit_message = payload['head_commit'].get('message', '')
if 'branch_regex' in config:
if 'branchtag_regex' in config:
try:
regex = re.compile(config['branch_regex'])
regex = re.compile(config['branchtag_regex'])
except:
regex = re.compile('.*')
branch = ref.split('/')[-1]
if not self.matches_branch(branch, regex):
if not self.matches_ref(ref, regex):
raise SkipRequestException()
if should_skip_commit(commit_message):
@ -403,17 +403,31 @@ class GithubBuildTrigger(BuildTrigger):
gh_client = self._get_client(auth_token)
repo = gh_client.get_repo(source)
master = repo.get_branch(repo.default_branch)
master_sha = master.commit.sha
short_sha = GithubBuildTrigger.get_display_name(master_sha)
ref = 'refs/heads/%s' % (run_parameters.get('branch_name') or repo.default_branch)
branch_name = run_parameters.get('branch_name') or repo.default_branch
branch = repo.get_branch(branch_name)
branch_sha = branch.commit.sha
short_sha = GithubBuildTrigger.get_display_name(branch_sha)
ref = 'refs/heads/%s' % (branch_name)
return self._prepare_build(config, repo, master_sha, short_sha, ref)
return self._prepare_build(config, repo, branch_sha, short_sha, ref)
except GithubException as ghe:
raise TriggerStartException(ghe.data['message'])
def list_field_values(self, auth_token, config, field_name):
if field_name == 'refs':
branches = self.list_field_values(auth_token, config, 'branch_name')
tags = self.list_field_values(auth_token, config, 'tag_name')
return ([{'kind': 'branch', 'name': b} for b in branches] +
[{'kind': 'tag', 'name': tag} for tag in tags])
if field_name == 'tag_name':
gh_client = self._get_client(auth_token)
source = config['build_source']
repo = gh_client.get_repo(source)
return [tag.name for tag in repo.get_tags()]
if field_name == 'branch_name':
gh_client = self._get_client(auth_token)
source = config['build_source']

46
endpoints/verbs.py
View file

@ -2,13 +2,15 @@ import logging
import json
import hashlib
from flask import redirect, Blueprint, abort, send_file
from flask import redirect, Blueprint, abort, send_file, request
from app import app
from auth.auth import process_auth
from auth.auth_context import get_authenticated_user
from auth.permissions import ReadRepositoryPermission
from data import model
from data import database
from endpoints.trackhelper import track_and_log
from storage import Storage
from util.queuefile import QueueFile
@ -16,28 +18,33 @@ from util.queueprocess import QueueProcess
from util.gzipwrap import GzipWrap
from util.dockerloadformat import build_docker_load_stream
verbs = Blueprint('verbs', __name__)
logger = logging.getLogger(__name__)
def _open_stream(namespace, repository, tag, synthetic_image_id, image_json, image_list):
def _open_stream(namespace, repository, tag, synthetic_image_id, image_json, image_id_list):
store = Storage(app)
# For performance reasons, we load the full image list here, cache it, then disconnect from
# the database.
with database.UseThenDisconnect(app.config):
image_list = list(model.get_matching_repository_images(namespace, repository, image_id_list))
image_list.sort(key=lambda image: image_id_list.index(image.docker_image_id))
def get_next_image():
for current_image_id in image_list:
yield model.get_repo_image(namespace, repository, current_image_id)
for current_image in image_list:
yield current_image
def get_next_layer():
for current_image_id in image_list:
current_image_entry = model.get_repo_image(namespace, repository, current_image_id)
for current_image_entry in image_list:
current_image_path = store.image_layer_path(current_image_entry.storage.uuid)
current_image_stream = store.stream_read_file(current_image_entry.storage.locations,
current_image_path)
current_image_id = current_image_entry.id
logger.debug('Returning image layer %s: %s' % (current_image_id, current_image_path))
yield current_image_stream
database.configure(app.config)
stream = build_docker_load_stream(namespace, repository, tag, synthetic_image_id, image_json,
get_next_image, get_next_layer)
@ -45,12 +52,13 @@ def _open_stream(namespace, repository, tag, synthetic_image_id, image_json, ima
def _write_synthetic_image_to_storage(linked_storage_uuid, linked_locations, queue_file):
database.configure(app.config)
store = Storage(app)
def handle_exception(ex):
logger.debug('Exception when building squashed image %s: %s', linked_storage_uuid, ex)
model.delete_derived_storage_by_uuid(linked_storage_uuid)
with database.UseThenDisconnect(app.config):
model.delete_derived_storage_by_uuid(linked_storage_uuid)
queue_file.add_exception_handler(handle_exception)
@ -59,9 +67,10 @@ def _write_synthetic_image_to_storage(linked_storage_uuid, linked_locations, que
queue_file.close()
if not queue_file.raised_exception:
done_uploading = model.get_storage_by_uuid(linked_storage_uuid)
done_uploading.uploading = False
done_uploading.save()
with database.UseThenDisconnect(app.config):
done_uploading = model.get_storage_by_uuid(linked_storage_uuid)
done_uploading.uploading = False
done_uploading.save()
@verbs.route('/squash/<namespace>/<repository>/<tag>', methods=['GET'])
@ -76,10 +85,13 @@ def get_squashed_tag(namespace, repository, tag):
abort(404)
# Lookup the tag's image and storage.
repo_image = model.get_repo_image(namespace, repository, tag_image.docker_image_id)
repo_image = model.get_repo_image_extended(namespace, repository, tag_image.docker_image_id)
if not repo_image:
abort(404)
# Log the action.
track_and_log('repo_verb', repo_image.repository, tag=tag, verb='squash')
store = Storage(app)
derived = model.find_or_create_derived_storage(repo_image.storage, 'squash',
store.preferred_locations[0])
@ -91,6 +103,9 @@ def get_squashed_tag(namespace, repository, tag):
logger.debug('Redirecting to download URL for derived image %s', derived.uuid)
return redirect(download_url)
# Close the database handle here for this process before we send the long download.
database.close_db_filter(None)
logger.debug('Sending cached derived image %s', derived.uuid)
return send_file(store.stream_read_file(derived.locations, derived_layer_path))
@ -128,6 +143,9 @@ def get_squashed_tag(namespace, repository, tag):
storage_args = (derived.uuid, derived.locations, storage_queue_file)
QueueProcess.run_process(_write_synthetic_image_to_storage, storage_args, finished=_cleanup)
# Close the database handle here for this process before we send the long download.
database.close_db_filter(None)
# Return the client's data.
return send_file(client_queue_file)

16
endpoints/web.py
View file

@ -5,6 +5,7 @@ from flask import (abort, redirect, request, url_for, make_response, Response,
Blueprint, send_from_directory, jsonify)
from flask.ext.login import current_user
from urlparse import urlparse
from health.healthcheck import HealthCheck
from data import model
from data.model.oauth import DatabaseAuthorizationProvider
@ -151,6 +152,20 @@ def v1():
return index('')
@web.route('/health', methods=['GET'])
@no_cache
def health():
db_healthy = model.check_health()
buildlogs_healthy = build_logs.check_health()
check = HealthCheck.get_check(app.config['HEALTH_CHECKER'][0], app.config['HEALTH_CHECKER'][1])
(data, is_healthy) = check.conduct_healthcheck(db_healthy, buildlogs_healthy)
response = jsonify(dict(data = data, is_healthy = is_healthy))
response.status_code = 200 if is_healthy else 503
return response
@web.route('/status', methods=['GET'])
@no_cache
def status():
@ -160,6 +175,7 @@ def status():
response = jsonify({
'db_healthy': db_healthy,
'buildlogs_healthy': buildlogs_healthy,
'is_testing': app.config['TESTING'],
})
response.status_code = 200 if db_healthy and buildlogs_healthy else 503