Merge remote-tracking branch 'origin/master' into touchdown

Conflicts:
	static/js/app.js
This commit is contained in:
Jake Moshenko 2014-05-07 15:32:27 -04:00
commit 73a0cc791b
48 changed files with 772 additions and 386 deletions

View file

@ -155,6 +155,7 @@ def render_page_template(name, **kwargs):
feature_set=json.dumps(features.get_features()),
config_set=json.dumps(getFrontendVisibleConfig(app.config)),
mixpanel_key=app.config.get('MIXPANEL_KEY', ''),
sentry_public_dsn=app.config.get('SENTRY_PUBLIC_DSN', ''),
is_debug=str(app.config.get('DEBUGGING', False)).lower(),
show_chat=features.OLARK_CHAT,
cache_buster=cache_buster,

View file

@ -21,6 +21,7 @@ from util.http import abort
logger = logging.getLogger(__name__)
profile = logging.getLogger('application.profiler')
index = Blueprint('index', __name__)
@ -112,9 +113,15 @@ def create_user():
else:
# New user case
profile.debug('Creating user')
new_user = model.create_user(username, password, user_data['email'])
profile.debug('Creating email code for user')
code = model.create_confirm_email_code(new_user)
profile.debug('Sending email code to user')
send_confirmation_email(new_user.username, new_user.email, code.code)
return make_response('Created', 201)
@ -149,12 +156,12 @@ def update_user(username):
update_request = request.get_json()
if 'password' in update_request:
logger.debug('Updating user password.')
profile.debug('Updating user password')
model.change_password(get_authenticated_user(),
update_request['password'])
if 'email' in update_request:
logger.debug('Updating user email')
profile.debug('Updating user email')
model.update_email(get_authenticated_user(), update_request['email'])
return jsonify({
@ -170,9 +177,13 @@ def update_user(username):
@parse_repository_name
@generate_headers(role='write')
def create_repository(namespace, repository):
profile.debug('Parsing image descriptions')
image_descriptions = json.loads(request.data)
profile.debug('Looking up repository')
repo = model.get_repository(namespace, repository)
profile.debug('Repository looked up')
if not repo and get_authenticated_user() is None:
logger.debug('Attempt to create new repository without user auth.')
abort(401,
@ -196,11 +207,11 @@ def create_repository(namespace, repository):
issue='no-create-permission',
namespace=namespace)
logger.debug('Creaing repository with owner: %s' %
get_authenticated_user().username)
profile.debug('Creaing repository with owner: %s', get_authenticated_user().username)
repo = model.create_repository(namespace, repository,
get_authenticated_user())
profile.debug('Determining added images')
added_images = OrderedDict([(desc['id'], desc)
for desc in image_descriptions])
new_repo_images = dict(added_images)
@ -209,12 +220,15 @@ def create_repository(namespace, repository):
if existing.docker_image_id in new_repo_images:
added_images.pop(existing.docker_image_id)
profile.debug('Creating/Linking necessary images')
username = get_authenticated_user() and get_authenticated_user().username
translations = {}
for image_description in added_images.values():
model.find_create_or_link_image(image_description['id'], repo, username,
translations)
profile.debug('Created images')
response = make_response('Created', 201)
extra_params = {
@ -268,21 +282,23 @@ def update_images(namespace, repository):
permission = ModifyRepositoryPermission(namespace, repository)
if permission.can():
profile.debug('Looking up repository')
repo = model.get_repository(namespace, repository)
if not repo:
# Make sure the repo actually exists.
abort(404, message='Unknown repository', issue='unknown-repo')
profile.debug('Parsing image data')
image_with_checksums = json.loads(request.data)
updated_tags = {}
for image in image_with_checksums:
logger.debug('Setting checksum for image id: %s to %s' %
(image['id'], image['checksum']))
profile.debug('Setting checksum for image id: %s to %s', image['id'], image['checksum'])
updated_tags[image['Tag']] = image['id']
model.set_image_checksum(image['id'], repo, image['checksum'])
if get_authenticated_user():
profile.debug('Publishing push event')
username = get_authenticated_user().username
# Mark that the user has pushed the repo.
@ -295,15 +311,18 @@ def update_images(namespace, repository):
event = app.config['USER_EVENTS'].get_event(username)
event.publish_event_data('docker-cli', user_data)
profile.debug('GCing repository')
num_removed = model.garbage_collect_repository(namespace, repository)
# Generate a job for each webhook that has been added to this repo
profile.debug('Adding webhooks for repository')
webhooks = model.list_webhooks(namespace, repository)
for webhook in webhooks:
webhook_data = json.loads(webhook.parameters)
repo_string = '%s/%s' % (namespace, repository)
logger.debug('Creating webhook for repository \'%s\' for url \'%s\'' %
(repo_string, webhook_data['url']))
profile.debug('Creating webhook for repository \'%s\' for url \'%s\'',
repo_string, webhook_data['url'])
webhook_data['payload'] = {
'repository': repo_string,
'namespace': namespace,
@ -330,14 +349,17 @@ def get_repository_images(namespace, repository):
permission = ReadRepositoryPermission(namespace, repository)
# TODO invalidate token?
profile.debug('Looking up public status of repository')
is_public = model.repository_is_public(namespace, repository)
if permission.can() or is_public:
# We can't rely on permissions to tell us if a repo exists anymore
profile.debug('Looking up repository')
repo = model.get_repository(namespace, repository)
if not repo:
abort(404, message='Unknown repository', issue='unknown-repo')
all_images = []
profile.debug('Retrieving repository images')
for image in model.get_repository_images(namespace, repository):
new_image_view = {
'id': image.docker_image_id,
@ -345,6 +367,7 @@ def get_repository_images(namespace, repository):
}
all_images.append(new_image_view)
profile.debug('Building repository image response')
resp = make_response(json.dumps(all_images), 200)
resp.mimetype = 'application/json'
@ -353,6 +376,7 @@ def get_repository_images(namespace, repository):
'namespace': namespace,
}
profile.debug('Logging the pull to Mixpanel and the log system')
if get_validated_oauth_token():
oauth_token = get_validated_oauth_token()
metadata['oauth_token_id'] = oauth_token.id
@ -408,4 +432,5 @@ def get_search():
def ping():
response = make_response('true', 200)
response.headers['X-Docker-Registry-Version'] = '0.6.0'
response.headers['X-Docker-Registry-Standalone'] = '0'
return response

View file

@ -21,7 +21,7 @@ from data import model
registry = Blueprint('registry', __name__)
logger = logging.getLogger(__name__)
profile = logging.getLogger('application.profiler')
class SocketReader(object):
def __init__(self, fp):
@ -40,16 +40,35 @@ class SocketReader(object):
return buf
def image_is_uploading(namespace, repository, image_id, repo_image):
if repo_image and repo_image.storage and repo_image.storage.uploading is not None:
return repo_image.storage.uploading
logger.warning('Setting legacy upload flag')
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
return store.exists(mark_path)
def mark_upload_complete(namespace, repository, image_id, repo_image):
if repo_image and repo_image.storage and repo_image.storage.uploading is not None:
repo_image.storage.uploading = False
repo_image.storage.save()
else:
logger.warning('Removing legacy upload flag')
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
if store.exists(mark_path):
store.remove(mark_path)
def require_completion(f):
"""This make sure that the image push correctly finished."""
@wraps(f)
def wrapper(namespace, repository, *args, **kwargs):
image_id = kwargs['image_id']
repo_image = model.get_repo_image(namespace, repository, image_id)
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
if store.exists(store.image_mark_path(namespace, repository, image_id,
uuid)):
if image_is_uploading(namespace, repository, image_id, repo_image):
abort(400, 'Image %(image_id)s is being uploaded, retry later',
issue='upload-in-progress', image_id=kwargs['image_id'])
@ -88,17 +107,28 @@ def set_cache_headers(f):
@set_cache_headers
def get_image_layer(namespace, repository, image_id, headers):
permission = ReadRepositoryPermission(namespace, repository)
profile.debug('Checking repo permissions')
if permission.can() or model.repository_is_public(namespace, repository):
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
profile.debug('Looking up the layer path')
path = store.image_layer_path(namespace, repository, image_id, uuid)
profile.debug('Looking up the direct download URL')
direct_download_url = store.get_direct_download_url(path)
if direct_download_url:
profile.debug('Returning direct download URL')
return redirect(direct_download_url)
try:
profile.debug('Streaming layer data')
return Response(store.stream_read(path), headers=headers)
except IOError:
profile.debug('Image not found')
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
image_id=image_id)
@ -109,25 +139,32 @@ def get_image_layer(namespace, repository, image_id, headers):
@process_auth
@extract_namespace_repo_from_session
def put_image_layer(namespace, repository, image_id):
profile.debug('Checking repo permissions')
permission = ModifyRepositoryPermission(namespace, repository)
if not permission.can():
abort(403)
profile.debug('Retrieving image')
repo_image = model.get_repo_image(namespace, repository, image_id)
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
try:
profile.debug('Retrieving image data')
json_data = store.get_content(store.image_json_path(namespace, repository,
image_id, uuid))
except IOError:
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
image_id=image_id)
profile.debug('Retrieving image path info')
layer_path = store.image_layer_path(namespace, repository, image_id, uuid)
mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
if store.exists(layer_path) and not store.exists(mark_path):
if (store.exists(layer_path) and not
image_is_uploading(namespace, repository, image_id, repo_image)):
abort(409, 'Image already exists', issue='image-exists', image_id=image_id)
profile.debug('Storing layer data')
input_stream = request.stream
if request.headers.get('transfer-encoding') == 'chunked':
# Careful, might work only with WSGI servers supporting chunked
@ -174,11 +211,11 @@ def put_image_layer(namespace, repository, image_id):
issue='checksum-mismatch', image_id=image_id)
# Checksum is ok, we remove the marker
store.remove(mark_path)
mark_upload_complete(namespace, repository, image_id, repo_image)
# The layer is ready for download, send a job to the work queue to
# process it.
logger.debug('Queing diffs job for image: %s' % image_id)
profile.debug('Adding layer to diff queue')
image_diff_queue.put([namespace, repository, image_id], json.dumps({
'namespace': namespace,
'repository': repository,
@ -192,6 +229,7 @@ def put_image_layer(namespace, repository, image_id):
@process_auth
@extract_namespace_repo_from_session
def put_image_checksum(namespace, repository, image_id):
profile.debug('Checking repo permissions')
permission = ModifyRepositoryPermission(namespace, repository)
if not permission.can():
abort(403)
@ -204,17 +242,22 @@ def put_image_checksum(namespace, repository, image_id):
abort(400, 'Checksum not found in Cookie for image %(imaage_id)s',
issue='missing-checksum-cookie', image_id=image_id)
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
profile.debug('Looking up repo layer data')
if not store.exists(store.image_json_path(namespace, repository, image_id,
uuid)):
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
if not store.exists(mark_path):
profile.debug('Marking image path')
if not image_is_uploading(namespace, repository, image_id, repo_image):
abort(409, 'Cannot set checksum for image %(image_id)s',
issue='image-write-error', image_id=image_id)
profile.debug('Storing image checksum')
err = store_checksum(namespace, repository, image_id, uuid, checksum)
if err:
abort(400, err)
@ -227,11 +270,11 @@ def put_image_checksum(namespace, repository, image_id):
issue='checksum-mismatch', image_id=image_id)
# Checksum is ok, we remove the marker
store.remove(mark_path)
mark_upload_complete(namespace, repository, image_id, repo_image)
# The layer is ready for download, send a job to the work queue to
# process it.
logger.debug('Queing diffs job for image: %s' % image_id)
profile.debug('Adding layer to diff queue')
image_diff_queue.put([namespace, repository, image_id], json.dumps({
'namespace': namespace,
'repository': repository,
@ -247,27 +290,31 @@ def put_image_checksum(namespace, repository, image_id):
@require_completion
@set_cache_headers
def get_image_json(namespace, repository, image_id, headers):
profile.debug('Checking repo permissions')
permission = ReadRepositoryPermission(namespace, repository)
if not permission.can() and not model.repository_is_public(namespace,
repository):
abort(403)
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
profile.debug('Looking up repo layer data')
try:
data = store.get_content(store.image_json_path(namespace, repository,
image_id, uuid))
except IOError:
flask_abort(404)
profile.debug('Looking up repo layer size')
try:
size = store.get_size(store.image_layer_path(namespace, repository,
image_id, uuid))
size = repo_image.image_size or repo_image.storage.image_size
headers['X-Docker-Size'] = str(size)
except OSError:
pass
profile.debug('Retrieving checksum')
checksum_path = store.image_checksum_path(namespace, repository, image_id,
uuid)
if store.exists(checksum_path):
@ -284,14 +331,17 @@ def get_image_json(namespace, repository, image_id, headers):
@require_completion
@set_cache_headers
def get_image_ancestry(namespace, repository, image_id, headers):
profile.debug('Checking repo permissions')
permission = ReadRepositoryPermission(namespace, repository)
if not permission.can() and not model.repository_is_public(namespace,
repository):
abort(403)
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
profile.debug('Looking up image data')
try:
data = store.get_content(store.image_ancestry_path(namespace, repository,
image_id, uuid))
@ -299,8 +349,11 @@ def get_image_ancestry(namespace, repository, image_id, headers):
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
image_id=image_id)
profile.debug('Converting to <-> from JSON')
response = make_response(json.dumps(json.loads(data)), 200)
response.headers.extend(headers)
profile.debug('Done')
return response
@ -335,10 +388,12 @@ def store_checksum(namespace, repository, image_id, uuid, checksum):
@process_auth
@extract_namespace_repo_from_session
def put_image_json(namespace, repository, image_id):
profile.debug('Checking repo permissions')
permission = ModifyRepositoryPermission(namespace, repository)
if not permission.can():
abort(403)
profile.debug('Parsing image JSON')
try:
data = json.loads(request.data)
except json.JSONDecodeError:
@ -351,6 +406,7 @@ def put_image_json(namespace, repository, image_id):
abort(400, 'Missing key `id` in JSON for image: %(image_id)s',
issue='invalid-request', image_id=image_id)
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
@ -358,12 +414,14 @@ def put_image_json(namespace, repository, image_id):
checksum = request.headers.get('X-Docker-Checksum')
if checksum:
# Storing the checksum is optional at this stage
profile.debug('Storing image checksum')
err = store_checksum(namespace, repository, image_id, uuid, checksum)
if err:
abort(400, err, issue='write-error')
else:
# We cleanup any old checksum in case it's a retry after a fail
profile.debug('Cleanup old checksum')
store.remove(store.image_checksum_path(namespace, repository, image_id,
uuid))
if image_id != data['id']:
@ -374,19 +432,27 @@ def put_image_json(namespace, repository, image_id):
parent_image = None
if parent_id:
profile.debug('Looking up parent image')
parent_image = model.get_repo_image(namespace, repository, parent_id)
parent_uuid = (parent_image and parent_image.storage and
parent_image.storage.uuid)
if parent_id:
profile.debug('Looking up parent image data')
if (parent_id and not
store.exists(store.image_json_path(namespace, repository, parent_id,
parent_uuid))):
abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s',
issue='invalid-request', image_id=image_id, parent_id=parent_id)
profile.debug('Looking up image storage paths')
json_path = store.image_json_path(namespace, repository, image_id, uuid)
mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
if store.exists(json_path) and not store.exists(mark_path):
profile.debug('Checking if image already exists')
if (store.exists(json_path) and not
image_is_uploading(namespace, repository, image_id, repo_image)):
abort(409, 'Image already exists', issue='image-exists', image_id=image_id)
# If we reach that point, it means that this is a new image or a retry
@ -394,13 +460,20 @@ def put_image_json(namespace, repository, image_id):
# save the metadata
command_list = data.get('container_config', {}).get('Cmd', None)
command = json.dumps(command_list) if command_list else None
profile.debug('Setting image metadata')
model.set_image_metadata(image_id, namespace, repository,
data.get('created'), data.get('comment'), command,
parent_image)
store.put_content(mark_path, 'true')
profile.debug('Putting json path')
store.put_content(json_path, request.data)
profile.debug('Generating image ancestry')
generate_ancestry(namespace, repository, image_id, uuid, parent_id,
parent_uuid)
profile.debug('Done')
return make_response('true', 200)

View file

@ -20,6 +20,10 @@ TARBALL_MIME = 'application/gzip'
CHUNK_SIZE = 512 * 1024
def should_skip_commit(message):
return '[skip build]' in message or '[build skip]' in message
class BuildArchiveException(Exception):
pass
@ -35,6 +39,9 @@ class TriggerDeactivationException(Exception):
class ValidationRequestException(Exception):
pass
class SkipRequestException(Exception):
pass
class EmptyRepositoryException(Exception):
pass
@ -308,13 +315,20 @@ class GithubBuildTrigger(BuildTrigger):
def handle_trigger_request(self, request, auth_token, config):
payload = request.get_json()
if not payload:
raise SkipRequestException()
if 'zen' in payload:
raise ValidationRequestException()
logger.debug('Payload %s', payload)
ref = payload['ref']
commit_sha = payload['head_commit']['id']
commit_message = payload['head_commit'].get('message', '')
if should_skip_commit(commit_message):
raise SkipRequestException()
short_sha = GithubBuildTrigger.get_display_name(commit_sha)
gh_client = self._get_client(auth_token)

View file

@ -11,7 +11,7 @@ from util.invoice import renderInvoiceToHtml
from util.email import send_invoice_email, send_subscription_change, send_payment_failed
from util.names import parse_repository_name
from util.http import abort
from endpoints.trigger import BuildTrigger, ValidationRequestException
from endpoints.trigger import BuildTrigger, ValidationRequestException, SkipRequestException
from endpoints.common import start_build
@ -30,7 +30,7 @@ def stripe_webhook():
event_type = request_data['type'] if 'type' in request_data else None
if event_type == 'charge.succeeded':
invoice_id = ['data']['object']['invoice']
invoice_id = request_data['data']['object']['invoice']
if user and user.invoice_email:
# Lookup the invoice.
@ -94,6 +94,10 @@ def build_trigger_webhook(namespace, repository, trigger_uuid):
# This was just a validation request, we don't need to build anything
return make_response('Okay')
except SkipRequestException:
# The build was requested to be skipped
return make_response('Okay')
pull_robot_name = model.get_pull_robot_name(trigger)
repo = model.get_repository(namespace, repository)
start_build(repo, dockerfile_id, tags, name, subdir, False, trigger,