Merge remote-tracking branch 'origin/master' into touchdown

Conflicts:
	static/js/app.js

endpoints/common.py

@@ -155,6 +155,7 @@ def render_page_template(name, **kwargs):
                                        feature_set=json.dumps(features.get_features()),
                                        config_set=json.dumps(getFrontendVisibleConfig(app.config)),
                                        mixpanel_key=app.config.get('MIXPANEL_KEY', ''),
+                                       sentry_public_dsn=app.config.get('SENTRY_PUBLIC_DSN', ''),
                                        is_debug=str(app.config.get('DEBUGGING', False)).lower(),
                                        show_chat=features.OLARK_CHAT,
                                        cache_buster=cache_buster,

endpoints/index.py

@@ -21,6 +21,7 @@ from util.http import abort
 
 
 logger = logging.getLogger(__name__)
+profile = logging.getLogger('application.profiler')
 
 index = Blueprint('index', __name__)
 
@@ -112,9 +113,15 @@ def create_user():
 
   else:
     # New user case
+    profile.debug('Creating user')
     new_user = model.create_user(username, password, user_data['email'])
+
+    profile.debug('Creating email code for user')
     code = model.create_confirm_email_code(new_user)
+
+    profile.debug('Sending email code to user')
     send_confirmation_email(new_user.username, new_user.email, code.code)
+
     return make_response('Created', 201)
 
 
@@ -149,12 +156,12 @@ def update_user(username):
     update_request = request.get_json()
 
     if 'password' in update_request:
-      logger.debug('Updating user password.')
+      profile.debug('Updating user password')
       model.change_password(get_authenticated_user(),
                             update_request['password'])
 
     if 'email' in update_request:
-      logger.debug('Updating user email')
+      profile.debug('Updating user email')
       model.update_email(get_authenticated_user(), update_request['email'])
 
     return jsonify({
@@ -170,9 +177,13 @@ def update_user(username):
 @parse_repository_name
 @generate_headers(role='write')
 def create_repository(namespace, repository):
+  profile.debug('Parsing image descriptions')
   image_descriptions = json.loads(request.data)
+
+  profile.debug('Looking up repository')
   repo = model.get_repository(namespace, repository)
 
+  profile.debug('Repository looked up')
   if not repo and get_authenticated_user() is None:
     logger.debug('Attempt to create new repository without user auth.')
     abort(401,
@@ -196,11 +207,11 @@ def create_repository(namespace, repository):
             issue='no-create-permission',
             namespace=namespace)
 
-    logger.debug('Creaing repository with owner: %s' %
-                 get_authenticated_user().username)
+    profile.debug('Creaing repository with owner: %s', get_authenticated_user().username)
     repo = model.create_repository(namespace, repository,
                                    get_authenticated_user())
 
+  profile.debug('Determining added images')
   added_images = OrderedDict([(desc['id'], desc)
                               for desc in image_descriptions])
   new_repo_images = dict(added_images)
@@ -209,12 +220,15 @@ def create_repository(namespace, repository):
     if existing.docker_image_id in new_repo_images:
       added_images.pop(existing.docker_image_id)
 
+  profile.debug('Creating/Linking necessary images')
   username = get_authenticated_user() and get_authenticated_user().username
   translations = {}
   for image_description in added_images.values():
     model.find_create_or_link_image(image_description['id'], repo, username,
                                     translations)
 
+
+  profile.debug('Created images')
   response = make_response('Created', 201)
 
   extra_params = {
@@ -268,21 +282,23 @@ def update_images(namespace, repository):
   permission = ModifyRepositoryPermission(namespace, repository)
 
   if permission.can():
+    profile.debug('Looking up repository')
     repo = model.get_repository(namespace, repository)
     if not repo:
       # Make sure the repo actually exists.
       abort(404, message='Unknown repository', issue='unknown-repo')
 
+    profile.debug('Parsing image data')
     image_with_checksums = json.loads(request.data)
 
     updated_tags = {}
     for image in image_with_checksums:
-      logger.debug('Setting checksum for image id: %s to %s' %
-                   (image['id'], image['checksum']))
+      profile.debug('Setting checksum for image id: %s to %s', image['id'], image['checksum'])
       updated_tags[image['Tag']] = image['id']
       model.set_image_checksum(image['id'], repo, image['checksum'])
 
     if get_authenticated_user():
+      profile.debug('Publishing push event')
       username = get_authenticated_user().username
 
       # Mark that the user has pushed the repo.
@@ -295,15 +311,18 @@ def update_images(namespace, repository):
       event = app.config['USER_EVENTS'].get_event(username)
       event.publish_event_data('docker-cli', user_data)
 
+    profile.debug('GCing repository')
     num_removed = model.garbage_collect_repository(namespace, repository)
 
     # Generate a job for each webhook that has been added to this repo
+    profile.debug('Adding webhooks for repository')
+
     webhooks = model.list_webhooks(namespace, repository)
     for webhook in webhooks:
       webhook_data = json.loads(webhook.parameters)
       repo_string = '%s/%s' % (namespace, repository)
-      logger.debug('Creating webhook for repository \'%s\' for url \'%s\'' %
-                   (repo_string, webhook_data['url']))
+      profile.debug('Creating webhook for repository \'%s\' for url \'%s\'',
+                    repo_string, webhook_data['url'])
       webhook_data['payload'] = {
         'repository': repo_string,
         'namespace': namespace,
@@ -330,14 +349,17 @@ def get_repository_images(namespace, repository):
   permission = ReadRepositoryPermission(namespace, repository)
 
   # TODO invalidate token?
+  profile.debug('Looking up public status of repository')
   is_public = model.repository_is_public(namespace, repository)
   if permission.can() or is_public:
     # We can't rely on permissions to tell us if a repo exists anymore
+    profile.debug('Looking up repository')
     repo = model.get_repository(namespace, repository)
     if not repo:
       abort(404, message='Unknown repository', issue='unknown-repo')
 
     all_images = []
+    profile.debug('Retrieving repository images')
     for image in model.get_repository_images(namespace, repository):
       new_image_view = {
         'id': image.docker_image_id,
@@ -345,6 +367,7 @@ def get_repository_images(namespace, repository):
       }
       all_images.append(new_image_view)
 
+    profile.debug('Building repository image response')
     resp = make_response(json.dumps(all_images), 200)
     resp.mimetype = 'application/json'
 
@@ -353,6 +376,7 @@ def get_repository_images(namespace, repository):
       'namespace': namespace,
     }
 
+    profile.debug('Logging the pull to Mixpanel and the log system')
     if get_validated_oauth_token():
       oauth_token = get_validated_oauth_token()
       metadata['oauth_token_id'] = oauth_token.id
@@ -408,4 +432,5 @@ def get_search():
 def ping():
   response = make_response('true', 200)
   response.headers['X-Docker-Registry-Version'] = '0.6.0'
+  response.headers['X-Docker-Registry-Standalone'] = '0'
   return response

endpoints/registry.py

@@ -21,7 +21,7 @@ from data import model
 registry = Blueprint('registry', __name__)
 
 logger = logging.getLogger(__name__)
-
+profile = logging.getLogger('application.profiler')
 
 class SocketReader(object):
   def __init__(self, fp):
@@ -40,16 +40,35 @@ class SocketReader(object):
     return buf
 
 
+def image_is_uploading(namespace, repository, image_id, repo_image):
+  if repo_image and repo_image.storage and repo_image.storage.uploading is not None:
+    return repo_image.storage.uploading
+
+  logger.warning('Setting legacy upload flag')
+  uuid = repo_image and repo_image.storage and repo_image.storage.uuid
+  mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
+  return store.exists(mark_path)
+
+
+def mark_upload_complete(namespace, repository, image_id, repo_image):
+  if repo_image and repo_image.storage and repo_image.storage.uploading is not None:
+    repo_image.storage.uploading = False
+    repo_image.storage.save()
+  else:
+    logger.warning('Removing legacy upload flag')
+    uuid = repo_image and repo_image.storage and repo_image.storage.uuid
+    mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
+    if store.exists(mark_path):
+      store.remove(mark_path)
+
+
 def require_completion(f):
   """This make sure that the image push correctly finished."""
   @wraps(f)
   def wrapper(namespace, repository, *args, **kwargs):
     image_id = kwargs['image_id']
     repo_image = model.get_repo_image(namespace, repository, image_id)
-    uuid = repo_image and repo_image.storage and repo_image.storage.uuid
-
-    if store.exists(store.image_mark_path(namespace, repository, image_id,
-                                          uuid)):
+    if image_is_uploading(namespace, repository, image_id, repo_image):
       abort(400, 'Image %(image_id)s is being uploaded, retry later',
             issue='upload-in-progress', image_id=kwargs['image_id'])
 
@@ -88,17 +107,28 @@ def set_cache_headers(f):
 @set_cache_headers
 def get_image_layer(namespace, repository, image_id, headers):
   permission = ReadRepositoryPermission(namespace, repository)
+
+  profile.debug('Checking repo permissions')
   if permission.can() or model.repository_is_public(namespace, repository):
+    profile.debug('Looking up repo image')
     repo_image = model.get_repo_image(namespace, repository, image_id)
+
     uuid = repo_image and repo_image.storage and repo_image.storage.uuid
 
+    profile.debug('Looking up the layer path')
     path = store.image_layer_path(namespace, repository, image_id, uuid)
+
+    profile.debug('Looking up the direct download URL')
     direct_download_url = store.get_direct_download_url(path)
+
     if direct_download_url:
+      profile.debug('Returning direct download URL')
       return redirect(direct_download_url)
     try:
+      profile.debug('Streaming layer data')
       return Response(store.stream_read(path), headers=headers)
     except IOError:
+      profile.debug('Image not found')
       abort(404, 'Image %(image_id)s not found', issue='unknown-image',
             image_id=image_id)
   
@@ -109,25 +139,32 @@ def get_image_layer(namespace, repository, image_id, headers):
 @process_auth
 @extract_namespace_repo_from_session
 def put_image_layer(namespace, repository, image_id):
+  profile.debug('Checking repo permissions')
   permission = ModifyRepositoryPermission(namespace, repository)
   if not permission.can():
     abort(403)
 
+  profile.debug('Retrieving image')
   repo_image = model.get_repo_image(namespace, repository, image_id)
+
   uuid = repo_image and repo_image.storage and repo_image.storage.uuid
   try:
+    profile.debug('Retrieving image data')
     json_data = store.get_content(store.image_json_path(namespace, repository,
                                                         image_id, uuid))
   except IOError:
     abort(404, 'Image %(image_id)s not found', issue='unknown-image',
           image_id=image_id)
 
+  profile.debug('Retrieving image path info')
   layer_path = store.image_layer_path(namespace, repository, image_id, uuid)
-  mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
 
-  if store.exists(layer_path) and not store.exists(mark_path):
+  if (store.exists(layer_path) and not
+      image_is_uploading(namespace, repository, image_id, repo_image)):
     abort(409, 'Image already exists', issue='image-exists', image_id=image_id)
 
+  profile.debug('Storing layer data')
+
   input_stream = request.stream
   if request.headers.get('transfer-encoding') == 'chunked':
     # Careful, might work only with WSGI servers supporting chunked
@@ -174,11 +211,11 @@ def put_image_layer(namespace, repository, image_id):
           issue='checksum-mismatch', image_id=image_id)
 
   # Checksum is ok, we remove the marker
-  store.remove(mark_path)
+  mark_upload_complete(namespace, repository, image_id, repo_image)
 
   # The layer is ready for download, send a job to the work queue to
   # process it.
-  logger.debug('Queing diffs job for image: %s' % image_id)
+  profile.debug('Adding layer to diff queue')
   image_diff_queue.put([namespace, repository, image_id], json.dumps({
     'namespace': namespace,
     'repository': repository,
@@ -192,6 +229,7 @@ def put_image_layer(namespace, repository, image_id):
 @process_auth
 @extract_namespace_repo_from_session
 def put_image_checksum(namespace, repository, image_id):
+  profile.debug('Checking repo permissions')
   permission = ModifyRepositoryPermission(namespace, repository)
   if not permission.can():
     abort(403)
@@ -204,17 +242,22 @@ def put_image_checksum(namespace, repository, image_id):
     abort(400, 'Checksum not found in Cookie for image %(imaage_id)s',
           issue='missing-checksum-cookie', image_id=image_id)
 
+  profile.debug('Looking up repo image')
   repo_image = model.get_repo_image(namespace, repository, image_id)
+
   uuid = repo_image and repo_image.storage and repo_image.storage.uuid
+
+  profile.debug('Looking up repo layer data')
   if not store.exists(store.image_json_path(namespace, repository, image_id,
                       uuid)):
     abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
 
-  mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
-  if not store.exists(mark_path):
+  profile.debug('Marking image path')
+  if not image_is_uploading(namespace, repository, image_id, repo_image):
     abort(409, 'Cannot set checksum for image %(image_id)s',
           issue='image-write-error', image_id=image_id)
 
+  profile.debug('Storing image checksum')
   err = store_checksum(namespace, repository, image_id, uuid, checksum)
   if err:
     abort(400, err)
@@ -227,11 +270,11 @@ def put_image_checksum(namespace, repository, image_id):
           issue='checksum-mismatch', image_id=image_id)
 
   # Checksum is ok, we remove the marker
-  store.remove(mark_path)
+  mark_upload_complete(namespace, repository, image_id, repo_image)
 
   # The layer is ready for download, send a job to the work queue to
   # process it.
-  logger.debug('Queing diffs job for image: %s' % image_id)
+  profile.debug('Adding layer to diff queue')
   image_diff_queue.put([namespace, repository, image_id], json.dumps({
     'namespace': namespace,
     'repository': repository,
@@ -247,27 +290,31 @@ def put_image_checksum(namespace, repository, image_id):
 @require_completion
 @set_cache_headers
 def get_image_json(namespace, repository, image_id, headers):
+  profile.debug('Checking repo permissions')
   permission = ReadRepositoryPermission(namespace, repository)
   if not permission.can() and not model.repository_is_public(namespace,
                                                              repository):
     abort(403)
 
+  profile.debug('Looking up repo image')
   repo_image = model.get_repo_image(namespace, repository, image_id)
   uuid = repo_image and repo_image.storage and repo_image.storage.uuid
 
+  profile.debug('Looking up repo layer data')
   try:
     data = store.get_content(store.image_json_path(namespace, repository,
                                                    image_id, uuid))
   except IOError:
     flask_abort(404)
 
+  profile.debug('Looking up repo layer size')
   try:
-    size = store.get_size(store.image_layer_path(namespace, repository,
-                                                 image_id, uuid))
+    size = repo_image.image_size or repo_image.storage.image_size
     headers['X-Docker-Size'] = str(size)
   except OSError:
     pass
 
+  profile.debug('Retrieving checksum')
   checksum_path = store.image_checksum_path(namespace, repository, image_id,
                                             uuid)
   if store.exists(checksum_path):
@@ -284,14 +331,17 @@ def get_image_json(namespace, repository, image_id, headers):
 @require_completion
 @set_cache_headers
 def get_image_ancestry(namespace, repository, image_id, headers):
+  profile.debug('Checking repo permissions')
   permission = ReadRepositoryPermission(namespace, repository)
   if not permission.can() and not model.repository_is_public(namespace,
                                                              repository):
     abort(403)
 
+  profile.debug('Looking up repo image')
   repo_image = model.get_repo_image(namespace, repository, image_id)
   uuid = repo_image and repo_image.storage and repo_image.storage.uuid
 
+  profile.debug('Looking up image data')
   try:
     data = store.get_content(store.image_ancestry_path(namespace, repository,
                                                        image_id, uuid))
@@ -299,8 +349,11 @@ def get_image_ancestry(namespace, repository, image_id, headers):
     abort(404, 'Image %(image_id)s not found', issue='unknown-image',
           image_id=image_id)
 
+  profile.debug('Converting to <-> from JSON')
   response = make_response(json.dumps(json.loads(data)), 200)
   response.headers.extend(headers)
+
+  profile.debug('Done')
   return response
 
 
@@ -335,10 +388,12 @@ def store_checksum(namespace, repository, image_id, uuid, checksum):
 @process_auth
 @extract_namespace_repo_from_session
 def put_image_json(namespace, repository, image_id):
+  profile.debug('Checking repo permissions')
   permission = ModifyRepositoryPermission(namespace, repository)
   if not permission.can():
     abort(403)
 
+  profile.debug('Parsing image JSON')
   try:
     data = json.loads(request.data)
   except json.JSONDecodeError:
@@ -351,6 +406,7 @@ def put_image_json(namespace, repository, image_id):
     abort(400, 'Missing key `id` in JSON for image: %(image_id)s',
           issue='invalid-request', image_id=image_id)
 
+  profile.debug('Looking up repo image')
   repo_image = model.get_repo_image(namespace, repository, image_id)
   uuid = repo_image and repo_image.storage and repo_image.storage.uuid
 
@@ -358,12 +414,14 @@ def put_image_json(namespace, repository, image_id):
   checksum = request.headers.get('X-Docker-Checksum')
   if checksum:
     # Storing the checksum is optional at this stage
+    profile.debug('Storing image checksum')
     err = store_checksum(namespace, repository, image_id, uuid, checksum)
     if err:
       abort(400, err, issue='write-error')
 
   else:
     # We cleanup any old checksum in case it's a retry after a fail
+    profile.debug('Cleanup old checksum')
     store.remove(store.image_checksum_path(namespace, repository, image_id,
                                            uuid))
   if image_id != data['id']:
@@ -374,19 +432,27 @@ def put_image_json(namespace, repository, image_id):
 
   parent_image = None
   if parent_id:
+    profile.debug('Looking up parent image')
     parent_image = model.get_repo_image(namespace, repository, parent_id)
+
   parent_uuid = (parent_image and parent_image.storage and
                  parent_image.storage.uuid)
 
+  if parent_id:
+    profile.debug('Looking up parent image data')
+
   if (parent_id and not
       store.exists(store.image_json_path(namespace, repository, parent_id,
                                          parent_uuid))):
     abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s',
           issue='invalid-request', image_id=image_id, parent_id=parent_id)
 
+  profile.debug('Looking up image storage paths')
   json_path = store.image_json_path(namespace, repository, image_id, uuid)
-  mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
-  if store.exists(json_path) and not store.exists(mark_path):
+
+  profile.debug('Checking if image already exists')
+  if (store.exists(json_path) and not
+      image_is_uploading(namespace, repository, image_id, repo_image)):
     abort(409, 'Image already exists', issue='image-exists', image_id=image_id)
 
   # If we reach that point, it means that this is a new image or a retry
@@ -394,13 +460,20 @@ def put_image_json(namespace, repository, image_id):
   # save the metadata
   command_list = data.get('container_config', {}).get('Cmd', None)
   command = json.dumps(command_list) if command_list else None
+
+  profile.debug('Setting image metadata')
   model.set_image_metadata(image_id, namespace, repository,
                            data.get('created'), data.get('comment'), command,
                            parent_image)
-  store.put_content(mark_path, 'true')
+
+  profile.debug('Putting json path')
   store.put_content(json_path, request.data)
+
+  profile.debug('Generating image ancestry')
   generate_ancestry(namespace, repository, image_id, uuid, parent_id,
                     parent_uuid)
+
+  profile.debug('Done')
   return make_response('true', 200)
 
 

endpoints/trigger.py

@@ -20,6 +20,10 @@ TARBALL_MIME = 'application/gzip'
 CHUNK_SIZE = 512 * 1024
 
 
+def should_skip_commit(message):
+  return '[skip build]' in message or '[build skip]' in message
+
+
 class BuildArchiveException(Exception):
   pass
 
@@ -35,6 +39,9 @@ class TriggerDeactivationException(Exception):
 class ValidationRequestException(Exception):
   pass
 
+class SkipRequestException(Exception):
+  pass
+
 class EmptyRepositoryException(Exception):
   pass
 
@@ -308,13 +315,20 @@ class GithubBuildTrigger(BuildTrigger):
 
   def handle_trigger_request(self, request, auth_token, config):
     payload = request.get_json()
-
+    if not payload:
+      raise SkipRequestException()      
+    
     if 'zen' in payload:
       raise ValidationRequestException()
 
     logger.debug('Payload %s', payload)
     ref = payload['ref']
     commit_sha = payload['head_commit']['id']
+    commit_message = payload['head_commit'].get('message', '')
+
+    if should_skip_commit(commit_message):
+      raise SkipRequestException() 
+
     short_sha = GithubBuildTrigger.get_display_name(commit_sha)
 
     gh_client = self._get_client(auth_token)

endpoints/webhooks.py

@@ -11,7 +11,7 @@ from util.invoice import renderInvoiceToHtml
 from util.email import send_invoice_email, send_subscription_change, send_payment_failed
 from util.names import parse_repository_name
 from util.http import abort
-from endpoints.trigger import BuildTrigger, ValidationRequestException
+from endpoints.trigger import BuildTrigger, ValidationRequestException, SkipRequestException
 from endpoints.common import start_build
 
 
@@ -30,7 +30,7 @@ def stripe_webhook():
 
   event_type = request_data['type'] if 'type' in request_data else None
   if event_type == 'charge.succeeded':
-    invoice_id = ['data']['object']['invoice']
+    invoice_id = request_data['data']['object']['invoice']
 
     if user and user.invoice_email:
       # Lookup the invoice.
@@ -94,6 +94,10 @@ def build_trigger_webhook(namespace, repository, trigger_uuid):
       # This was just a validation request, we don't need to build anything
       return make_response('Okay')
 
+    except SkipRequestException:
+      # The build was requested to be skipped
+      return make_response('Okay')
+
     pull_robot_name = model.get_pull_robot_name(trigger)
     repo = model.get_repository(namespace, repository)
     start_build(repo, dockerfile_id, tags, name, subdir, False, trigger,