Merge remote-tracking branch 'origin/master' into tagyourit

Conflicts:
	test/data/test.db

endpoints/api/build.py

@@ -11,6 +11,7 @@ from endpoints.common import start_build
 from endpoints.trigger import BuildTrigger
 from data import model
 from auth.permissions import ModifyRepositoryPermission
+from data.buildlogs import BuildStatusRetrievalError
 
 
 logger = logging.getLogger(__name__)
@@ -48,11 +49,17 @@ def trigger_view(trigger):
 
 
 def build_status_view(build_obj, can_write=False):
-  status = build_logs.get_status(build_obj.uuid)
+  phase = build_obj.phase
+  try:
+    status = build_logs.get_status(build_obj.uuid)
+  except BuildStatusRetrievalError:
+    status = {}
+    phase = 'cannot_load'
+
   logger.debug('Can write: %s job_config: %s', can_write, build_obj.job_config)
   resp = {
     'id': build_obj.uuid,
-    'phase': build_obj.phase if status else 'cannot_load',
+    'phase': phase,
     'started': format_date(build_obj.started),
     'display_name': build_obj.display_name,
     'status': status or {},
@@ -169,7 +176,10 @@ class RepositoryBuildLogs(RepositoryParamResource):
 
     start = int(request.args.get('start', 0))
 
-    count, logs = build_logs.get_log_entries(build.uuid, start)
+    try:
+      count, logs = build_logs.get_log_entries(build.uuid, start)
+    except BuildStatusRetrievalError:
+      count, logs = (0, [])
 
     response_obj.update({
       'start': start,

endpoints/api/trigger.py

@@ -13,7 +13,8 @@ from endpoints.api.build import (build_status_view, trigger_view, RepositoryBuil
                                  get_trigger_config)
 from endpoints.common import start_build
 from endpoints.trigger import (BuildTrigger as BuildTriggerBase, TriggerDeactivationException,
-                               TriggerActivationException, EmptyRepositoryException)
+                               TriggerActivationException, EmptyRepositoryException,
+                               RepositoryReadException)
 from data import model
 from auth.permissions import UserAdminPermission
 
@@ -116,9 +117,14 @@ class BuildTriggerSubdirs(RepositoryParamResource):
           'status': 'success'
         }
       except EmptyRepositoryException as exc:
+        return {
+          'status': 'success',
+          'subdir': []
+        }
+      except RepositoryReadException as exc:
         return {
           'status': 'error',
-          'message': exc.msg
+          'message': exc.message
         }
     else:
       raise Unauthorized()

endpoints/common.py

@@ -91,7 +91,7 @@ def random_string():
 
 def render_page_template(name, **kwargs):
   resp = make_response(render_template(name, route_data=json.dumps(get_route_data()),
-                       cache_buster=random_string(), **kwargs))
+                       cache_buster='foobarbaz', **kwargs))
   resp.headers['X-FRAME-OPTIONS'] = 'DENY'
   return resp
 

endpoints/trigger.py

@@ -1,7 +1,7 @@
 import logging
 import io
 import os.path
-import zipfile
+import tarfile
 
 from github import Github, UnknownObjectException, GithubException
 from tempfile import SpooledTemporaryFile
@@ -16,7 +16,7 @@ client = app.config['HTTPCLIENT']
 logger = logging.getLogger(__name__)
 
 
-ZIPBALL = 'application/zip'
+TARBALL_MIME = 'application/gzip'
 CHUNK_SIZE = 512 * 1024
 
 
@@ -38,6 +38,9 @@ class ValidationRequestException(Exception):
 class EmptyRepositoryException(Exception):
   pass
 
+class RepositoryReadException(Exception):
+  pass
+
 
 class BuildTrigger(object):
   def __init__(self):
@@ -209,26 +212,32 @@ class GithubBuildTrigger(BuildTrigger):
       return [os.path.dirname(elem.path) for elem in commit_tree.tree
               if (elem.type == u'blob' and
                   os.path.basename(elem.path) == u'Dockerfile')]
-    except GithubException:
-      msg = 'Unable to list contents of repository: %s' % source
-      raise EmptyRepositoryException(msg)
+    except GithubException as ge:
+      message = ge.data.get('message', 'Unable to list contents of repository: %s' % source)
+      if message == 'Branch not found':
+        raise EmptyRepositoryException()
+
+      raise RepositoryReadException(message)
 
   @staticmethod
   def _prepare_build(config, repo, commit_sha, build_name, ref):
     # Prepare the download and upload URLs
-    archive_link = repo.get_archive_link('zipball', commit_sha)
+    archive_link = repo.get_archive_link('tarball', commit_sha)
     download_archive = client.get(archive_link, stream=True)
 
-    zipball_subdir = ''
-    with SpooledTemporaryFile(CHUNK_SIZE) as zipball:
+    tarball_subdir = ''
+    with SpooledTemporaryFile(CHUNK_SIZE) as tarball:
       for chunk in download_archive.iter_content(CHUNK_SIZE):
-        zipball.write(chunk)
+        tarball.write(chunk)
+
+      # Seek to position 0 to make tarfile happy
+      tarball.seek(0)
 
       # Pull out the name of the subdir that GitHub generated
-      with zipfile.ZipFile(zipball) as archive:
-        zipball_subdir = archive.namelist()[0]
+      with tarfile.open(fileobj=tarball) as archive:
+        tarball_subdir = archive.getnames()[0]
 
-      dockerfile_id = user_files.store_file(zipball, ZIPBALL)
+      dockerfile_id = user_files.store_file(tarball, TARBALL_MIME)
 
     logger.debug('Successfully prepared job')
 
@@ -241,7 +250,7 @@ class GithubBuildTrigger(BuildTrigger):
 
     # compute the subdir
     repo_subdir = config['subdir']
-    joined_subdir = os.path.join(zipball_subdir, repo_subdir)
+    joined_subdir = os.path.join(tarball_subdir, repo_subdir)
     logger.debug('Final subdir: %s' % joined_subdir)
 
     return dockerfile_id, list(tags), build_name, joined_subdir
@@ -274,7 +283,6 @@ class GithubBuildTrigger(BuildTrigger):
 
   def manual_start(self, auth_token, config):
     source = config['build_source']
-    subdir = config['subdir']
 
     gh_client = self._get_client(auth_token)
     repo = gh_client.get_repo(source)