Add tarball support to the builder and pull github code as a tarball.

endpoints/trigger.py

@@ -1,7 +1,7 @@
 import logging
 import io
 import os.path
-import zipfile
+import tarfile
 
 from github import Github, UnknownObjectException, GithubException
 from tempfile import SpooledTemporaryFile
@@ -16,7 +16,7 @@ client = app.config['HTTPCLIENT']
 logger = logging.getLogger(__name__)
 
 
-ZIPBALL = 'application/zip'
+TARBALL_MIME = 'application/gzip'
 CHUNK_SIZE = 512 * 1024
 
 
@@ -222,19 +222,22 @@ class GithubBuildTrigger(BuildTrigger):
   @staticmethod
   def _prepare_build(config, repo, commit_sha, build_name, ref):
     # Prepare the download and upload URLs
-    archive_link = repo.get_archive_link('zipball', commit_sha)
+    archive_link = repo.get_archive_link('tarball', commit_sha)
     download_archive = client.get(archive_link, stream=True)
 
-    zipball_subdir = ''
+    tarball_subdir = ''
-    with SpooledTemporaryFile(CHUNK_SIZE) as zipball:
+    with SpooledTemporaryFile(CHUNK_SIZE) as tarball:
       for chunk in download_archive.iter_content(CHUNK_SIZE):
-        zipball.write(chunk)
+        tarball.write(chunk)
+
+      # Seek to position 0 to make tarfile happy
+      tarball.seek(0)
 
       # Pull out the name of the subdir that GitHub generated
-      with zipfile.ZipFile(zipball) as archive:
+      with tarfile.open(fileobj=tarball) as archive:
-        zipball_subdir = archive.namelist()[0]
+        tarball_subdir = archive.getnames()[0]
 
-      dockerfile_id = user_files.store_file(zipball, ZIPBALL)
+      dockerfile_id = user_files.store_file(tarball, TARBALL_MIME)
 
     logger.debug('Successfully prepared job')
 
@@ -247,7 +250,7 @@ class GithubBuildTrigger(BuildTrigger):
 
     # compute the subdir
     repo_subdir = config['subdir']
-    joined_subdir = os.path.join(zipball_subdir, repo_subdir)
+    joined_subdir = os.path.join(tarball_subdir, repo_subdir)
     logger.debug('Final subdir: %s' % joined_subdir)
 
     return dockerfile_id, list(tags), build_name, joined_subdir
@@ -280,7 +283,6 @@ class GithubBuildTrigger(BuildTrigger):
 
   def manual_start(self, auth_token, config):
     source = config['build_source']
-    subdir = config['subdir']
 
     gh_client = self._get_client(auth_token)
     repo = gh_client.get_repo(source)

workers/dockerfilebuild.py

@@ -6,6 +6,7 @@ import requests
 import re
 import json
 import shutil
+import tarfile
 
 from docker import Client, APIError
 from tempfile import TemporaryFile, mkdtemp
@@ -290,6 +291,8 @@ class DockerfileBuildWorker(Worker):
       'application/x-zip-compressed': DockerfileBuildWorker.__prepare_zip,
       'text/plain': DockerfileBuildWorker.__prepare_dockerfile,
       'application/octet-stream': DockerfileBuildWorker.__prepare_dockerfile,
+      'application/x-tar': DockerfileBuildWorker.__prepare_tarball,
+      'application/gzip': DockerfileBuildWorker.__prepare_tarball,
     }
 
   @staticmethod
@@ -298,7 +301,7 @@ class DockerfileBuildWorker(Worker):
 
     # Save the zip file to temp somewhere
     with TemporaryFile() as zip_file:
-      zip_file.write(request_file.content)
+      zip_file.write(request_file.raw)
       to_extract = ZipFile(zip_file)
       to_extract.extractall(build_dir)
 
@@ -313,6 +316,16 @@ class DockerfileBuildWorker(Worker):
 
     return build_dir
 
+  @staticmethod
+  def __prepare_tarball(request_file):
+    build_dir = mkdtemp(prefix='docker-build-')
+
+    # Save the zip file to temp somewhere
+    with tarfile.open(mode='r|*', fileobj=request_file.raw) as tar_stream:
+      tar_stream.extractall(build_dir)
+
+    return build_dir
+
   def process_queue_item(self, job_details):
     repository_build = model.get_repository_build(job_details['namespace'],
                                                   job_details['repository'],
@@ -336,7 +349,7 @@ class DockerfileBuildWorker(Worker):
                                                                   repo))
     log_appender(start_msg)
 
-    docker_resource = requests.get(resource_url)
+    docker_resource = requests.get(resource_url, stream=True)
     c_type = docker_resource.headers['content-type']
 
     filetype_msg = ('Request to build type: %s with repo: %s and tags: %s' %