diff --git a/endpoints/trigger.py b/endpoints/trigger.py index 873f51c9b..d5573a513 100644 --- a/endpoints/trigger.py +++ b/endpoints/trigger.py @@ -1,7 +1,7 @@ import logging import io import os.path -import zipfile +import tarfile from github import Github, UnknownObjectException, GithubException from tempfile import SpooledTemporaryFile @@ -16,7 +16,7 @@ client = app.config['HTTPCLIENT'] logger = logging.getLogger(__name__) -ZIPBALL = 'application/zip' +TARBALL_MIME = 'application/gzip' CHUNK_SIZE = 512 * 1024 @@ -222,19 +222,22 @@ class GithubBuildTrigger(BuildTrigger): @staticmethod def _prepare_build(config, repo, commit_sha, build_name, ref): # Prepare the download and upload URLs - archive_link = repo.get_archive_link('zipball', commit_sha) + archive_link = repo.get_archive_link('tarball', commit_sha) download_archive = client.get(archive_link, stream=True) - zipball_subdir = '' - with SpooledTemporaryFile(CHUNK_SIZE) as zipball: + tarball_subdir = '' + with SpooledTemporaryFile(CHUNK_SIZE) as tarball: for chunk in download_archive.iter_content(CHUNK_SIZE): - zipball.write(chunk) + tarball.write(chunk) + + # Seek to position 0 to make tarfile happy + tarball.seek(0) # Pull out the name of the subdir that GitHub generated - with zipfile.ZipFile(zipball) as archive: - zipball_subdir = archive.namelist()[0] + with tarfile.open(fileobj=tarball) as archive: + tarball_subdir = archive.getnames()[0] - dockerfile_id = user_files.store_file(zipball, ZIPBALL) + dockerfile_id = user_files.store_file(tarball, TARBALL_MIME) logger.debug('Successfully prepared job') @@ -247,7 +250,7 @@ class GithubBuildTrigger(BuildTrigger): # compute the subdir repo_subdir = config['subdir'] - joined_subdir = os.path.join(zipball_subdir, repo_subdir) + joined_subdir = os.path.join(tarball_subdir, repo_subdir) logger.debug('Final subdir: %s' % joined_subdir) return dockerfile_id, list(tags), build_name, joined_subdir @@ -280,7 +283,6 @@ class GithubBuildTrigger(BuildTrigger): def manual_start(self, auth_token, config): source = config['build_source'] - subdir = config['subdir'] gh_client = self._get_client(auth_token) repo = gh_client.get_repo(source) diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 7e75fc429..f44642ff2 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -6,6 +6,7 @@ import requests import re import json import shutil +import tarfile from docker import Client, APIError from tempfile import TemporaryFile, mkdtemp @@ -290,6 +291,8 @@ class DockerfileBuildWorker(Worker): 'application/x-zip-compressed': DockerfileBuildWorker.__prepare_zip, 'text/plain': DockerfileBuildWorker.__prepare_dockerfile, 'application/octet-stream': DockerfileBuildWorker.__prepare_dockerfile, + 'application/x-tar': DockerfileBuildWorker.__prepare_tarball, + 'application/gzip': DockerfileBuildWorker.__prepare_tarball, } @staticmethod @@ -298,7 +301,7 @@ class DockerfileBuildWorker(Worker): # Save the zip file to temp somewhere with TemporaryFile() as zip_file: - zip_file.write(request_file.content) + zip_file.write(request_file.raw) to_extract = ZipFile(zip_file) to_extract.extractall(build_dir) @@ -313,6 +316,16 @@ class DockerfileBuildWorker(Worker): return build_dir + @staticmethod + def __prepare_tarball(request_file): + build_dir = mkdtemp(prefix='docker-build-') + + # Save the zip file to temp somewhere + with tarfile.open(mode='r|*', fileobj=request_file.raw) as tar_stream: + tar_stream.extractall(build_dir) + + return build_dir + def process_queue_item(self, job_details): repository_build = model.get_repository_build(job_details['namespace'], job_details['repository'], @@ -336,7 +349,7 @@ class DockerfileBuildWorker(Worker): repo)) log_appender(start_msg) - docker_resource = requests.get(resource_url) + docker_resource = requests.get(resource_url, stream=True) c_type = docker_resource.headers['content-type'] filetype_msg = ('Request to build type: %s with repo: %s and tags: %s' %