From f092c006215c952c671dc8d161b3512e35818495 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 14 Aug 2015 17:22:19 -0400 Subject: [PATCH] Allow builds to be started with an external archive URL Fixes #114 --- buildman/component/buildcomponent.py | 6 +-- buildman/jobutil/buildjob.py | 11 ++++++ endpoints/api/build.py | 55 ++++++++++++++++++++++------ endpoints/building.py | 15 +++++++- requirements-nover.txt | 1 + requirements.txt | 1 + test/test_api_usage.py | 51 +++++++++++++++++++++++++- 7 files changed, 120 insertions(+), 20 deletions(-) diff --git a/buildman/component/buildcomponent.py b/buildman/component/buildcomponent.py index a728020dd..a567910c6 100644 --- a/buildman/component/buildcomponent.py +++ b/buildman/component/buildcomponent.py @@ -116,14 +116,10 @@ class BuildComponent(BaseComponent): # push_token: The token to use to push the built image. # tag_names: The name(s) of the tag(s) for the newly built image. # base_image: The image name and credentials to use to conduct the base image pull. - # repository: The repository to pull (DEPRECATED 0.2) - # tag: The tag to pull (DEPRECATED in 0.2) # username: The username for pulling the base image (if any). # password: The password for pulling the base image (if any). build_arguments = { - 'build_package': self.user_files.get_file_url(build_job.repo_build.resource_key, - requires_cors=False) - if build_job.repo_build.resource_key is not None else "", + 'build_package': build_job.get_build_package_url(self.user_files), 'sub_directory': build_config.get('build_subdir', ''), 'repository': repository_name, 'registry': self.registry_hostname, diff --git a/buildman/jobutil/buildjob.py b/buildman/jobutil/buildjob.py index 5635f0622..f6291f62b 100644 --- a/buildman/jobutil/buildjob.py +++ b/buildman/jobutil/buildjob.py @@ -62,6 +62,17 @@ class BuildJob(object): def repo_build(self): return self._load_repo_build() + def get_build_package_url(self, user_files): + """ Returns the URL of the build package for this build, if any or empty string if none. """ + archive_url = self.build_config.get('archive_url', None) + if archive_url: + return archive_url + + if not self.repo_build.resource_key: + return '' + + return user_files.get_file_url(self.repo_build.resource_key, requires_cors=False) + @property def pull_credentials(self): """ Returns the pull credentials for this job, or None if none. """ diff --git a/endpoints/api/build.py b/endpoints/api/build.py index 21002f4a3..397e40ac2 100644 --- a/endpoints/api/build.py +++ b/endpoints/api/build.py @@ -3,8 +3,10 @@ import logging import json import datetime +import hashlib from flask import request +from rfc3987 import parse as uri_parse from app import app, userfiles as user_files, build_logs, log_archive, dockerfile_build_queue from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource, @@ -134,8 +136,11 @@ def build_status_view(build_obj): } } - if can_write and build_obj.resource_key is not None: - resp['archive_url'] = user_files.get_file_url(build_obj.resource_key, requires_cors=True) + if can_write: + if build_obj.resource_key is not None: + resp['archive_url'] = user_files.get_file_url(build_obj.resource_key, requires_cors=True) + elif job_config.get('archive_url', None): + resp['archive_url'] = job_config['archive_url'] return resp @@ -148,14 +153,15 @@ class RepositoryBuildList(RepositoryParamResource): 'RepositoryBuildRequest': { 'type': 'object', 'description': 'Description of a new repository build.', - 'required': [ - 'file_id', - ], 'properties': { 'file_id': { 'type': 'string', 'description': 'The file id that was generated when the build spec was uploaded', }, + 'archive_url': { + 'type': 'string', + 'description': 'The URL of the .tar.gz to build. Must start with "http" or "https".', + }, 'subdirectory': { 'type': 'string', 'description': 'Subdirectory in which the Dockerfile can be found', @@ -204,7 +210,26 @@ class RepositoryBuildList(RepositoryParamResource): logger.debug('User requested repository initialization.') request_json = request.get_json() - dockerfile_id = request_json['file_id'] + dockerfile_id = request_json.get('file_id', None) + archive_url = request_json.get('archive_url', None) + + if not dockerfile_id and not archive_url: + raise InvalidRequest('file_id or archive_url required') + + if archive_url: + archive_match = None + try: + archive_match = uri_parse(archive_url, 'URI') + except ValueError: + pass + + if not archive_match: + raise InvalidRequest('Invalid Archive URL: Must be a valid URI') + + scheme = archive_match.get('scheme', None) + if scheme != 'http' and scheme != 'https': + raise InvalidRequest('Invalid Archive URL: Must be http or https') + subdir = request_json['subdirectory'] if 'subdirectory' in request_json else '' tags = request_json.get('docker_tags', ['latest']) pull_robot_name = request_json.get('pull_robot', None) @@ -228,18 +253,24 @@ class RepositoryBuildList(RepositoryParamResource): # Check if the dockerfile resource has already been used. If so, then it # can only be reused if the user has access to the repository in which the # dockerfile was previously built. - associated_repository = model.build.get_repository_for_resource(dockerfile_id) - if associated_repository: - if not ModifyRepositoryPermission(associated_repository.namespace_user.username, - associated_repository.name): - raise Unauthorized() + if dockerfile_id: + associated_repository = model.build.get_repository_for_resource(dockerfile_id) + if associated_repository: + if not ModifyRepositoryPermission(associated_repository.namespace_user.username, + associated_repository.name): + raise Unauthorized() # Start the build. repo = model.repository.get_repository(namespace, repository) + build_name = (user_files.get_file_checksum(dockerfile_id) + if dockerfile_id + else hashlib.sha224(archive_url).hexdigest()[0:7]) + prepared = PreparedBuild() - prepared.build_name = user_files.get_file_checksum(dockerfile_id) + prepared.build_name = build_name prepared.dockerfile_id = dockerfile_id + prepared.archive_url = archive_url prepared.tags = tags prepared.subdirectory = subdir prepared.is_manual = True diff --git a/endpoints/building.py b/endpoints/building.py index 02d89a323..93d76be7b 100644 --- a/endpoints/building.py +++ b/endpoints/building.py @@ -28,7 +28,8 @@ def start_build(repository, prepared_build, pull_robot_name=None): 'build_subdir': prepared_build.subdirectory, 'trigger_metadata': prepared_build.metadata or {}, 'is_manual': prepared_build.is_manual, - 'manual_user': get_authenticated_user().username if get_authenticated_user() else None + 'manual_user': get_authenticated_user().username if get_authenticated_user() else None, + 'archive_url': prepared_build.archive_url } with app.config['DB_TRANSACTION_FACTORY'](db): @@ -83,6 +84,7 @@ class PreparedBuild(object): """ def __init__(self, trigger=None): self._dockerfile_id = None + self._archive_url = None self._tags = None self._build_name = None self._subdirectory = None @@ -124,6 +126,17 @@ class PreparedBuild(object): def trigger(self): return self._trigger + @property + def archive_url(self): + return self._archive_url + + @archive_url.setter + def archive_url(self, value): + if self._archive_url: + raise Exception('Property archive_url already set') + + self._archive_url = value + @property def dockerfile_id(self): return self._dockerfile_id diff --git a/requirements-nover.txt b/requirements-nover.txt index 6fe50fc80..5eb9dff3c 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -52,3 +52,4 @@ python-keystoneclient Flask-Testing pyjwt toposort +rfc3987 diff --git a/requirements.txt b/requirements.txt index 1061d96e9..40828e1f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -73,6 +73,7 @@ redis==2.10.3 reportlab==2.7 requests==2.7.0 requests-oauthlib==0.5.0 +rfc3987==1.3.4 simplejson==3.7.3 six==1.9.0 SQLAlchemy==1.0.6 diff --git a/test/test_api_usage.py b/test/test_api_usage.py index eadade1b7..a982ec847 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -1759,8 +1759,55 @@ class TestRepoBuilds(ApiTestCase): self.assertEquals(status_json['resource_key'], build['resource_key']) self.assertEquals(status_json['trigger'], build['trigger']) + class TestRequestRepoBuild(ApiTestCase): - def test_requestrepobuild(self): + def test_requestbuild_noidurl(self): + self.login(ADMIN_ACCESS_USER) + + # Request a (fake) build without a file ID or URL. + self.postResponse(RepositoryBuildList, + params=dict(repository=ADMIN_ACCESS_USER + '/simple'), + data=dict(), + expected_code=400) + + def test_requestbuild_invalidurls(self): + self.login(ADMIN_ACCESS_USER) + + # Request a (fake) build with and invalid URL. + self.postResponse(RepositoryBuildList, + params=dict(repository=ADMIN_ACCESS_USER + '/simple'), + data=dict(archive_url='foobarbaz'), + expected_code=400) + + self.postResponse(RepositoryBuildList, + params=dict(repository=ADMIN_ACCESS_USER + '/simple'), + data=dict(archive_url='file://foobarbaz'), + expected_code=400) + + def test_requestrepobuild_withurl(self): + self.login(ADMIN_ACCESS_USER) + + # Ensure we are not yet building. + json = self.getJsonResponse(RepositoryBuildList, + params=dict(repository=ADMIN_ACCESS_USER + '/simple')) + + assert len(json['builds']) == 0 + + # Request a (fake) build. + self.postResponse(RepositoryBuildList, + params=dict(repository=ADMIN_ACCESS_USER + '/simple'), + data=dict(archive_url='http://quay.io/robots.txt'), + expected_code=201) + + # Check for the build. + json = self.getJsonResponse(RepositoryBuildList, + params=dict(repository=ADMIN_ACCESS_USER + '/simple')) + + assert len(json['builds']) > 0 + self.assertEquals('http://quay.io/robots.txt', json['builds'][0]['archive_url']) + + + def test_requestrepobuild_withfile(self): self.login(ADMIN_ACCESS_USER) # Ensure we are not yet building. @@ -1777,7 +1824,7 @@ class TestRequestRepoBuild(ApiTestCase): # Check for the build. json = self.getJsonResponse(RepositoryBuildList, - params=dict(repository=ADMIN_ACCESS_USER + '/building')) + params=dict(repository=ADMIN_ACCESS_USER + '/simple')) assert len(json['builds']) > 0