Allow builds to be started with an external archive URL

Fixes #114

buildman/component/buildcomponent.py

@@ -116,14 +116,10 @@ class BuildComponent(BaseComponent):
     #  push_token: The token to use to push the built image.
     #  tag_names: The name(s) of the tag(s) for the newly built image.
     #  base_image: The image name and credentials to use to conduct the base image pull.
-    #   repository: The repository to pull (DEPRECATED 0.2)
-    #   tag: The tag to pull (DEPRECATED in 0.2)
     #   username: The username for pulling the base image (if any).
     #   password: The password for pulling the base image (if any).
     build_arguments = {
-      'build_package': self.user_files.get_file_url(build_job.repo_build.resource_key,
+      'build_package': build_job.get_build_package_url(self.user_files),
-                                                    requires_cors=False)
-                       if build_job.repo_build.resource_key is not None else "",
       'sub_directory': build_config.get('build_subdir', ''),
       'repository': repository_name,
       'registry': self.registry_hostname,

buildman/jobutil/buildjob.py

@@ -62,6 +62,17 @@ class BuildJob(object):
   def repo_build(self):
     return self._load_repo_build()
 
+  def get_build_package_url(self, user_files):
+    """ Returns the URL of the build package for this build, if any or empty string if none. """
+    archive_url = self.build_config.get('archive_url', None)
+    if archive_url:
+      return archive_url
+
+    if not self.repo_build.resource_key:
+      return ''
+
+    return user_files.get_file_url(self.repo_build.resource_key, requires_cors=False)
+
   @property
   def pull_credentials(self):
     """ Returns the pull credentials for this job, or None if none. """

endpoints/api/build.py

@@ -3,8 +3,10 @@
 import logging
 import json
 import datetime
+import hashlib
 
 from flask import request
+from rfc3987 import parse as uri_parse
 
 from app import app, userfiles as user_files, build_logs, log_archive, dockerfile_build_queue
 from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource,
@@ -134,8 +136,11 @@ def build_status_view(build_obj):
     }
   }
 
-  if can_write and build_obj.resource_key is not None:
+  if can_write:
-    resp['archive_url'] = user_files.get_file_url(build_obj.resource_key, requires_cors=True)
+    if build_obj.resource_key is not None:
+      resp['archive_url'] = user_files.get_file_url(build_obj.resource_key, requires_cors=True)
+    elif job_config.get('archive_url', None):
+      resp['archive_url'] = job_config['archive_url']
 
   return resp
 
@@ -148,14 +153,15 @@ class RepositoryBuildList(RepositoryParamResource):
     'RepositoryBuildRequest': {
       'type': 'object',
       'description': 'Description of a new repository build.',
-      'required': [
-        'file_id',
-      ],
       'properties': {
         'file_id': {
           'type': 'string',
           'description': 'The file id that was generated when the build spec was uploaded',
         },
+        'archive_url': {
+          'type': 'string',
+          'description': 'The URL of the .tar.gz to build. Must start with "http" or "https".',
+        },
         'subdirectory': {
           'type': 'string',
           'description': 'Subdirectory in which the Dockerfile can be found',
@@ -204,7 +210,26 @@ class RepositoryBuildList(RepositoryParamResource):
     logger.debug('User requested repository initialization.')
     request_json = request.get_json()
 
-    dockerfile_id = request_json['file_id']
+    dockerfile_id = request_json.get('file_id', None)
+    archive_url = request_json.get('archive_url', None)
+
+    if not dockerfile_id and not archive_url:
+      raise InvalidRequest('file_id or archive_url required')
+
+    if archive_url:
+      archive_match = None
+      try:
+        archive_match = uri_parse(archive_url, 'URI')
+      except ValueError:
+        pass
+
+      if not archive_match:
+        raise InvalidRequest('Invalid Archive URL: Must be a valid URI')
+
+      scheme = archive_match.get('scheme', None)
+      if scheme != 'http' and scheme != 'https':
+        raise InvalidRequest('Invalid Archive URL: Must be http or https')
+
     subdir = request_json['subdirectory'] if 'subdirectory' in request_json else ''
     tags = request_json.get('docker_tags', ['latest'])
     pull_robot_name = request_json.get('pull_robot', None)
@@ -228,18 +253,24 @@ class RepositoryBuildList(RepositoryParamResource):
     # Check if the dockerfile resource has already been used. If so, then it
     # can only be reused if the user has access to the repository in which the
     # dockerfile was previously built.
-    associated_repository = model.build.get_repository_for_resource(dockerfile_id)
+    if dockerfile_id:
-    if associated_repository:
+      associated_repository = model.build.get_repository_for_resource(dockerfile_id)
-      if not ModifyRepositoryPermission(associated_repository.namespace_user.username,
+      if associated_repository:
-                                        associated_repository.name):
+        if not ModifyRepositoryPermission(associated_repository.namespace_user.username,
-        raise Unauthorized()
+                                          associated_repository.name):
+          raise Unauthorized()
 
     # Start the build.
     repo = model.repository.get_repository(namespace, repository)
 
+    build_name =  (user_files.get_file_checksum(dockerfile_id)
+                    if dockerfile_id
+                    else hashlib.sha224(archive_url).hexdigest()[0:7])
+
     prepared = PreparedBuild()
-    prepared.build_name = user_files.get_file_checksum(dockerfile_id)
+    prepared.build_name = build_name
     prepared.dockerfile_id = dockerfile_id
+    prepared.archive_url = archive_url
     prepared.tags = tags
     prepared.subdirectory = subdir
     prepared.is_manual = True

endpoints/building.py

@@ -28,7 +28,8 @@ def start_build(repository, prepared_build, pull_robot_name=None):
     'build_subdir': prepared_build.subdirectory,
     'trigger_metadata': prepared_build.metadata or {},
     'is_manual': prepared_build.is_manual,
-    'manual_user': get_authenticated_user().username if get_authenticated_user() else None
+    'manual_user': get_authenticated_user().username if get_authenticated_user() else None,
+    'archive_url': prepared_build.archive_url
   }
 
   with app.config['DB_TRANSACTION_FACTORY'](db):
@@ -83,6 +84,7 @@ class PreparedBuild(object):
   """
   def __init__(self, trigger=None):
     self._dockerfile_id = None
+    self._archive_url = None
     self._tags = None
     self._build_name = None
     self._subdirectory = None
@@ -124,6 +126,17 @@ class PreparedBuild(object):
   def trigger(self):
     return self._trigger
 
+  @property
+  def archive_url(self):
+    return self._archive_url
+
+  @archive_url.setter
+  def archive_url(self, value):
+    if self._archive_url:
+      raise Exception('Property archive_url already set')
+
+    self._archive_url = value
+
   @property
   def dockerfile_id(self):
     return self._dockerfile_id

requirements-nover.txt

@@ -52,3 +52,4 @@ python-keystoneclient
 Flask-Testing
 pyjwt
 toposort
+rfc3987

requirements.txt

@@ -73,6 +73,7 @@ redis==2.10.3
 reportlab==2.7
 requests==2.7.0
 requests-oauthlib==0.5.0
+rfc3987==1.3.4
 simplejson==3.7.3
 six==1.9.0
 SQLAlchemy==1.0.6

test/test_api_usage.py

@@ -1759,8 +1759,55 @@ class TestRepoBuilds(ApiTestCase):
     self.assertEquals(status_json['resource_key'], build['resource_key'])
     self.assertEquals(status_json['trigger'], build['trigger'])
 
+
 class TestRequestRepoBuild(ApiTestCase):
-  def test_requestrepobuild(self):
+  def test_requestbuild_noidurl(self):
+    self.login(ADMIN_ACCESS_USER)
+
+    # Request a (fake) build without a file ID or URL.
+    self.postResponse(RepositoryBuildList,
+                      params=dict(repository=ADMIN_ACCESS_USER + '/simple'),
+                      data=dict(),
+                      expected_code=400)
+
+  def test_requestbuild_invalidurls(self):
+    self.login(ADMIN_ACCESS_USER)
+
+    # Request a (fake) build with and invalid URL.
+    self.postResponse(RepositoryBuildList,
+                      params=dict(repository=ADMIN_ACCESS_USER + '/simple'),
+                      data=dict(archive_url='foobarbaz'),
+                      expected_code=400)
+
+    self.postResponse(RepositoryBuildList,
+                      params=dict(repository=ADMIN_ACCESS_USER + '/simple'),
+                      data=dict(archive_url='file://foobarbaz'),
+                      expected_code=400)
+
+  def test_requestrepobuild_withurl(self):
+    self.login(ADMIN_ACCESS_USER)
+
+    # Ensure we are not yet building.
+    json = self.getJsonResponse(RepositoryBuildList,
+                                params=dict(repository=ADMIN_ACCESS_USER + '/simple'))
+
+    assert len(json['builds']) == 0
+
+    # Request a (fake) build.
+    self.postResponse(RepositoryBuildList,
+                      params=dict(repository=ADMIN_ACCESS_USER + '/simple'),
+                      data=dict(archive_url='http://quay.io/robots.txt'),
+                      expected_code=201)
+
+    # Check for the build.
+    json = self.getJsonResponse(RepositoryBuildList,
+                                params=dict(repository=ADMIN_ACCESS_USER + '/simple'))
+
+    assert len(json['builds']) > 0
+    self.assertEquals('http://quay.io/robots.txt', json['builds'][0]['archive_url'])
+
+
+  def test_requestrepobuild_withfile(self):
     self.login(ADMIN_ACCESS_USER)
 
     # Ensure we are not yet building.
@@ -1777,7 +1824,7 @@ class TestRequestRepoBuild(ApiTestCase):
 
     # Check for the build.
     json = self.getJsonResponse(RepositoryBuildList,
-                                params=dict(repository=ADMIN_ACCESS_USER + '/building'))
+                                params=dict(repository=ADMIN_ACCESS_USER + '/simple'))
 
     assert len(json['builds']) > 0