diff --git a/data/database.py b/data/database.py index beaa033dc..dd6542106 100644 --- a/data/database.py +++ b/data/database.py @@ -237,8 +237,7 @@ class RepositoryBuild(BaseModel): uuid = CharField(default=uuid_generator, index=True) repository = ForeignKeyField(Repository, index=True) access_token = ForeignKeyField(AccessToken) - resource_key = CharField() - tag = CharField() + job_config = TextField() phase = CharField(default='waiting') started = DateTimeField(default=datetime.now) display_name = CharField() diff --git a/data/model.py b/data/model.py index 767e7ca78..b8c68ce24 100644 --- a/data/model.py +++ b/data/model.py @@ -1409,10 +1409,10 @@ def list_repository_builds(namespace_name, repository_name, return query -def create_repository_build(repo, access_token, resource_key, tag, +def create_repository_build(repo, access_token, job_config_obj, display_name, trigger=None): return RepositoryBuild.create(repository=repo, access_token=access_token, - resource_key=resource_key, tag=tag, + job_config=json.dumps(job_config_obj), display_name=display_name, trigger=trigger) diff --git a/data/queue.py b/data/queue.py index 46db150bf..09e90f1a1 100644 --- a/data/queue.py +++ b/data/queue.py @@ -68,5 +68,5 @@ class WorkQueue(object): image_diff_queue = WorkQueue('imagediff') -dockerfile_build_queue = WorkQueue('dockerfilebuild2') +dockerfile_build_queue = WorkQueue('dockerfilebuild3') webhook_queue = WorkQueue('webhook') diff --git a/endpoints/api.py b/endpoints/api.py index 877a14384..169bb5c7a 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -1144,7 +1144,7 @@ def build_status_view(build_obj, can_write=False): 'started': build_obj.started, 'display_name': build_obj.display_name, 'status': status, - 'resource_key': build_obj.resource_key if can_write else None, + 'job_config': json.loads(build_obj.job_config) if can_write else None, 'is_writer': can_write, 'trigger': trigger_view(build_obj.trigger), } @@ -1234,11 +1234,13 @@ def request_repo_build(namespace, repository): logger.debug('User requested repository initialization.') dockerfile_id = request.get_json()['file_id'] - # Check if the dockerfile resource has already been used. If so, then it can only be reused if the - # user has access to the repository for which it was used. + # Check if the dockerfile resource has already been used. If so, then it + # can only be reused if the user has access to the repository for which it + # was used. associated_repository = model.get_repository_for_resource(dockerfile_id) if associated_repository: - if not ModifyRepositoryPermission(associated_repository.namespace, associated_repository.name): + if not ModifyRepositoryPermission(associated_repository.namespace, + associated_repository.name): abort(403) # Start the build. @@ -1248,9 +1250,15 @@ def request_repo_build(namespace, repository): logger.debug('**********Md5: %s' % display_name) host = urlparse.urlparse(request.url).netloc - tag = '%s/%s/%s' % (host, repo.namespace, repo.name) - build_request = model.create_repository_build(repo, token, dockerfile_id, - tag, display_name) + repo = '%s/%s/%s' % (host, repo.namespace, repo.name) + job_config = { + 'docker_tags': ['latest'], + 'build_subdir': '', + 'repository': repo, + 'resource_key': dockerfile_id, + } + build_request = model.create_repository_build(repo, token, job_config, + display_name) dockerfile_build_queue.put(json.dumps({ 'build_uuid': build_request.uuid, 'namespace': namespace, diff --git a/endpoints/trigger.py b/endpoints/trigger.py index 990fb5278..ab902fd2e 100644 --- a/endpoints/trigger.py +++ b/endpoints/trigger.py @@ -55,7 +55,8 @@ class BuildTrigger(object): def handle_trigger_request(self, request, auth_token, config): """ - Transform the incoming request data into a set of actions. + Transform the incoming request data into a set of actions. Returns a tuple + of usefiles resource id, docker tags, build name, and resource subdir. """ raise NotImplementedError @@ -163,7 +164,7 @@ class GithubBuildTrigger(BuildTrigger): try: repo = gh_client.get_repo(source) - default_commit = repo.get_branch(repo.default_branch).commit + default_commit = repo.get_branch(repo.master_branch).commit commit_tree = repo.get_git_tree(default_commit.sha, recursive=True) return [os.path.dirname(elem.path) for elem in commit_tree.tree @@ -181,7 +182,8 @@ class GithubBuildTrigger(BuildTrigger): logger.debug('Payload %s', payload) ref = payload['ref'] - commit_id = payload['head_commit']['id'][0:7] + commit_sha = payload['head_commit']['id'] + short_sha = commit_sha[0:7] gh_client = self._get_client(auth_token) @@ -192,8 +194,7 @@ class GithubBuildTrigger(BuildTrigger): logger.debug('Github repo: %s', repo) # Prepare the download and upload URLs - branch_name = ref.split('/')[-1] - archive_link = repo.get_archive_link('zipball', branch_name) + archive_link = repo.get_archive_link('zipball', short_sha) download_archive = client.get(archive_link, stream=True) with SpooledTemporaryFile(CHUNK_SIZE) as zipball: @@ -204,4 +205,17 @@ class GithubBuildTrigger(BuildTrigger): logger.debug('Successfully prepared job') - return dockerfile_id, branch_name, commit_id + # compute the tag(s) + pushed_branch = ref.split('/')[-1] + tags = {pushed_branch} + if pushed_branch == repo.master_branch: + tags.add('latest') + logger.debug('Pushing to tags: %s' % tags) + + # compute the subdir + repo_subdir = config['subdir'] + zipball_subdir = '%s-%s-%s' % (repo.owner.login, repo.name, short_sha) + joined_subdir = os.path.join(zipball_subdir, repo_subdir) + logger.debug('Final subdir: %s' % joined_subdir) + + return dockerfile_id, list(tags), short_sha, joined_subdir diff --git a/endpoints/webhooks.py b/endpoints/webhooks.py index 0e424b052..d2beccc2d 100644 --- a/endpoints/webhooks.py +++ b/endpoints/webhooks.py @@ -1,10 +1,12 @@ import logging import stripe import urlparse +import json from flask import request, make_response, Blueprint from data import model +from data.queue import dockerfile_build_queue from auth.auth import process_auth from auth.permissions import ModifyRepositoryPermission from util.invoice import renderInvoiceToHtml @@ -62,22 +64,36 @@ def build_trigger_webhook(namespace, repository, trigger_uuid): logger.debug('Passing webhook request to handler %s', handler) try: - df_id, tag, name = handler.handle_trigger_request(request, - trigger.auth_token, - trigger.config) + specs = handler.handle_trigger_request(request, trigger.auth_token, + json.loads(trigger.config)) + dockerfile_id, tags, name, subdir = specs + except ValidationRequestException: - # This was just a validation request, don't need to build anything + # This was just a validation request, we don't need to build anything return make_response('Okay') host = urlparse.urlparse(request.url).netloc - full_tag = '%s/%s/%s:%s' % (host, trigger.repository.namespace, - trigger.repository.name, tag) + repo = '%s/%s/%s' % (host, trigger.repository.namespace, + trigger.repository.name) token = model.create_access_token(trigger.repository, 'write') - logger.debug('Creating build %s with full_tag %s and dockerfile_id %s', - name, full_tag, df_id) - model.create_repository_build(trigger.repository, token, df_id, full_tag, - name) + logger.debug('Creating build %s with repo %s tags %s and dockerfile_id %s', + name, repo, tags, dockerfile_id) + + job_config = { + 'docker_tags': tags, + 'repository': repo, + 'build_subdir': subdir, + 'resource_key': dockerfile_id, + } + build_request = model.create_repository_build(trigger.repository, token, + job_config, name) + + dockerfile_build_queue.put(json.dumps({ + 'build_uuid': build_request.uuid, + 'namespace': namespace, + 'repository': repository, + }), retries_remaining=1) return make_response('Okay') diff --git a/initdb.py b/initdb.py index 30e56efc5..35c3dd475 100644 --- a/initdb.py +++ b/initdb.py @@ -309,7 +309,6 @@ def populate_database(): False, [], (0, [], None)) token = model.create_access_token(building, 'write') - tag = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name) trigger = model.create_build_trigger(building, 'github', '123authtoken', new_user_1) @@ -318,9 +317,15 @@ def populate_database(): }) trigger.save() - build = model.create_repository_build(building, token, - '701dcc3724fb4f2ea6c31400528343cd', - tag, 'build-name', trigger) + repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name) + job_config = { + 'repository': repo, + 'docker_tags': ['latest'], + 'build_subdir': '', + 'resource_key': '701dcc3724fb4f2ea6c31400528343cd', + } + build = model.create_repository_build(building, token, job_config, + 'build-name', trigger) build.uuid = 'deadbeef-dead-beef-dead-beefdeadbeef' build.save() diff --git a/test/data/test.db b/test/data/test.db index e733ca294..8caf9ba9f 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 6d50601da..a6c53c798 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -50,18 +50,23 @@ class StatusWrapper(object): class DockerfileBuildContext(object): - def __init__(self, build_context_dir, tag_name, push_token, build_uuid): + def __init__(self, build_context_dir, dockerfile_subdir, repo, tag_names, + push_token, build_uuid): self._build_dir = build_context_dir - self._tag_name = tag_name + self._dockerfile_subdir = dockerfile_subdir + self._repo = repo + self._tag_names = tag_names self._push_token = push_token self._cl = Client(timeout=1200, version='1.7') self._status = StatusWrapper(build_uuid) self._build_logger = partial(build_logs.append_log_message, build_uuid) - dockerfile_path = os.path.join(self._build_dir, "Dockerfile") + dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir, + "Dockerfile") self._num_steps = DockerfileBuildContext.__count_steps(dockerfile_path) - logger.debug('Will build and push to tag named: %s' % self._tag_name) + logger.debug('Will build and push to repo %s with tags named: %s' % + (self._repo, self._tag_names)) def __enter__(self): return self @@ -94,9 +99,13 @@ class DockerfileBuildContext(object): with self._status as status: status['total_commands'] = self._num_steps - logger.debug('Building to tag named: %s' % self._tag_name) - build_status = self._cl.build(path=self._build_dir, tag=self._tag_name, - stream=True) + logger.debug('Building to tags named: %s' % self._tag_names) + context_path = os.path.join(self._build_dir, self._dockerfile_subdir) + + logger.debug('Final context path: %s exists: %s' % + (context_path, os.path.exists(context_path))) + + build_status = self._cl.build(path=context_path, stream=True) current_step = 0 built_image = None @@ -128,9 +137,9 @@ class DockerfileBuildContext(object): def push(self, built_image): # Login to the registry - host = re.match(r'([a-z0-9.:]+)/.+/.+$', self._tag_name) + host = re.match(r'([a-z0-9.:]+)/.+/.+$', self._repo) if not host: - raise RuntimeError('Invalid tag name: %s' % self._tag_name) + raise RuntimeError('Invalid repo name: %s' % self._repo) for protocol in ['https', 'http']: registry_endpoint = '%s://%s/v1/' % (protocol, host.group(1)) @@ -142,13 +151,18 @@ class DockerfileBuildContext(object): except APIError: pass # Probably the wrong protocol + for tag in self._tag_names: + logger.debug('Tagging image %s as %s:%s' % + (built_image, self._repo, tag)) + self._cl.tag(built_image, self._repo, tag) + history = json.loads(self._cl.history(built_image)) num_images = len(history) with self._status as status: status['total_images'] = num_images - logger.debug('Pushing to tag name: %s' % self._tag_name) - resp = self._cl.push(self._tag_name, stream=True) + logger.debug('Pushing to repo %s' % self._repo) + resp = self._cl.push(self._repo, stream=True) for status_str in resp: status = json.loads(status_str) @@ -258,8 +272,13 @@ class DockerfileBuildWorker(Worker): job_details['repository'], job_details['build_uuid']) - resource_url = user_files.get_file_url(repository_build.resource_key) - tag_name = repository_build.tag + job_config = json.loads(repository_build.job_config) + + resource_url = user_files.get_file_url(job_config['resource_key']) + tag_names = job_config['docker_tags'] + build_subdir = job_config['build_subdir'] + repo = job_config['repository'] + access_token = repository_build.access_token.code log_appender = partial(build_logs.append_log_message, @@ -267,16 +286,15 @@ class DockerfileBuildWorker(Worker): log_appender('initializing', build_logs.PHASE) - start_msg = ('Starting job with resource url: %s tag: %s' % (resource_url, - tag_name)) - logger.debug(start_msg) + start_msg = ('Starting job with resource url: %s repo: %s' % (resource_url, + repo)) log_appender(start_msg) docker_resource = requests.get(resource_url) c_type = docker_resource.headers['content-type'] - filetype_msg = ('Request to build file of type: %s with tag: %s' % - (c_type, tag_name)) + filetype_msg = ('Request to build type: %s with repo: %s and tags: %s' % + (c_type, repo, tag_names)) logger.info(filetype_msg) log_appender(filetype_msg) @@ -288,7 +306,8 @@ class DockerfileBuildWorker(Worker): repository_build.phase = 'building' repository_build.save() - with DockerfileBuildContext(build_dir, tag_name, access_token, + with DockerfileBuildContext(build_dir, build_subdir, repo, tag_names, + access_token, repository_build.uuid) as build_ctxt: try: built_image = build_ctxt.build() @@ -298,7 +317,7 @@ class DockerfileBuildWorker(Worker): repository_build.phase = 'error' repository_build.save() log_appender('Unable to build dockerfile.', build_logs.ERROR) - return False + return True log_appender('pushing', build_logs.PHASE) repository_build.phase = 'pushing' @@ -316,7 +335,7 @@ class DockerfileBuildWorker(Worker): repository_build.phase = 'error' repository_build.save() log_appender(str(exc), build_logs.ERROR) - return False + return True return True