Prepare the build worker to support multiple tags and subdirectories. Change the build database config to accept a job config object instead of breaking out the parameters into independent blocks.
This commit is contained in:
jakedt
parent
4b0f4c0a7b
commit
13dea98499
9 changed files with 114 additions and 53 deletions
|
|
@ -237,8 +237,7 @@ class RepositoryBuild(BaseModel):
|
|||
uuid = CharField(default=uuid_generator, index=True)
|
||||
repository = ForeignKeyField(Repository, index=True)
|
||||
access_token = ForeignKeyField(AccessToken)
|
||||
resource_key = CharField()
|
||||
tag = CharField()
|
||||
job_config = TextField()
|
||||
phase = CharField(default='waiting')
|
||||
started = DateTimeField(default=datetime.now)
|
||||
display_name = CharField()
|
||||
|
|
|
|||
|
|
@ -1409,10 +1409,10 @@ def list_repository_builds(namespace_name, repository_name,
|
|||
return query
|
||||
|
||||
|
||||
def create_repository_build(repo, access_token, resource_key, tag,
|
||||
def create_repository_build(repo, access_token, job_config_obj,
|
||||
display_name, trigger=None):
|
||||
return RepositoryBuild.create(repository=repo, access_token=access_token,
|
||||
resource_key=resource_key, tag=tag,
|
||||
job_config=json.dumps(job_config_obj),
|
||||
display_name=display_name, trigger=trigger)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -68,5 +68,5 @@ class WorkQueue(object):
|
|||
|
||||
|
||||
image_diff_queue = WorkQueue('imagediff')
|
||||
dockerfile_build_queue = WorkQueue('dockerfilebuild2')
|
||||
dockerfile_build_queue = WorkQueue('dockerfilebuild3')
|
||||
webhook_queue = WorkQueue('webhook')
|
||||
|
|
|
|||
|
|
@ -1144,7 +1144,7 @@ def build_status_view(build_obj, can_write=False):
|
|||
'started': build_obj.started,
|
||||
'display_name': build_obj.display_name,
|
||||
'status': status,
|
||||
'resource_key': build_obj.resource_key if can_write else None,
|
||||
'job_config': json.loads(build_obj.job_config) if can_write else None,
|
||||
'is_writer': can_write,
|
||||
'trigger': trigger_view(build_obj.trigger),
|
||||
}
|
||||
|
|
@ -1234,11 +1234,13 @@ def request_repo_build(namespace, repository):
|
|||
logger.debug('User requested repository initialization.')
|
||||
dockerfile_id = request.get_json()['file_id']
|
||||
|
||||
# Check if the dockerfile resource has already been used. If so, then it can only be reused if the
|
||||
# user has access to the repository for which it was used.
|
||||
# Check if the dockerfile resource has already been used. If so, then it
|
||||
# can only be reused if the user has access to the repository for which it
|
||||
# was used.
|
||||
associated_repository = model.get_repository_for_resource(dockerfile_id)
|
||||
if associated_repository:
|
||||
if not ModifyRepositoryPermission(associated_repository.namespace, associated_repository.name):
|
||||
if not ModifyRepositoryPermission(associated_repository.namespace,
|
||||
associated_repository.name):
|
||||
abort(403)
|
||||
|
||||
# Start the build.
|
||||
|
|
@ -1248,9 +1250,15 @@ def request_repo_build(namespace, repository):
|
|||
logger.debug('**********Md5: %s' % display_name)
|
||||
|
||||
host = urlparse.urlparse(request.url).netloc
|
||||
tag = '%s/%s/%s' % (host, repo.namespace, repo.name)
|
||||
build_request = model.create_repository_build(repo, token, dockerfile_id,
|
||||
tag, display_name)
|
||||
repo = '%s/%s/%s' % (host, repo.namespace, repo.name)
|
||||
job_config = {
|
||||
'docker_tags': ['latest'],
|
||||
'build_subdir': '',
|
||||
'repository': repo,
|
||||
'resource_key': dockerfile_id,
|
||||
}
|
||||
build_request = model.create_repository_build(repo, token, job_config,
|
||||
display_name)
|
||||
dockerfile_build_queue.put(json.dumps({
|
||||
'build_uuid': build_request.uuid,
|
||||
'namespace': namespace,
|
||||
|
|
|
|||
|
|
@ -55,7 +55,8 @@ class BuildTrigger(object):
|
|||
|
||||
def handle_trigger_request(self, request, auth_token, config):
|
||||
"""
|
||||
Transform the incoming request data into a set of actions.
|
||||
Transform the incoming request data into a set of actions. Returns a tuple
|
||||
of usefiles resource id, docker tags, build name, and resource subdir.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
|
@ -163,7 +164,7 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
|
||||
try:
|
||||
repo = gh_client.get_repo(source)
|
||||
default_commit = repo.get_branch(repo.default_branch).commit
|
||||
default_commit = repo.get_branch(repo.master_branch).commit
|
||||
commit_tree = repo.get_git_tree(default_commit.sha, recursive=True)
|
||||
|
||||
return [os.path.dirname(elem.path) for elem in commit_tree.tree
|
||||
|
|
@ -181,7 +182,8 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
|
||||
logger.debug('Payload %s', payload)
|
||||
ref = payload['ref']
|
||||
commit_id = payload['head_commit']['id'][0:7]
|
||||
commit_sha = payload['head_commit']['id']
|
||||
short_sha = commit_sha[0:7]
|
||||
|
||||
gh_client = self._get_client(auth_token)
|
||||
|
||||
|
|
@ -192,8 +194,7 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
logger.debug('Github repo: %s', repo)
|
||||
|
||||
# Prepare the download and upload URLs
|
||||
branch_name = ref.split('/')[-1]
|
||||
archive_link = repo.get_archive_link('zipball', branch_name)
|
||||
archive_link = repo.get_archive_link('zipball', short_sha)
|
||||
download_archive = client.get(archive_link, stream=True)
|
||||
|
||||
with SpooledTemporaryFile(CHUNK_SIZE) as zipball:
|
||||
|
|
@ -204,4 +205,17 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
|
||||
logger.debug('Successfully prepared job')
|
||||
|
||||
return dockerfile_id, branch_name, commit_id
|
||||
# compute the tag(s)
|
||||
pushed_branch = ref.split('/')[-1]
|
||||
tags = {pushed_branch}
|
||||
if pushed_branch == repo.master_branch:
|
||||
tags.add('latest')
|
||||
logger.debug('Pushing to tags: %s' % tags)
|
||||
|
||||
# compute the subdir
|
||||
repo_subdir = config['subdir']
|
||||
zipball_subdir = '%s-%s-%s' % (repo.owner.login, repo.name, short_sha)
|
||||
joined_subdir = os.path.join(zipball_subdir, repo_subdir)
|
||||
logger.debug('Final subdir: %s' % joined_subdir)
|
||||
|
||||
return dockerfile_id, list(tags), short_sha, joined_subdir
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
import logging
|
||||
import stripe
|
||||
import urlparse
|
||||
import json
|
||||
|
||||
from flask import request, make_response, Blueprint
|
||||
|
||||
from data import model
|
||||
from data.queue import dockerfile_build_queue
|
||||
from auth.auth import process_auth
|
||||
from auth.permissions import ModifyRepositoryPermission
|
||||
from util.invoice import renderInvoiceToHtml
|
||||
|
|
@ -62,22 +64,36 @@ def build_trigger_webhook(namespace, repository, trigger_uuid):
|
|||
|
||||
logger.debug('Passing webhook request to handler %s', handler)
|
||||
try:
|
||||
df_id, tag, name = handler.handle_trigger_request(request,
|
||||
trigger.auth_token,
|
||||
trigger.config)
|
||||
specs = handler.handle_trigger_request(request, trigger.auth_token,
|
||||
json.loads(trigger.config))
|
||||
dockerfile_id, tags, name, subdir = specs
|
||||
|
||||
except ValidationRequestException:
|
||||
# This was just a validation request, don't need to build anything
|
||||
# This was just a validation request, we don't need to build anything
|
||||
return make_response('Okay')
|
||||
|
||||
host = urlparse.urlparse(request.url).netloc
|
||||
full_tag = '%s/%s/%s:%s' % (host, trigger.repository.namespace,
|
||||
trigger.repository.name, tag)
|
||||
repo = '%s/%s/%s' % (host, trigger.repository.namespace,
|
||||
trigger.repository.name)
|
||||
|
||||
token = model.create_access_token(trigger.repository, 'write')
|
||||
logger.debug('Creating build %s with full_tag %s and dockerfile_id %s',
|
||||
name, full_tag, df_id)
|
||||
model.create_repository_build(trigger.repository, token, df_id, full_tag,
|
||||
name)
|
||||
logger.debug('Creating build %s with repo %s tags %s and dockerfile_id %s',
|
||||
name, repo, tags, dockerfile_id)
|
||||
|
||||
job_config = {
|
||||
'docker_tags': tags,
|
||||
'repository': repo,
|
||||
'build_subdir': subdir,
|
||||
'resource_key': dockerfile_id,
|
||||
}
|
||||
build_request = model.create_repository_build(trigger.repository, token,
|
||||
job_config, name)
|
||||
|
||||
dockerfile_build_queue.put(json.dumps({
|
||||
'build_uuid': build_request.uuid,
|
||||
'namespace': namespace,
|
||||
'repository': repository,
|
||||
}), retries_remaining=1)
|
||||
|
||||
return make_response('Okay')
|
||||
|
||||
|
|
|
|||
13
initdb.py
13
initdb.py
|
|
@ -309,7 +309,6 @@ def populate_database():
|
|||
False, [], (0, [], None))
|
||||
|
||||
token = model.create_access_token(building, 'write')
|
||||
tag = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name)
|
||||
|
||||
trigger = model.create_build_trigger(building, 'github', '123authtoken',
|
||||
new_user_1)
|
||||
|
|
@ -318,9 +317,15 @@ def populate_database():
|
|||
})
|
||||
trigger.save()
|
||||
|
||||
build = model.create_repository_build(building, token,
|
||||
'701dcc3724fb4f2ea6c31400528343cd',
|
||||
tag, 'build-name', trigger)
|
||||
repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name)
|
||||
job_config = {
|
||||
'repository': repo,
|
||||
'docker_tags': ['latest'],
|
||||
'build_subdir': '',
|
||||
'resource_key': '701dcc3724fb4f2ea6c31400528343cd',
|
||||
}
|
||||
build = model.create_repository_build(building, token, job_config,
|
||||
'build-name', trigger)
|
||||
build.uuid = 'deadbeef-dead-beef-dead-beefdeadbeef'
|
||||
build.save()
|
||||
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -50,18 +50,23 @@ class StatusWrapper(object):
|
|||
|
||||
|
||||
class DockerfileBuildContext(object):
|
||||
def __init__(self, build_context_dir, tag_name, push_token, build_uuid):
|
||||
def __init__(self, build_context_dir, dockerfile_subdir, repo, tag_names,
|
||||
push_token, build_uuid):
|
||||
self._build_dir = build_context_dir
|
||||
self._tag_name = tag_name
|
||||
self._dockerfile_subdir = dockerfile_subdir
|
||||
self._repo = repo
|
||||
self._tag_names = tag_names
|
||||
self._push_token = push_token
|
||||
self._cl = Client(timeout=1200, version='1.7')
|
||||
self._status = StatusWrapper(build_uuid)
|
||||
self._build_logger = partial(build_logs.append_log_message, build_uuid)
|
||||
|
||||
dockerfile_path = os.path.join(self._build_dir, "Dockerfile")
|
||||
dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir,
|
||||
"Dockerfile")
|
||||
self._num_steps = DockerfileBuildContext.__count_steps(dockerfile_path)
|
||||
|
||||
logger.debug('Will build and push to tag named: %s' % self._tag_name)
|
||||
logger.debug('Will build and push to repo %s with tags named: %s' %
|
||||
(self._repo, self._tag_names))
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
|
@ -94,9 +99,13 @@ class DockerfileBuildContext(object):
|
|||
with self._status as status:
|
||||
status['total_commands'] = self._num_steps
|
||||
|
||||
logger.debug('Building to tag named: %s' % self._tag_name)
|
||||
build_status = self._cl.build(path=self._build_dir, tag=self._tag_name,
|
||||
stream=True)
|
||||
logger.debug('Building to tags named: %s' % self._tag_names)
|
||||
context_path = os.path.join(self._build_dir, self._dockerfile_subdir)
|
||||
|
||||
logger.debug('Final context path: %s exists: %s' %
|
||||
(context_path, os.path.exists(context_path)))
|
||||
|
||||
build_status = self._cl.build(path=context_path, stream=True)
|
||||
|
||||
current_step = 0
|
||||
built_image = None
|
||||
|
|
@ -128,9 +137,9 @@ class DockerfileBuildContext(object):
|
|||
|
||||
def push(self, built_image):
|
||||
# Login to the registry
|
||||
host = re.match(r'([a-z0-9.:]+)/.+/.+$', self._tag_name)
|
||||
host = re.match(r'([a-z0-9.:]+)/.+/.+$', self._repo)
|
||||
if not host:
|
||||
raise RuntimeError('Invalid tag name: %s' % self._tag_name)
|
||||
raise RuntimeError('Invalid repo name: %s' % self._repo)
|
||||
|
||||
for protocol in ['https', 'http']:
|
||||
registry_endpoint = '%s://%s/v1/' % (protocol, host.group(1))
|
||||
|
|
@ -142,13 +151,18 @@ class DockerfileBuildContext(object):
|
|||
except APIError:
|
||||
pass # Probably the wrong protocol
|
||||
|
||||
for tag in self._tag_names:
|
||||
logger.debug('Tagging image %s as %s:%s' %
|
||||
(built_image, self._repo, tag))
|
||||
self._cl.tag(built_image, self._repo, tag)
|
||||
|
||||
history = json.loads(self._cl.history(built_image))
|
||||
num_images = len(history)
|
||||
with self._status as status:
|
||||
status['total_images'] = num_images
|
||||
|
||||
logger.debug('Pushing to tag name: %s' % self._tag_name)
|
||||
resp = self._cl.push(self._tag_name, stream=True)
|
||||
logger.debug('Pushing to repo %s' % self._repo)
|
||||
resp = self._cl.push(self._repo, stream=True)
|
||||
|
||||
for status_str in resp:
|
||||
status = json.loads(status_str)
|
||||
|
|
@ -258,8 +272,13 @@ class DockerfileBuildWorker(Worker):
|
|||
job_details['repository'],
|
||||
job_details['build_uuid'])
|
||||
|
||||
resource_url = user_files.get_file_url(repository_build.resource_key)
|
||||
tag_name = repository_build.tag
|
||||
job_config = json.loads(repository_build.job_config)
|
||||
|
||||
resource_url = user_files.get_file_url(job_config['resource_key'])
|
||||
tag_names = job_config['docker_tags']
|
||||
build_subdir = job_config['build_subdir']
|
||||
repo = job_config['repository']
|
||||
|
||||
access_token = repository_build.access_token.code
|
||||
|
||||
log_appender = partial(build_logs.append_log_message,
|
||||
|
|
@ -267,16 +286,15 @@ class DockerfileBuildWorker(Worker):
|
|||
|
||||
log_appender('initializing', build_logs.PHASE)
|
||||
|
||||
start_msg = ('Starting job with resource url: %s tag: %s' % (resource_url,
|
||||
tag_name))
|
||||
logger.debug(start_msg)
|
||||
start_msg = ('Starting job with resource url: %s repo: %s' % (resource_url,
|
||||
repo))
|
||||
log_appender(start_msg)
|
||||
|
||||
docker_resource = requests.get(resource_url)
|
||||
c_type = docker_resource.headers['content-type']
|
||||
|
||||
filetype_msg = ('Request to build file of type: %s with tag: %s' %
|
||||
(c_type, tag_name))
|
||||
filetype_msg = ('Request to build type: %s with repo: %s and tags: %s' %
|
||||
(c_type, repo, tag_names))
|
||||
logger.info(filetype_msg)
|
||||
log_appender(filetype_msg)
|
||||
|
||||
|
|
@ -288,7 +306,8 @@ class DockerfileBuildWorker(Worker):
|
|||
repository_build.phase = 'building'
|
||||
repository_build.save()
|
||||
|
||||
with DockerfileBuildContext(build_dir, tag_name, access_token,
|
||||
with DockerfileBuildContext(build_dir, build_subdir, repo, tag_names,
|
||||
access_token,
|
||||
repository_build.uuid) as build_ctxt:
|
||||
try:
|
||||
built_image = build_ctxt.build()
|
||||
|
|
@ -298,7 +317,7 @@ class DockerfileBuildWorker(Worker):
|
|||
repository_build.phase = 'error'
|
||||
repository_build.save()
|
||||
log_appender('Unable to build dockerfile.', build_logs.ERROR)
|
||||
return False
|
||||
return True
|
||||
|
||||
log_appender('pushing', build_logs.PHASE)
|
||||
repository_build.phase = 'pushing'
|
||||
|
|
@ -316,7 +335,7 @@ class DockerfileBuildWorker(Worker):
|
|||
repository_build.phase = 'error'
|
||||
repository_build.save()
|
||||
log_appender(str(exc), build_logs.ERROR)
|
||||
return False
|
||||
return True
|
||||
|
||||
return True
|
||||
|
||||
|
|
|
|||
Reference in a new issue