448 lines
13 KiB
Python
448 lines
13 KiB
Python
import logging
|
|
import io
|
|
import os.path
|
|
import tarfile
|
|
import base64
|
|
import re
|
|
|
|
from github import Github, UnknownObjectException, GithubException
|
|
from tempfile import SpooledTemporaryFile
|
|
|
|
from app import app, userfiles as user_files, github_trigger
|
|
from util.tarfileappender import TarfileAppender
|
|
|
|
|
|
client = app.config['HTTPCLIENT']
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
TARBALL_MIME = 'application/gzip'
|
|
CHUNK_SIZE = 512 * 1024
|
|
|
|
|
|
def should_skip_commit(message):
|
|
return '[skip build]' in message or '[build skip]' in message
|
|
|
|
|
|
class BuildArchiveException(Exception):
|
|
pass
|
|
|
|
class InvalidServiceException(Exception):
|
|
pass
|
|
|
|
class TriggerActivationException(Exception):
|
|
pass
|
|
|
|
class TriggerDeactivationException(Exception):
|
|
pass
|
|
|
|
class TriggerStartException(Exception):
|
|
pass
|
|
|
|
class ValidationRequestException(Exception):
|
|
pass
|
|
|
|
class SkipRequestException(Exception):
|
|
pass
|
|
|
|
class EmptyRepositoryException(Exception):
|
|
pass
|
|
|
|
class RepositoryReadException(Exception):
|
|
pass
|
|
|
|
|
|
class BuildTrigger(object):
|
|
def __init__(self):
|
|
pass
|
|
|
|
def dockerfile_url(self, auth_token, config):
|
|
"""
|
|
Returns the URL at which the Dockerfile for the trigger can be found or None if none/not applicable.
|
|
"""
|
|
return None
|
|
|
|
def load_dockerfile_contents(self, auth_token, config):
|
|
"""
|
|
Loads the Dockerfile found for the trigger's config and returns them or None if none could
|
|
be found/loaded.
|
|
"""
|
|
return None
|
|
|
|
def list_build_sources(self, auth_token):
|
|
"""
|
|
Take the auth information for the specific trigger type and load the
|
|
list of build sources(repositories).
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
def list_build_subdirs(self, auth_token, config):
|
|
"""
|
|
Take the auth information and the specified config so far and list all of
|
|
the possible subdirs containing dockerfiles.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
def handle_trigger_request(self, request, auth_token, config):
|
|
"""
|
|
Transform the incoming request data into a set of actions. Returns a tuple
|
|
of usefiles resource id, docker tags, build name, and resource subdir.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
def is_active(self, config):
|
|
"""
|
|
Returns True if the current build trigger is active. Inactive means further
|
|
setup is needed.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
|
|
"""
|
|
Activates the trigger for the service, with the given new configuration.
|
|
Returns new configuration that should be stored if successful.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
def deactivate(self, auth_token, config):
|
|
"""
|
|
Deactivates the trigger for the service, removing any hooks installed in
|
|
the remote service. Returns the new config that should be stored if this
|
|
trigger is going to be re-activated.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
def manual_start(self, auth_token, config, run_parameters = None):
|
|
"""
|
|
Manually creates a repository build for this trigger.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
def list_field_values(self, auth_token, config, field_name):
|
|
"""
|
|
Lists all values for the given custom trigger field. For example, a trigger might have a
|
|
field named "branches", and this method would return all branches.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
@classmethod
|
|
def service_name(cls):
|
|
"""
|
|
Particular service implemented by subclasses.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
@classmethod
|
|
def get_trigger_for_service(cls, service):
|
|
for subc in cls.__subclasses__():
|
|
if subc.service_name() == service:
|
|
return subc()
|
|
|
|
raise InvalidServiceException('Unable to find service: %s' % service)
|
|
|
|
|
|
def raise_unsupported():
|
|
raise io.UnsupportedOperation
|
|
|
|
|
|
class GithubBuildTrigger(BuildTrigger):
|
|
@staticmethod
|
|
def _get_client(auth_token):
|
|
return Github(auth_token,
|
|
base_url=github_trigger.api_endpoint(),
|
|
client_id=github_trigger.client_id(),
|
|
client_secret=github_trigger.client_secret())
|
|
|
|
@classmethod
|
|
def service_name(cls):
|
|
return 'github'
|
|
|
|
def is_active(self, config):
|
|
return 'hook_id' in config
|
|
|
|
def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
|
|
new_build_source = config['build_source']
|
|
gh_client = self._get_client(auth_token)
|
|
|
|
try:
|
|
to_add_webhook = gh_client.get_repo(new_build_source)
|
|
except UnknownObjectException:
|
|
msg = 'Unable to find GitHub repository for source: %s'
|
|
raise TriggerActivationException(msg % new_build_source)
|
|
|
|
webhook_config = {
|
|
'url': standard_webhook_url,
|
|
'content_type': 'json',
|
|
}
|
|
|
|
try:
|
|
hook = to_add_webhook.create_hook('web', webhook_config)
|
|
config['hook_id'] = hook.id
|
|
config['master_branch'] = to_add_webhook.default_branch
|
|
except GithubException:
|
|
msg = 'Unable to create webhook on repository: %s'
|
|
raise TriggerActivationException(msg % new_build_source)
|
|
|
|
return config
|
|
|
|
def deactivate(self, auth_token, config):
|
|
gh_client = self._get_client(auth_token)
|
|
|
|
try:
|
|
repo = gh_client.get_repo(config['build_source'])
|
|
to_delete = repo.get_hook(config['hook_id'])
|
|
to_delete.delete()
|
|
except GithubException:
|
|
msg = 'Unable to remove hook: %s' % config['hook_id']
|
|
raise TriggerDeactivationException(msg)
|
|
|
|
config.pop('hook_id', None)
|
|
|
|
return config
|
|
|
|
def list_build_sources(self, auth_token):
|
|
gh_client = self._get_client(auth_token)
|
|
usr = gh_client.get_user()
|
|
|
|
personal = {
|
|
'personal': True,
|
|
'repos': [repo.full_name for repo in usr.get_repos()],
|
|
'info': {
|
|
'name': usr.login,
|
|
'avatar_url': usr.avatar_url,
|
|
}
|
|
}
|
|
|
|
repos_by_org = [personal]
|
|
|
|
for org in usr.get_orgs():
|
|
repo_list = []
|
|
for repo in org.get_repos(type='member'):
|
|
repo_list.append(repo.full_name)
|
|
|
|
repos_by_org.append({
|
|
'personal': False,
|
|
'repos': repo_list,
|
|
'info': {
|
|
'name': org.name,
|
|
'avatar_url': org.avatar_url
|
|
}
|
|
})
|
|
|
|
return repos_by_org
|
|
|
|
def matches_ref(self, ref, regex):
|
|
match_string = ref.split('/', 1)[1]
|
|
if not regex:
|
|
return False
|
|
|
|
m = regex.match(match_string)
|
|
if not m:
|
|
return False
|
|
|
|
return len(m.group(0)) == len(match_string)
|
|
|
|
def list_build_subdirs(self, auth_token, config):
|
|
gh_client = self._get_client(auth_token)
|
|
source = config['build_source']
|
|
|
|
try:
|
|
repo = gh_client.get_repo(source)
|
|
|
|
# Find the first matching branch.
|
|
branches = None
|
|
if 'branchtag_regex' in config:
|
|
try:
|
|
regex = re.compile(config['branchtag_regex'])
|
|
branches = [branch.name for branch in repo.get_branches()
|
|
if self.matches_ref('refs/heads/' + branch.name, regex)]
|
|
except:
|
|
pass
|
|
|
|
branches = branches or [repo.default_branch or 'master']
|
|
default_commit = repo.get_branch(branches[0]).commit
|
|
commit_tree = repo.get_git_tree(default_commit.sha, recursive=True)
|
|
|
|
return [os.path.dirname(elem.path) for elem in commit_tree.tree
|
|
if (elem.type == u'blob' and
|
|
os.path.basename(elem.path) == u'Dockerfile')]
|
|
except GithubException as ge:
|
|
message = ge.data.get('message', 'Unable to list contents of repository: %s' % source)
|
|
if message == 'Branch not found':
|
|
raise EmptyRepositoryException()
|
|
|
|
raise RepositoryReadException(message)
|
|
|
|
def dockerfile_url(self, auth_token, config):
|
|
source = config['build_source']
|
|
subdirectory = config.get('subdir', '')
|
|
path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
|
|
|
|
gh_client = self._get_client(auth_token)
|
|
try:
|
|
repo = gh_client.get_repo(source)
|
|
master_branch = repo.default_branch or 'master'
|
|
return 'https://github.com/%s/blob/%s/%s' % (source, master_branch, path)
|
|
except GithubException as ge:
|
|
return None
|
|
|
|
def load_dockerfile_contents(self, auth_token, config):
|
|
gh_client = self._get_client(auth_token)
|
|
|
|
source = config['build_source']
|
|
subdirectory = config.get('subdir', '')
|
|
path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
|
|
|
|
try:
|
|
repo = gh_client.get_repo(source)
|
|
file_info = repo.get_file_contents(path)
|
|
if file_info is None:
|
|
return None
|
|
|
|
content = file_info.content
|
|
if file_info.encoding == 'base64':
|
|
content = base64.b64decode(content)
|
|
return content
|
|
|
|
except GithubException as ge:
|
|
message = ge.data.get('message', 'Unable to read Dockerfile: %s' % source)
|
|
raise RepositoryReadException(message)
|
|
|
|
@staticmethod
|
|
def _prepare_build(config, repo, commit_sha, build_name, ref):
|
|
# Prepare the download and upload URLs
|
|
archive_link = repo.get_archive_link('tarball', commit_sha)
|
|
download_archive = client.get(archive_link, stream=True)
|
|
|
|
tarball_subdir = ''
|
|
with SpooledTemporaryFile(CHUNK_SIZE) as tarball:
|
|
for chunk in download_archive.iter_content(CHUNK_SIZE):
|
|
tarball.write(chunk)
|
|
|
|
# Seek to position 0 to make tarfile happy
|
|
tarball.seek(0)
|
|
|
|
# Pull out the name of the subdir that GitHub generated
|
|
with tarfile.open(fileobj=tarball) as archive:
|
|
tarball_subdir = archive.getnames()[0]
|
|
|
|
# Seek to position 0 to make tarfile happy.
|
|
tarball.seek(0)
|
|
|
|
entries = {
|
|
tarball_subdir + '/.git/HEAD': commit_sha,
|
|
tarball_subdir + '/.git/objects/': None,
|
|
tarball_subdir + '/.git/refs/': None
|
|
}
|
|
|
|
appender = TarfileAppender(tarball, entries).get_stream()
|
|
dockerfile_id = user_files.store_file(appender, TARBALL_MIME)
|
|
|
|
logger.debug('Successfully prepared job')
|
|
|
|
# compute the tag(s)
|
|
branch = ref.split('/')[-1]
|
|
tags = {branch}
|
|
if branch == repo.default_branch:
|
|
tags.add('latest')
|
|
logger.debug('Pushing to tags: %s' % tags)
|
|
|
|
# compute the subdir
|
|
repo_subdir = config['subdir']
|
|
joined_subdir = os.path.join(tarball_subdir, repo_subdir)
|
|
logger.debug('Final subdir: %s' % joined_subdir)
|
|
|
|
return dockerfile_id, list(tags), build_name, joined_subdir
|
|
|
|
@staticmethod
|
|
def get_display_name(sha):
|
|
return sha[0:7]
|
|
|
|
def handle_trigger_request(self, request, auth_token, config):
|
|
payload = request.get_json()
|
|
if not payload or payload.get('head_commit') is None:
|
|
raise SkipRequestException()
|
|
|
|
if 'zen' in payload:
|
|
raise ValidationRequestException()
|
|
|
|
logger.debug('Payload %s', payload)
|
|
ref = payload['ref']
|
|
commit_sha = payload['head_commit']['id']
|
|
commit_message = payload['head_commit'].get('message', '')
|
|
|
|
if 'branchtag_regex' in config:
|
|
try:
|
|
regex = re.compile(config['branchtag_regex'])
|
|
except:
|
|
regex = re.compile('.*')
|
|
|
|
if not self.matches_ref(ref, regex):
|
|
raise SkipRequestException()
|
|
|
|
if should_skip_commit(commit_message):
|
|
raise SkipRequestException()
|
|
|
|
short_sha = GithubBuildTrigger.get_display_name(commit_sha)
|
|
|
|
gh_client = self._get_client(auth_token)
|
|
|
|
repo_full_name = '%s/%s' % (payload['repository']['owner']['name'],
|
|
payload['repository']['name'])
|
|
repo = gh_client.get_repo(repo_full_name)
|
|
|
|
logger.debug('Github repo: %s', repo)
|
|
|
|
return GithubBuildTrigger._prepare_build(config, repo, commit_sha,
|
|
short_sha, ref)
|
|
|
|
def manual_start(self, auth_token, config, run_parameters = None):
|
|
try:
|
|
source = config['build_source']
|
|
run_parameters = run_parameters or {}
|
|
|
|
gh_client = self._get_client(auth_token)
|
|
repo = gh_client.get_repo(source)
|
|
branch_name = run_parameters.get('branch_name') or repo.default_branch
|
|
branch = repo.get_branch(branch_name)
|
|
branch_sha = branch.commit.sha
|
|
short_sha = GithubBuildTrigger.get_display_name(branch_sha)
|
|
ref = 'refs/heads/%s' % (branch_name)
|
|
|
|
return self._prepare_build(config, repo, branch_sha, short_sha, ref)
|
|
except GithubException as ghe:
|
|
raise TriggerStartException(ghe.data['message'])
|
|
|
|
|
|
def list_field_values(self, auth_token, config, field_name):
|
|
if field_name == 'refs':
|
|
branches = self.list_field_values(auth_token, config, 'branch_name')
|
|
tags = self.list_field_values(auth_token, config, 'tag_name')
|
|
|
|
return ([{'kind': 'branch', 'name': b} for b in branches] +
|
|
[{'kind': 'tag', 'name': tag} for tag in tags])
|
|
|
|
if field_name == 'tag_name':
|
|
gh_client = self._get_client(auth_token)
|
|
source = config['build_source']
|
|
repo = gh_client.get_repo(source)
|
|
return [tag.name for tag in repo.get_tags()]
|
|
|
|
if field_name == 'branch_name':
|
|
gh_client = self._get_client(auth_token)
|
|
source = config['build_source']
|
|
repo = gh_client.get_repo(source)
|
|
branches = [branch.name for branch in repo.get_branches()]
|
|
|
|
if not repo.default_branch in branches:
|
|
branches.insert(0, repo.default_branch)
|
|
|
|
if branches[0] != repo.default_branch:
|
|
branches.remove(repo.default_branch)
|
|
branches.insert(0, repo.default_branch)
|
|
|
|
return branches
|
|
|
|
return None
|