endpoints: initial github trigger that uses keys
This commit is contained in:
Jimmy Zelinskie
parent
288f847e9a
commit
1594b92f71
1 changed files with 384 additions and 11 deletions
|
|
@ -10,6 +10,7 @@ from tempfile import SpooledTemporaryFile
|
|||
|
||||
from app import app, userfiles as user_files, github_trigger
|
||||
from util.tarfileappender import TarfileAppender
|
||||
from util.ssh import generate_ssh_keypair
|
||||
|
||||
|
||||
client = app.config['HTTPCLIENT']
|
||||
|
|
@ -60,7 +61,7 @@ class BuildTrigger(object):
|
|||
|
||||
def dockerfile_url(self, auth_token, config):
|
||||
"""
|
||||
Returns the URL at which the Dockerfile for the trigger can be found or None if none/not applicable.
|
||||
Returns the URL at which the Dockerfile for the trigger is found or None if none/not applicable.
|
||||
"""
|
||||
return None
|
||||
|
||||
|
|
@ -114,7 +115,7 @@ class BuildTrigger(object):
|
|||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def manual_start(self, auth_token, config, run_parameters = None):
|
||||
def manual_start(self, auth_token, config, run_parameters=None):
|
||||
"""
|
||||
Manually creates a repository build for this trigger.
|
||||
"""
|
||||
|
|
@ -148,6 +149,9 @@ def raise_unsupported():
|
|||
|
||||
|
||||
class GithubBuildTrigger(BuildTrigger):
|
||||
"""
|
||||
BuildTrigger for GitHub that uses the archive API and buildpacks.
|
||||
"""
|
||||
@staticmethod
|
||||
def _get_client(auth_token):
|
||||
return Github(auth_token,
|
||||
|
|
@ -233,7 +237,8 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
|
||||
return repos_by_org
|
||||
|
||||
def matches_ref(self, ref, regex):
|
||||
@staticmethod
|
||||
def matches_ref(ref, regex):
|
||||
match_string = ref.split('/', 1)[1]
|
||||
if not regex:
|
||||
return False
|
||||
|
|
@ -257,7 +262,7 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
try:
|
||||
regex = re.compile(config['branchtag_regex'])
|
||||
branches = [branch.name for branch in repo.get_branches()
|
||||
if self.matches_ref('refs/heads/' + branch.name, regex)]
|
||||
if GithubBuildTrigger.matches_ref('refs/heads/' + branch.name, regex)]
|
||||
except:
|
||||
pass
|
||||
|
||||
|
|
@ -275,7 +280,7 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
|
||||
raise RepositoryReadException(message)
|
||||
|
||||
def dockerfile_url(self, auth_token, config):
|
||||
def dockerfile_url(self, auth_token, config):
|
||||
source = config['build_source']
|
||||
subdirectory = config.get('subdir', '')
|
||||
path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
|
||||
|
|
@ -285,7 +290,7 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
repo = gh_client.get_repo(source)
|
||||
master_branch = repo.default_branch or 'master'
|
||||
return 'https://github.com/%s/blob/%s/%s' % (source, master_branch, path)
|
||||
except GithubException as ge:
|
||||
except GithubException:
|
||||
return None
|
||||
|
||||
def load_dockerfile_contents(self, auth_token, config):
|
||||
|
|
@ -373,12 +378,12 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
if branch == repo.default_branch:
|
||||
tags.add('latest')
|
||||
|
||||
logger.debug('Pushing to tags: %s' % tags)
|
||||
logger.debug('Pushing to tags: %s', tags)
|
||||
|
||||
# compute the subdir
|
||||
repo_subdir = config['subdir']
|
||||
joined_subdir = os.path.join(tarball_subdir, repo_subdir)
|
||||
logger.debug('Final subdir: %s' % joined_subdir)
|
||||
logger.debug('Final subdir: %s', joined_subdir)
|
||||
|
||||
# compute the metadata
|
||||
metadata = {
|
||||
|
|
@ -417,7 +422,7 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
except:
|
||||
regex = re.compile('.*')
|
||||
|
||||
if not self.matches_ref(ref, regex):
|
||||
if not GithubBuildTrigger.matches_ref(ref, regex):
|
||||
raise SkipRequestException()
|
||||
|
||||
if should_skip_commit(commit_message):
|
||||
|
|
@ -436,7 +441,7 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
return GithubBuildTrigger._prepare_build(config, repo, commit_sha,
|
||||
short_sha, ref)
|
||||
|
||||
def manual_start(self, auth_token, config, run_parameters = None):
|
||||
def manual_start(self, auth_token, config, run_parameters=None):
|
||||
try:
|
||||
source = config['build_source']
|
||||
run_parameters = run_parameters or {}
|
||||
|
|
@ -446,7 +451,6 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
branch_name = run_parameters.get('branch_name') or repo.default_branch
|
||||
branch = repo.get_branch(branch_name)
|
||||
branch_sha = branch.commit.sha
|
||||
commit_info = branch.commit
|
||||
short_sha = GithubBuildTrigger.get_display_name(branch_sha)
|
||||
ref = 'refs/heads/%s' % (branch_name)
|
||||
|
||||
|
|
@ -455,6 +459,375 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
raise TriggerStartException(ghe.data['message'])
|
||||
|
||||
|
||||
def list_field_values(self, auth_token, config, field_name):
|
||||
if field_name == 'refs':
|
||||
branches = self.list_field_values(auth_token, config, 'branch_name')
|
||||
tags = self.list_field_values(auth_token, config, 'tag_name')
|
||||
|
||||
return ([{'kind': 'branch', 'name': b} for b in branches] +
|
||||
[{'kind': 'tag', 'name': tag} for tag in tags])
|
||||
|
||||
if field_name == 'tag_name':
|
||||
gh_client = self._get_client(auth_token)
|
||||
source = config['build_source']
|
||||
repo = gh_client.get_repo(source)
|
||||
return [tag.name for tag in repo.get_tags()]
|
||||
|
||||
if field_name == 'branch_name':
|
||||
gh_client = self._get_client(auth_token)
|
||||
source = config['build_source']
|
||||
repo = gh_client.get_repo(source)
|
||||
branches = [branch.name for branch in repo.get_branches()]
|
||||
|
||||
if not repo.default_branch in branches:
|
||||
branches.insert(0, repo.default_branch)
|
||||
|
||||
if branches[0] != repo.default_branch:
|
||||
branches.remove(repo.default_branch)
|
||||
branches.insert(0, repo.default_branch)
|
||||
|
||||
return branches
|
||||
|
||||
return None
|
||||
|
||||
class GitHubBuildTrigger(BuildTrigger):
|
||||
"""
|
||||
BuildTrigger for GitHub that uses deploy keys and git.
|
||||
"""
|
||||
@staticmethod
|
||||
def _get_client(auth_token):
|
||||
return Github(auth_token,
|
||||
base_url=github_trigger.api_endpoint(),
|
||||
client_id=github_trigger.client_id(),
|
||||
client_secret=github_trigger.client_secret())
|
||||
|
||||
@classmethod
|
||||
def service_name(cls):
|
||||
return 'github-git'
|
||||
|
||||
def is_active(self, config):
|
||||
return 'hook_id' in config and 'deploy_key_id' in config
|
||||
|
||||
def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
|
||||
new_build_source = config['build_source']
|
||||
gh_client = self._get_client(auth_token)
|
||||
|
||||
# Find the GitHub repository.
|
||||
try:
|
||||
gh_repo = gh_client.get_repo(new_build_source)
|
||||
except UnknownObjectException:
|
||||
msg = 'Unable to find GitHub repository for source: %s' % new_build_source
|
||||
raise TriggerActivationException(msg)
|
||||
|
||||
# Generate an SSH keypair and add the public key to the repository.
|
||||
config['public_key'], config['private_key'] = generate_ssh_keypair()
|
||||
try:
|
||||
deploy_key = gh_repo.create_key('Quay.io Builder', config['public_key'])
|
||||
config['deploy_key_id'] = deploy_key.id
|
||||
except GithubException:
|
||||
msg = 'Unable to add deploy key to repository: %s' % new_build_source
|
||||
raise TriggerActivationException(msg)
|
||||
|
||||
# Create a webhook config.
|
||||
webhook_config = {
|
||||
'url': standard_webhook_url,
|
||||
'content_type': 'json',
|
||||
}
|
||||
|
||||
# Add the webhook to the GitHub repository.
|
||||
try:
|
||||
hook = gh_repo.create_hook('web', webhook_config)
|
||||
config['hook_id'] = hook.id
|
||||
config['master_branch'] = gh_repo.default_branch
|
||||
except GithubException:
|
||||
msg = 'Unable to create webhook on repository: %s' % new_build_source
|
||||
raise TriggerActivationException(msg)
|
||||
|
||||
return config
|
||||
|
||||
def deactivate(self, auth_token, config):
|
||||
gh_client = self._get_client(auth_token)
|
||||
|
||||
# Find the GitHub repository.
|
||||
try:
|
||||
repo = gh_client.get_repo(config['build_source'])
|
||||
except UnknownObjectException:
|
||||
msg = 'Unable to find GitHub repository for source: %s' % config['build_source']
|
||||
raise TriggerDeactivationException(msg)
|
||||
|
||||
# Remove the deploy key.
|
||||
try:
|
||||
deploy_key = repo.get_key(config['deploy_key_id'])
|
||||
deploy_key.delete()
|
||||
except GithubException:
|
||||
msg = 'Unable to remove deploy key to repository: %s' % config['build_source']
|
||||
raise TriggerActivationException(msg)
|
||||
config.pop('deploy_key_id', None)
|
||||
|
||||
# Remove the webhook.
|
||||
try:
|
||||
hook = repo.get_hook(config['hook_id'])
|
||||
hook.delete()
|
||||
except GithubException:
|
||||
msg = 'Unable to remove hook: %s' % config['hook_id']
|
||||
raise TriggerDeactivationException(msg)
|
||||
config.pop('hook_id', None)
|
||||
|
||||
return config
|
||||
|
||||
def list_build_sources(self, auth_token):
|
||||
gh_client = self._get_client(auth_token)
|
||||
usr = gh_client.get_user()
|
||||
|
||||
personal = {
|
||||
'personal': True,
|
||||
'repos': [repo.full_name for repo in usr.get_repos()],
|
||||
'info': {
|
||||
'name': usr.login,
|
||||
'avatar_url': usr.avatar_url,
|
||||
}
|
||||
}
|
||||
|
||||
repos_by_org = [personal]
|
||||
|
||||
for org in usr.get_orgs():
|
||||
repo_list = []
|
||||
for repo in org.get_repos(type='member'):
|
||||
repo_list.append(repo.full_name)
|
||||
|
||||
repos_by_org.append({
|
||||
'personal': False,
|
||||
'repos': repo_list,
|
||||
'info': {
|
||||
'name': org.name or org.login,
|
||||
'avatar_url': org.avatar_url
|
||||
}
|
||||
})
|
||||
|
||||
return repos_by_org
|
||||
|
||||
@staticmethod
|
||||
def matches_ref(ref, regex):
|
||||
match_string = ref.split('/', 1)[1]
|
||||
if not regex:
|
||||
return False
|
||||
|
||||
m = regex.match(match_string)
|
||||
if not m:
|
||||
return False
|
||||
|
||||
return len(m.group(0)) == len(match_string)
|
||||
|
||||
def list_build_subdirs(self, auth_token, config):
|
||||
gh_client = self._get_client(auth_token)
|
||||
source = config['build_source']
|
||||
|
||||
try:
|
||||
repo = gh_client.get_repo(source)
|
||||
|
||||
# Find the first matching branch.
|
||||
branches = None
|
||||
if 'branchtag_regex' in config:
|
||||
try:
|
||||
regex = re.compile(config['branchtag_regex'])
|
||||
branches = [branch.name for branch in repo.get_branches()
|
||||
if GitHubBuildTrigger.matches_ref('refs/heads/' + branch.name, regex)]
|
||||
except:
|
||||
pass
|
||||
|
||||
branches = branches or [repo.default_branch or 'master']
|
||||
default_commit = repo.get_branch(branches[0]).commit
|
||||
commit_tree = repo.get_git_tree(default_commit.sha, recursive=True)
|
||||
|
||||
return [os.path.dirname(elem.path) for elem in commit_tree.tree
|
||||
if (elem.type == u'blob' and
|
||||
os.path.basename(elem.path) == u'Dockerfile')]
|
||||
except GithubException as ge:
|
||||
message = ge.data.get('message', 'Unable to list contents of repository: %s' % source)
|
||||
if message == 'Branch not found':
|
||||
raise EmptyRepositoryException()
|
||||
|
||||
raise RepositoryReadException(message)
|
||||
|
||||
def dockerfile_url(self, auth_token, config):
|
||||
source = config['build_source']
|
||||
subdirectory = config.get('subdir', '')
|
||||
path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
|
||||
|
||||
gh_client = self._get_client(auth_token)
|
||||
try:
|
||||
repo = gh_client.get_repo(source)
|
||||
master_branch = repo.default_branch or 'master'
|
||||
return 'https://github.com/%s/blob/%s/%s' % (source, master_branch, path)
|
||||
except GithubException:
|
||||
return None
|
||||
|
||||
def load_dockerfile_contents(self, auth_token, config):
|
||||
gh_client = self._get_client(auth_token)
|
||||
|
||||
source = config['build_source']
|
||||
subdirectory = config.get('subdir', '')
|
||||
path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
|
||||
|
||||
try:
|
||||
repo = gh_client.get_repo(source)
|
||||
file_info = repo.get_file_contents(path)
|
||||
if file_info is None:
|
||||
return None
|
||||
|
||||
content = file_info.content
|
||||
if file_info.encoding == 'base64':
|
||||
content = base64.b64decode(content)
|
||||
return content
|
||||
|
||||
except GithubException as ge:
|
||||
message = ge.data.get('message', 'Unable to read Dockerfile: %s' % source)
|
||||
raise RepositoryReadException(message)
|
||||
|
||||
@staticmethod
|
||||
def _build_commit_info(repo, commit_sha):
|
||||
try:
|
||||
commit = repo.get_commit(commit_sha)
|
||||
except GithubException:
|
||||
logger.exception('Could not load data for commit')
|
||||
return
|
||||
|
||||
return {
|
||||
'url': commit.html_url,
|
||||
'message': commit.commit.message,
|
||||
'author': {
|
||||
'username': commit.author.login,
|
||||
'avatar_url': commit.author.avatar_url,
|
||||
'url': commit.author.html_url
|
||||
},
|
||||
'committer': {
|
||||
'username': commit.committer.login,
|
||||
'avatar_url': commit.committer.avatar_url,
|
||||
'url': commit.committer.html_url
|
||||
},
|
||||
'date': commit.last_modified
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _prepare_build(config, repo, commit_sha, build_name, ref):
|
||||
# Prepare the download and upload URLs
|
||||
archive_link = repo.get_archive_link('tarball', commit_sha)
|
||||
download_archive = client.get(archive_link, stream=True)
|
||||
|
||||
tarball_subdir = ''
|
||||
with SpooledTemporaryFile(CHUNK_SIZE) as tarball:
|
||||
for chunk in download_archive.iter_content(CHUNK_SIZE):
|
||||
tarball.write(chunk)
|
||||
|
||||
# Seek to position 0 to make tarfile happy
|
||||
tarball.seek(0)
|
||||
|
||||
# Pull out the name of the subdir that GitHub generated
|
||||
with tarfile.open(fileobj=tarball) as archive:
|
||||
tarball_subdir = archive.getnames()[0]
|
||||
|
||||
# Seek to position 0 to make tarfile happy.
|
||||
tarball.seek(0)
|
||||
|
||||
entries = {
|
||||
tarball_subdir + '/.git/HEAD': commit_sha,
|
||||
tarball_subdir + '/.git/objects/': None,
|
||||
tarball_subdir + '/.git/refs/': None
|
||||
}
|
||||
|
||||
appender = TarfileAppender(tarball, entries).get_stream()
|
||||
dockerfile_id = user_files.store_file(appender, TARBALL_MIME)
|
||||
|
||||
logger.debug('Successfully prepared job')
|
||||
|
||||
# compute the tag(s)
|
||||
branch = ref.split('/')[-1]
|
||||
tags = {branch}
|
||||
|
||||
if branch == repo.default_branch:
|
||||
tags.add('latest')
|
||||
|
||||
logger.debug('Pushing to tags: %s', tags)
|
||||
|
||||
# compute the subdir
|
||||
repo_subdir = config['subdir']
|
||||
joined_subdir = os.path.join(tarball_subdir, repo_subdir)
|
||||
logger.debug('Final subdir: %s', joined_subdir)
|
||||
|
||||
# compute the metadata
|
||||
metadata = {
|
||||
'commit_sha': commit_sha,
|
||||
'ref': ref,
|
||||
'default_branch': repo.default_branch,
|
||||
}
|
||||
|
||||
# add the commit info.
|
||||
commit_info = GitHubBuildTrigger._build_commit_info(repo, commit_sha)
|
||||
if commit_info is not None:
|
||||
metadata['commit_info'] = commit_info
|
||||
|
||||
return dockerfile_id, list(tags), build_name, joined_subdir, metadata
|
||||
|
||||
@staticmethod
|
||||
def get_display_name(sha):
|
||||
return sha[0:7]
|
||||
|
||||
def handle_trigger_request(self, request, auth_token, config):
|
||||
payload = request.get_json()
|
||||
if not payload or payload.get('head_commit') is None:
|
||||
raise SkipRequestException()
|
||||
|
||||
if 'zen' in payload:
|
||||
raise ValidationRequestException()
|
||||
|
||||
logger.debug('Payload %s', payload)
|
||||
ref = payload['ref']
|
||||
commit_sha = payload['head_commit']['id']
|
||||
commit_message = payload['head_commit'].get('message', '')
|
||||
|
||||
if 'branchtag_regex' in config:
|
||||
try:
|
||||
regex = re.compile(config['branchtag_regex'])
|
||||
except:
|
||||
regex = re.compile('.*')
|
||||
|
||||
if not GitHubBuildTrigger.matches_ref(ref, regex):
|
||||
raise SkipRequestException()
|
||||
|
||||
if should_skip_commit(commit_message):
|
||||
raise SkipRequestException()
|
||||
|
||||
short_sha = GitHubBuildTrigger.get_display_name(commit_sha)
|
||||
|
||||
gh_client = self._get_client(auth_token)
|
||||
|
||||
repo_full_name = '%s/%s' % (payload['repository']['owner']['name'],
|
||||
payload['repository']['name'])
|
||||
repo = gh_client.get_repo(repo_full_name)
|
||||
|
||||
logger.debug('Github repo: %s', repo)
|
||||
|
||||
return GitHubBuildTrigger._prepare_build(config, repo, commit_sha,
|
||||
short_sha, ref)
|
||||
|
||||
def manual_start(self, auth_token, config, run_parameters=None):
|
||||
try:
|
||||
source = config['build_source']
|
||||
run_parameters = run_parameters or {}
|
||||
|
||||
gh_client = self._get_client(auth_token)
|
||||
repo = gh_client.get_repo(source)
|
||||
branch_name = run_parameters.get('branch_name') or repo.default_branch
|
||||
branch = repo.get_branch(branch_name)
|
||||
branch_sha = branch.commit.sha
|
||||
short_sha = GitHubBuildTrigger.get_display_name(branch_sha)
|
||||
ref = 'refs/heads/%s' % (branch_name)
|
||||
|
||||
return self._prepare_build(config, repo, branch_sha, short_sha, ref)
|
||||
except GithubException as ghe:
|
||||
raise TriggerStartException(ghe.data['message'])
|
||||
|
||||
|
||||
def list_field_values(self, auth_token, config, field_name):
|
||||
if field_name == 'refs':
|
||||
branches = self.list_field_values(auth_token, config, 'branch_name')
|
||||
|
|
|
|||
Reference in a new issue