consolidate everything into one GitHub trigger
This commit is contained in:
Jimmy Zelinskie
parent
80ae3fd310
commit
f6f93e9079
6 changed files with 42 additions and 426 deletions
|
|
@ -248,8 +248,6 @@ def github_oauth_attach():
|
|||
|
||||
@callback.route('/github/callback/trigger/<path:repository>', methods=['GET'])
|
||||
@callback.route('/github/callback/trigger/<path:repository>/__new', methods=['GET'])
|
||||
@callback.route('/github/callback/trigger/<path:repostiory>/__git', methods=['GET'])
|
||||
@callback.route('/github/callback/trigger/<path:repostiory>/__git/__new', methods=['GET'])
|
||||
@route_show_if(features.GITHUB_BUILD)
|
||||
@require_session_login
|
||||
@parse_repository_name
|
||||
|
|
@ -262,8 +260,7 @@ def attach_github_build_trigger(namespace, repository):
|
|||
msg = 'Invalid repository: %s/%s' % (namespace, repository)
|
||||
abort(404, message=msg)
|
||||
|
||||
service_name = 'github-git' if '__git' in request.url else 'github'
|
||||
trigger = model.create_build_trigger(repo, service_name, token, current_user.db_user())
|
||||
trigger = model.create_build_trigger(repo, 'github', token, current_user.db_user())
|
||||
|
||||
# TODO(jschorr): Remove once the new layout is in place.
|
||||
admin_path = '%s/%s/%s' % (namespace, repository, 'admin')
|
||||
|
|
|
|||
|
|
@ -169,34 +169,61 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
new_build_source = config['build_source']
|
||||
gh_client = self._get_client(auth_token)
|
||||
|
||||
# Find the GitHub repository.
|
||||
try:
|
||||
to_add_webhook = gh_client.get_repo(new_build_source)
|
||||
gh_repo = gh_client.get_repo(new_build_source)
|
||||
except UnknownObjectException:
|
||||
msg = 'Unable to find GitHub repository for source: %s'
|
||||
raise TriggerActivationException(msg % new_build_source)
|
||||
msg = 'Unable to find GitHub repository for source: %s' % new_build_source
|
||||
raise TriggerActivationException(msg)
|
||||
|
||||
# Add a deploy key to the GitHub repository.
|
||||
try:
|
||||
deploy_key = gh_repo.create_key('Quay.io Builder', config['public_key'])
|
||||
config['deploy_key_id'] = deploy_key.id
|
||||
except GithubException:
|
||||
msg = 'Unable to add deploy key to repository: %s' % new_build_source
|
||||
raise TriggerActivationException(msg)
|
||||
|
||||
# Create a webhook config.
|
||||
webhook_config = {
|
||||
'url': standard_webhook_url,
|
||||
'content_type': 'json',
|
||||
}
|
||||
|
||||
# Add the webhook to the GitHub repository.
|
||||
try:
|
||||
hook = to_add_webhook.create_hook('web', webhook_config)
|
||||
hook = gh_repo.create_hook('web', webhook_config)
|
||||
config['hook_id'] = hook.id
|
||||
config['master_branch'] = to_add_webhook.default_branch
|
||||
config['master_branch'] = gh_repo.default_branch
|
||||
except GithubException:
|
||||
msg = 'Unable to create webhook on repository: %s'
|
||||
raise TriggerActivationException(msg % new_build_source)
|
||||
msg = 'Unable to create webhook on repository: %s' % new_build_source
|
||||
raise TriggerActivationException(msg)
|
||||
|
||||
return config
|
||||
|
||||
def deactivate(self, auth_token, config):
|
||||
gh_client = self._get_client(auth_token)
|
||||
|
||||
# Find the GitHub repository.
|
||||
try:
|
||||
repo = gh_client.get_repo(config['build_source'])
|
||||
to_delete = repo.get_hook(config['hook_id'])
|
||||
to_delete.delete()
|
||||
except UnknownObjectException:
|
||||
msg = 'Unable to find GitHub repository for source: %s' % config['build_source']
|
||||
raise TriggerDeactivationException(msg)
|
||||
|
||||
# If the trigger uses a deploy key, remove it.
|
||||
if config['deploy_key_id']:
|
||||
try:
|
||||
deploy_key = repo.get_key(config['deploy_key_id'])
|
||||
deploy_key.delete()
|
||||
except GithubException:
|
||||
msg = 'Unable to remove deploy key: %s' % config['deploy_key_id']
|
||||
raise TriggerDeactivationException(msg)
|
||||
|
||||
# Remove the webhook.
|
||||
try:
|
||||
hook = repo.get_hook(config['hook_id'])
|
||||
hook.delete()
|
||||
except GithubException:
|
||||
msg = 'Unable to remove hook: %s' % config['hook_id']
|
||||
raise TriggerDeactivationException(msg)
|
||||
|
|
@ -283,8 +310,8 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
source = config['build_source']
|
||||
subdirectory = config.get('subdir', '')
|
||||
path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
|
||||
|
||||
gh_client = self._get_client(auth_token)
|
||||
|
||||
try:
|
||||
repo = gh_client.get_repo(source)
|
||||
master_branch = repo.default_branch or 'master'
|
||||
|
|
@ -464,374 +491,6 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
raise TriggerStartException(ghe.data['message'])
|
||||
|
||||
|
||||
def list_field_values(self, auth_token, config, field_name):
|
||||
if field_name == 'refs':
|
||||
branches = self.list_field_values(auth_token, config, 'branch_name')
|
||||
tags = self.list_field_values(auth_token, config, 'tag_name')
|
||||
|
||||
return ([{'kind': 'branch', 'name': b} for b in branches] +
|
||||
[{'kind': 'tag', 'name': tag} for tag in tags])
|
||||
|
||||
if field_name == 'tag_name':
|
||||
gh_client = self._get_client(auth_token)
|
||||
source = config['build_source']
|
||||
repo = gh_client.get_repo(source)
|
||||
return [tag.name for tag in repo.get_tags()]
|
||||
|
||||
if field_name == 'branch_name':
|
||||
gh_client = self._get_client(auth_token)
|
||||
source = config['build_source']
|
||||
repo = gh_client.get_repo(source)
|
||||
branches = [branch.name for branch in repo.get_branches()]
|
||||
|
||||
if not repo.default_branch in branches:
|
||||
branches.insert(0, repo.default_branch)
|
||||
|
||||
if branches[0] != repo.default_branch:
|
||||
branches.remove(repo.default_branch)
|
||||
branches.insert(0, repo.default_branch)
|
||||
|
||||
return branches
|
||||
|
||||
return None
|
||||
|
||||
class GitHubBuildTrigger(BuildTrigger):
|
||||
"""
|
||||
BuildTrigger for GitHub that uses deploy keys and git.
|
||||
"""
|
||||
@staticmethod
|
||||
def _get_client(auth_token):
|
||||
return Github(auth_token,
|
||||
base_url=github_trigger.api_endpoint(),
|
||||
client_id=github_trigger.client_id(),
|
||||
client_secret=github_trigger.client_secret())
|
||||
|
||||
@classmethod
|
||||
def service_name(cls):
|
||||
return 'github-git'
|
||||
|
||||
def is_active(self, config):
|
||||
return 'hook_id' in config and 'deploy_key_id' in config
|
||||
|
||||
def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
|
||||
new_build_source = config['build_source']
|
||||
gh_client = self._get_client(auth_token)
|
||||
|
||||
# Find the GitHub repository.
|
||||
try:
|
||||
gh_repo = gh_client.get_repo(new_build_source)
|
||||
except UnknownObjectException:
|
||||
msg = 'Unable to find GitHub repository for source: %s' % new_build_source
|
||||
raise TriggerActivationException(msg)
|
||||
|
||||
# Add a deploy key to the GitHub repository.
|
||||
try:
|
||||
deploy_key = gh_repo.create_key('Quay.io Builder', config['public_key'])
|
||||
config['deploy_key_id'] = deploy_key.id
|
||||
except GithubException:
|
||||
msg = 'Unable to add deploy key to repository: %s' % new_build_source
|
||||
raise TriggerActivationException(msg)
|
||||
|
||||
# Create a webhook config.
|
||||
webhook_config = {
|
||||
'url': standard_webhook_url,
|
||||
'content_type': 'json',
|
||||
}
|
||||
|
||||
# Add the webhook to the GitHub repository.
|
||||
try:
|
||||
hook = gh_repo.create_hook('web', webhook_config)
|
||||
config['hook_id'] = hook.id
|
||||
config['master_branch'] = gh_repo.default_branch
|
||||
except GithubException:
|
||||
msg = 'Unable to create webhook on repository: %s' % new_build_source
|
||||
raise TriggerActivationException(msg)
|
||||
|
||||
return config
|
||||
|
||||
def deactivate(self, auth_token, config):
|
||||
gh_client = self._get_client(auth_token)
|
||||
|
||||
# Find the GitHub repository.
|
||||
try:
|
||||
repo = gh_client.get_repo(config['build_source'])
|
||||
except UnknownObjectException:
|
||||
msg = 'Unable to find GitHub repository for source: %s' % config['build_source']
|
||||
raise TriggerDeactivationException(msg)
|
||||
|
||||
# Remove the deploy key.
|
||||
try:
|
||||
deploy_key = repo.get_key(config['deploy_key_id'])
|
||||
deploy_key.delete()
|
||||
except GithubException:
|
||||
msg = 'Unable to remove deploy key to repository: %s' % config['build_source']
|
||||
raise TriggerActivationException(msg)
|
||||
config.pop('deploy_key_id', None)
|
||||
|
||||
# Remove the webhook.
|
||||
try:
|
||||
hook = repo.get_hook(config['hook_id'])
|
||||
hook.delete()
|
||||
except GithubException:
|
||||
msg = 'Unable to remove hook: %s' % config['hook_id']
|
||||
raise TriggerDeactivationException(msg)
|
||||
config.pop('hook_id', None)
|
||||
|
||||
return config
|
||||
|
||||
def list_build_sources(self, auth_token):
|
||||
gh_client = self._get_client(auth_token)
|
||||
usr = gh_client.get_user()
|
||||
|
||||
personal = {
|
||||
'personal': True,
|
||||
'repos': [repo.full_name for repo in usr.get_repos()],
|
||||
'info': {
|
||||
'name': usr.login,
|
||||
'avatar_url': usr.avatar_url,
|
||||
}
|
||||
}
|
||||
|
||||
repos_by_org = [personal]
|
||||
|
||||
for org in usr.get_orgs():
|
||||
repo_list = []
|
||||
for repo in org.get_repos(type='member'):
|
||||
repo_list.append(repo.full_name)
|
||||
|
||||
repos_by_org.append({
|
||||
'personal': False,
|
||||
'repos': repo_list,
|
||||
'info': {
|
||||
'name': org.name or org.login,
|
||||
'avatar_url': org.avatar_url
|
||||
}
|
||||
})
|
||||
|
||||
return repos_by_org
|
||||
|
||||
@staticmethod
|
||||
def matches_ref(ref, regex):
|
||||
match_string = ref.split('/', 1)[1]
|
||||
if not regex:
|
||||
return False
|
||||
|
||||
m = regex.match(match_string)
|
||||
if not m:
|
||||
return False
|
||||
|
||||
return len(m.group(0)) == len(match_string)
|
||||
|
||||
def list_build_subdirs(self, auth_token, config):
|
||||
gh_client = self._get_client(auth_token)
|
||||
source = config['build_source']
|
||||
|
||||
try:
|
||||
repo = gh_client.get_repo(source)
|
||||
|
||||
# Find the first matching branch.
|
||||
branches = None
|
||||
if 'branchtag_regex' in config:
|
||||
try:
|
||||
regex = re.compile(config['branchtag_regex'])
|
||||
branches = [branch.name for branch in repo.get_branches()
|
||||
if GitHubBuildTrigger.matches_ref('refs/heads/' + branch.name, regex)]
|
||||
except:
|
||||
pass
|
||||
|
||||
branches = branches or [repo.default_branch or 'master']
|
||||
default_commit = repo.get_branch(branches[0]).commit
|
||||
commit_tree = repo.get_git_tree(default_commit.sha, recursive=True)
|
||||
|
||||
return [os.path.dirname(elem.path) for elem in commit_tree.tree
|
||||
if (elem.type == u'blob' and
|
||||
os.path.basename(elem.path) == u'Dockerfile')]
|
||||
except GithubException as ge:
|
||||
message = ge.data.get('message', 'Unable to list contents of repository: %s' % source)
|
||||
if message == 'Branch not found':
|
||||
raise EmptyRepositoryException()
|
||||
|
||||
raise RepositoryReadException(message)
|
||||
|
||||
def dockerfile_url(self, auth_token, config):
|
||||
source = config['build_source']
|
||||
subdirectory = config.get('subdir', '')
|
||||
path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
|
||||
|
||||
gh_client = self._get_client(auth_token)
|
||||
try:
|
||||
repo = gh_client.get_repo(source)
|
||||
master_branch = repo.default_branch or 'master'
|
||||
return 'https://github.com/%s/blob/%s/%s' % (source, master_branch, path)
|
||||
except GithubException:
|
||||
return None
|
||||
|
||||
def load_dockerfile_contents(self, auth_token, config):
|
||||
gh_client = self._get_client(auth_token)
|
||||
|
||||
source = config['build_source']
|
||||
subdirectory = config.get('subdir', '')
|
||||
path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
|
||||
|
||||
try:
|
||||
repo = gh_client.get_repo(source)
|
||||
file_info = repo.get_file_contents(path)
|
||||
if file_info is None:
|
||||
return None
|
||||
|
||||
content = file_info.content
|
||||
if file_info.encoding == 'base64':
|
||||
content = base64.b64decode(content)
|
||||
return content
|
||||
|
||||
except GithubException as ge:
|
||||
message = ge.data.get('message', 'Unable to read Dockerfile: %s' % source)
|
||||
raise RepositoryReadException(message)
|
||||
|
||||
@staticmethod
|
||||
def _build_commit_info(repo, commit_sha):
|
||||
try:
|
||||
commit = repo.get_commit(commit_sha)
|
||||
except GithubException:
|
||||
logger.exception('Could not load data for commit')
|
||||
return
|
||||
|
||||
return {
|
||||
'url': commit.html_url,
|
||||
'message': commit.commit.message,
|
||||
'author': {
|
||||
'username': commit.author.login,
|
||||
'avatar_url': commit.author.avatar_url,
|
||||
'url': commit.author.html_url
|
||||
},
|
||||
'committer': {
|
||||
'username': commit.committer.login,
|
||||
'avatar_url': commit.committer.avatar_url,
|
||||
'url': commit.committer.html_url
|
||||
},
|
||||
'date': commit.last_modified
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _prepare_build(config, repo, commit_sha, build_name, ref):
|
||||
# Prepare the download and upload URLs
|
||||
archive_link = repo.get_archive_link('tarball', commit_sha)
|
||||
download_archive = client.get(archive_link, stream=True)
|
||||
|
||||
tarball_subdir = ''
|
||||
with SpooledTemporaryFile(CHUNK_SIZE) as tarball:
|
||||
for chunk in download_archive.iter_content(CHUNK_SIZE):
|
||||
tarball.write(chunk)
|
||||
|
||||
# Seek to position 0 to make tarfile happy
|
||||
tarball.seek(0)
|
||||
|
||||
# Pull out the name of the subdir that GitHub generated
|
||||
with tarfile.open(fileobj=tarball) as archive:
|
||||
tarball_subdir = archive.getnames()[0]
|
||||
|
||||
# Seek to position 0 to make tarfile happy.
|
||||
tarball.seek(0)
|
||||
|
||||
entries = {
|
||||
tarball_subdir + '/.git/HEAD': commit_sha,
|
||||
tarball_subdir + '/.git/objects/': None,
|
||||
tarball_subdir + '/.git/refs/': None
|
||||
}
|
||||
|
||||
appender = TarfileAppender(tarball, entries).get_stream()
|
||||
dockerfile_id = user_files.store_file(appender, TARBALL_MIME)
|
||||
|
||||
logger.debug('Successfully prepared job')
|
||||
|
||||
# compute the tag(s)
|
||||
branch = ref.split('/')[-1]
|
||||
tags = {branch}
|
||||
|
||||
if branch == repo.default_branch:
|
||||
tags.add('latest')
|
||||
|
||||
logger.debug('Pushing to tags: %s', tags)
|
||||
|
||||
# compute the subdir
|
||||
repo_subdir = config['subdir']
|
||||
joined_subdir = os.path.join(tarball_subdir, repo_subdir)
|
||||
logger.debug('Final subdir: %s', joined_subdir)
|
||||
|
||||
# compute the metadata
|
||||
metadata = {
|
||||
'commit_sha': commit_sha,
|
||||
'ref': ref,
|
||||
'default_branch': repo.default_branch,
|
||||
}
|
||||
|
||||
# add the commit info.
|
||||
commit_info = GitHubBuildTrigger._build_commit_info(repo, commit_sha)
|
||||
if commit_info is not None:
|
||||
metadata['commit_info'] = commit_info
|
||||
|
||||
return dockerfile_id, list(tags), build_name, joined_subdir, metadata
|
||||
|
||||
@staticmethod
|
||||
def get_display_name(sha):
|
||||
return sha[0:7]
|
||||
|
||||
def handle_trigger_request(self, request, auth_token, config):
|
||||
payload = request.get_json()
|
||||
if not payload or payload.get('head_commit') is None:
|
||||
raise SkipRequestException()
|
||||
|
||||
if 'zen' in payload:
|
||||
raise ValidationRequestException()
|
||||
|
||||
logger.debug('Payload %s', payload)
|
||||
ref = payload['ref']
|
||||
commit_sha = payload['head_commit']['id']
|
||||
commit_message = payload['head_commit'].get('message', '')
|
||||
|
||||
if 'branchtag_regex' in config:
|
||||
try:
|
||||
regex = re.compile(config['branchtag_regex'])
|
||||
except:
|
||||
regex = re.compile('.*')
|
||||
|
||||
if not GitHubBuildTrigger.matches_ref(ref, regex):
|
||||
raise SkipRequestException()
|
||||
|
||||
if should_skip_commit(commit_message):
|
||||
raise SkipRequestException()
|
||||
|
||||
short_sha = GitHubBuildTrigger.get_display_name(commit_sha)
|
||||
|
||||
gh_client = self._get_client(auth_token)
|
||||
|
||||
repo_full_name = '%s/%s' % (payload['repository']['owner']['name'],
|
||||
payload['repository']['name'])
|
||||
repo = gh_client.get_repo(repo_full_name)
|
||||
|
||||
logger.debug('Github repo: %s', repo)
|
||||
|
||||
return GitHubBuildTrigger._prepare_build(config, repo, commit_sha,
|
||||
short_sha, ref)
|
||||
|
||||
def manual_start(self, auth_token, config, run_parameters=None):
|
||||
try:
|
||||
source = config['build_source']
|
||||
run_parameters = run_parameters or {}
|
||||
|
||||
gh_client = self._get_client(auth_token)
|
||||
repo = gh_client.get_repo(source)
|
||||
branch_name = run_parameters.get('branch_name') or repo.default_branch
|
||||
branch = repo.get_branch(branch_name)
|
||||
branch_sha = branch.commit.sha
|
||||
short_sha = GitHubBuildTrigger.get_display_name(branch_sha)
|
||||
ref = 'refs/heads/%s' % (branch_name)
|
||||
|
||||
return self._prepare_build(config, repo, branch_sha, short_sha, ref)
|
||||
except GithubException as ghe:
|
||||
raise TriggerStartException(ghe.data['message'])
|
||||
|
||||
|
||||
def list_field_values(self, auth_token, config, field_name):
|
||||
if field_name == 'refs':
|
||||
branches = self.list_field_values(auth_token, config, 'branch_name')
|
||||
|
|
|
|||
|
|
@ -195,7 +195,6 @@ def initialize_database():
|
|||
LoginService.create(name='ldap')
|
||||
|
||||
BuildTriggerService.create(name='github')
|
||||
BuildTriggerService.create(name='github-git')
|
||||
|
||||
AccessTokenKind.create(name='build-worker')
|
||||
AccessTokenKind.create(name='pushpull-token')
|
||||
|
|
|
|||
|
|
@ -8,45 +8,6 @@ angular.module('quay').factory('TriggerService', ['UtilService', '$sanitize', 'K
|
|||
|
||||
var triggerTypes = {
|
||||
'github': {
|
||||
'description': function(config) {
|
||||
var source = UtilService.textToSafeHtml(config['build_source']);
|
||||
var desc = '<i class="fa fa-github fa-lg" style="margin-left: 2px; margin-right: 2px"></i> Push to Github Repository (Deprecated) ';
|
||||
desc += '<a href="https://github.com/' + source + '" target="_blank">' + source + '</a>';
|
||||
desc += '<br>Dockerfile folder: //' + UtilService.textToSafeHtml(config['subdir']);
|
||||
return desc;
|
||||
},
|
||||
'run_parameters': [
|
||||
{
|
||||
'title': 'Branch',
|
||||
'type': 'option',
|
||||
'name': 'branch_name'
|
||||
}
|
||||
],
|
||||
'get_redirect_url': function(namespace, repository) {
|
||||
var redirect_uri = KeyService['githubRedirectUri'] + '/trigger/' +
|
||||
namespace + '/' + repository;
|
||||
|
||||
var authorize_url = KeyService['githubTriggerAuthorizeUrl'];
|
||||
var client_id = KeyService['githubTriggerClientId'];
|
||||
|
||||
return authorize_url + 'client_id=' + client_id +
|
||||
'&scope=repo,user:email&redirect_uri=' + redirect_uri;
|
||||
},
|
||||
'is_enabled': function() {
|
||||
return Features.GITHUB_BUILD;
|
||||
},
|
||||
'icon': 'fa-github',
|
||||
'title': function() {
|
||||
var isEnterprise = KeyService.isEnterprise('github-trigger');
|
||||
if (isEnterprise) {
|
||||
return 'GitHub Enterprise Repository Push (API)';
|
||||
}
|
||||
|
||||
return 'GitHub Repository Push (API)';
|
||||
}
|
||||
},
|
||||
|
||||
'github-git': {
|
||||
'description': function(config) {
|
||||
var source = UtilService.textToSafeHtml(config['build_source']);
|
||||
var desc = '<i class="fa fa-github fa-lg" style="margin-left: 2px; margin-right: 2px"></i> Push to Github Repository ';
|
||||
|
|
@ -63,7 +24,7 @@ angular.module('quay').factory('TriggerService', ['UtilService', '$sanitize', 'K
|
|||
],
|
||||
'get_redirect_url': function(namespace, repository) {
|
||||
var redirect_uri = KeyService['githubRedirectUri'] + '/trigger/' +
|
||||
namespace + '/' + repository + '/__git';
|
||||
namespace + '/' + repository;
|
||||
|
||||
// TODO(jschorr): Remove once the new layout is in place.
|
||||
if (CookieService.get('quay.exp-new-layout') == 'true') {
|
||||
|
|
@ -83,10 +44,10 @@ angular.module('quay').factory('TriggerService', ['UtilService', '$sanitize', 'K
|
|||
'title': function() {
|
||||
var isEnterprise = KeyService.isEnterprise('github-trigger');
|
||||
if (isEnterprise) {
|
||||
return 'GitHub Enterprise Repository Push (Deploy Key)';
|
||||
return 'GitHub Enterprise Repository Push';
|
||||
}
|
||||
|
||||
return 'GitHub Repository Push (Deploy Key)';
|
||||
return 'GitHub Repository Push';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -5,6 +5,6 @@ def generate_ssh_keypair():
|
|||
Generates a new 2048 bit RSA public key in OpenSSH format and private key in PEM format.
|
||||
"""
|
||||
key = RSA.generate(2048)
|
||||
public_key = key.publicKey().exportKey('OpenSSH')
|
||||
public_key = key.publickey().exportKey('OpenSSH')
|
||||
private_key = key.exportKey('PEM')
|
||||
return (public_key, private_key)
|
||||
|
|
|
|||
Reference in a new issue