import logging import io import os.path import tarfile import base64 from StringIO import StringIO from github import Github, UnknownObjectException, GithubException from tempfile import SpooledTemporaryFile from app import app, userfiles as user_files client = app.config['HTTPCLIENT'] logger = logging.getLogger(__name__) TARBALL_MIME = 'application/gzip' CHUNK_SIZE = 512 * 1024 def should_skip_commit(message): return '[skip build]' in message or '[build skip]' in message class BuildArchiveException(Exception): pass class InvalidServiceException(Exception): pass class TriggerActivationException(Exception): pass class TriggerDeactivationException(Exception): pass class TriggerStartException(Exception): pass class ValidationRequestException(Exception): pass class SkipRequestException(Exception): pass class EmptyRepositoryException(Exception): pass class RepositoryReadException(Exception): pass class BuildTrigger(object): def __init__(self): pass def dockerfile_url(self, auth_token, config): """ Returns the URL at which the Dockerfile for the trigger can be found or None if none/not applicable. """ return None def load_dockerfile_contents(self, auth_token, config): """ Loads the Dockerfile found for the trigger's config and returns them or None if none could be found/loaded. """ return None def list_build_sources(self, auth_token): """ Take the auth information for the specific trigger type and load the list of build sources(repositories). """ raise NotImplementedError def list_build_subdirs(self, auth_token, config): """ Take the auth information and the specified config so far and list all of the possible subdirs containing dockerfiles. """ raise NotImplementedError def handle_trigger_request(self, request, auth_token, config): """ Transform the incoming request data into a set of actions. Returns a tuple of usefiles resource id, docker tags, build name, and resource subdir. """ raise NotImplementedError def is_active(self, config): """ Returns True if the current build trigger is active. Inactive means further setup is needed. """ raise NotImplementedError def activate(self, trigger_uuid, standard_webhook_url, auth_token, config): """ Activates the trigger for the service, with the given new configuration. Returns new configuration that should be stored if successful. """ raise NotImplementedError def deactivate(self, auth_token, config): """ Deactivates the trigger for the service, removing any hooks installed in the remote service. Returns the new config that should be stored if this trigger is going to be re-activated. """ raise NotImplementedError def manual_start(self, auth_token, config, run_parameters = None): """ Manually creates a repository build for this trigger. """ raise NotImplementedError def list_field_values(self, auth_token, config, field_name): """ Lists all values for the given custom trigger field. For example, a trigger might have a field named "branches", and this method would return all branches. """ raise NotImplementedError @classmethod def service_name(cls): """ Particular service implemented by subclasses. """ raise NotImplementedError @classmethod def get_trigger_for_service(cls, service): for subc in cls.__subclasses__(): if subc.service_name() == service: return subc() raise InvalidServiceException('Unable to find service: %s' % service) def raise_unsupported(): raise io.UnsupportedOperation class GithubBuildTrigger(BuildTrigger): @staticmethod def _get_client(auth_token): return Github(auth_token, client_id=app.config['GITHUB_CLIENT_ID'], client_secret=app.config['GITHUB_CLIENT_SECRET']) @classmethod def service_name(cls): return 'github' def is_active(self, config): return 'hook_id' in config def activate(self, trigger_uuid, standard_webhook_url, auth_token, config): new_build_source = config['build_source'] gh_client = self._get_client(auth_token) try: to_add_webhook = gh_client.get_repo(new_build_source) except UnknownObjectException: msg = 'Unable to find GitHub repository for source: %s' raise TriggerActivationException(msg % new_build_source) webhook_config = { 'url': standard_webhook_url, 'content_type': 'json', } try: hook = to_add_webhook.create_hook('web', webhook_config) config['hook_id'] = hook.id config['master_branch'] = to_add_webhook.default_branch except GithubException: msg = 'Unable to create webhook on repository: %s' raise TriggerActivationException(msg % new_build_source) return config def deactivate(self, auth_token, config): gh_client = self._get_client(auth_token) try: repo = gh_client.get_repo(config['build_source']) to_delete = repo.get_hook(config['hook_id']) to_delete.delete() except GithubException: msg = 'Unable to remove hook: %s' % config['hook_id'] raise TriggerDeactivationException(msg) config.pop('hook_id', None) return config def list_build_sources(self, auth_token): gh_client = self._get_client(auth_token) usr = gh_client.get_user() personal = { 'personal': True, 'repos': [repo.full_name for repo in usr.get_repos()], 'info': { 'name': usr.login, 'avatar_url': usr.avatar_url, } } repos_by_org = [personal] for org in usr.get_orgs(): repo_list = [] for repo in org.get_repos(type='member'): repo_list.append(repo.full_name) repos_by_org.append({ 'personal': False, 'repos': repo_list, 'info': { 'name': org.name, 'avatar_url': org.avatar_url } }) return repos_by_org def list_build_subdirs(self, auth_token, config): gh_client = self._get_client(auth_token) source = config['build_source'] try: repo = gh_client.get_repo(source) default_commit = repo.get_branch(repo.default_branch or 'master').commit commit_tree = repo.get_git_tree(default_commit.sha, recursive=True) return [os.path.dirname(elem.path) for elem in commit_tree.tree if (elem.type == u'blob' and os.path.basename(elem.path) == u'Dockerfile')] except GithubException as ge: message = ge.data.get('message', 'Unable to list contents of repository: %s' % source) if message == 'Branch not found': raise EmptyRepositoryException() raise RepositoryReadException(message) def dockerfile_url(self, auth_token, config): source = config['build_source'] subdirectory = config.get('subdir', '') path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile' gh_client = self._get_client(auth_token) try: repo = gh_client.get_repo(source) master_branch = repo.default_branch or 'master' return 'https://github.com/%s/blob/%s/%s' % (source, master_branch, path) except GithubException as ge: return None def load_dockerfile_contents(self, auth_token, config): gh_client = self._get_client(auth_token) source = config['build_source'] subdirectory = config.get('subdir', '') path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile' try: repo = gh_client.get_repo(source) file_info = repo.get_file_contents(path) if file_info is None: return None content = file_info.content if file_info.encoding == 'base64': content = base64.b64decode(content) return content except GithubException as ge: message = ge.data.get('message', 'Unable to read Dockerfile: %s' % source) raise RepositoryReadException(message) @staticmethod def _prepare_build(config, repo, commit_sha, build_name, ref): # Prepare the download and upload URLs archive_link = repo.get_archive_link('tarball', commit_sha) download_archive = client.get(archive_link, stream=True) tarball_subdir = '' with SpooledTemporaryFile(CHUNK_SIZE) as tarball: for chunk in download_archive.iter_content(CHUNK_SIZE): tarball.write(chunk) # Seek to position 0 to make tarfile happy tarball.seek(0) # Pull out the name of the subdir that GitHub generated. with tarfile.open(fileobj=tarball) as archive: tarball_subdir = archive.getnames()[0] tarball_subdir_info = archive.getmember(tarball_subdir) # Seek to position 0 to make tarfile happy tarball.seek(0) with SpooledTemporaryFile(CHUNK_SIZE) as updated_tarball: def add_entry(arch, dir_path, base_info, contents=None): info = tarfile.TarInfo(dir_path) info.uid = base_info.uid info.gid = base_info.gid info.uname = base_info.uname info.gname = base_info.gname info.mode = base_info.mode info.mtime = base_info.mtime info.type = tarfile.REGTYPE if contents else tarfile.DIRTYPE if contents: info.size = len(contents) arch.addfile(info, fileobj=StringIO(contents) if contents else None) with tarfile.open(fileobj=updated_tarball, mode='w|gz') as updated_archive: # Copy existing members of the tar to the updated archive. with tarfile.open(fileobj=tarball) as archive: for tar_info in archive: if tar_info.isreg(): updated_archive.addfile(tar_info, archive.extractfile(tar_info.name)) else: updated_archive.addfile(tar_info) # Add the synthetic .git directory to the tarball, containing the commit_sha. add_entry(updated_archive, tarball_subdir + '/.git/HEAD', tarball_subdir_info, contents=commit_sha) add_entry(updated_archive, tarball_subdir + '/.git/objects/', tarball_subdir_info) add_entry(updated_archive, tarball_subdir + '/.git/refs/', tarball_subdir_info) # Seek to position 0 to make boto multipart happy updated_tarball.seek(0) dockerfile_id = user_files.store_file(updated_tarball, TARBALL_MIME) logger.debug('Successfully prepared job') # compute the tag(s) branch = ref.split('/')[-1] tags = {branch} if branch == repo.default_branch: tags.add('latest') logger.debug('Pushing to tags: %s' % tags) # compute the subdir repo_subdir = config['subdir'] joined_subdir = os.path.join(tarball_subdir, repo_subdir) logger.debug('Final subdir: %s' % joined_subdir) return dockerfile_id, list(tags), build_name, joined_subdir @staticmethod def get_display_name(sha): return sha[0:7] def handle_trigger_request(self, request, auth_token, config): payload = request.get_json() if not payload or payload.get('head_commit') is None: raise SkipRequestException() if 'zen' in payload: raise ValidationRequestException() logger.debug('Payload %s', payload) ref = payload['ref'] commit_sha = payload['head_commit']['id'] commit_message = payload['head_commit'].get('message', '') if should_skip_commit(commit_message): raise SkipRequestException() short_sha = GithubBuildTrigger.get_display_name(commit_sha) gh_client = self._get_client(auth_token) repo_full_name = '%s/%s' % (payload['repository']['owner']['name'], payload['repository']['name']) repo = gh_client.get_repo(repo_full_name) logger.debug('Github repo: %s', repo) return GithubBuildTrigger._prepare_build(config, repo, commit_sha, short_sha, ref) def manual_start(self, auth_token, config, run_parameters = None): try: source = config['build_source'] run_parameters = run_parameters or {} gh_client = self._get_client(auth_token) repo = gh_client.get_repo(source) master = repo.get_branch(repo.default_branch) master_sha = master.commit.sha short_sha = GithubBuildTrigger.get_display_name(master_sha) ref = 'refs/heads/%s' % (run_parameters.get('branch_name') or repo.default_branch) return self._prepare_build(config, repo, master_sha, short_sha, ref) except GithubException as ghe: raise TriggerStartException(ghe.data['message']) def list_field_values(self, auth_token, config, field_name): if field_name == 'branch_name': gh_client = self._get_client(auth_token) source = config['build_source'] repo = gh_client.get_repo(source) branches = [branch.name for branch in repo.get_branches()] if not repo.default_branch in branches: branches.insert(0, repo.default_branch) if branches[0] != repo.default_branch: branches.remove(repo.default_branch) branches.insert(0, repo.default_branch) return branches return None