Catch SSL errors due to timeouts in Github calls

Fixes https://sentry.io/coreos/backend-production/issues/219378902/
This commit is contained in:
Joseph Schorr 2017-02-21 12:31:11 -05:00
parent ef9cb3757d
commit 9db20ff961
3 changed files with 31 additions and 8 deletions

View file

@ -2,9 +2,14 @@ import logging
import os.path import os.path
import base64 import base64
from app import app, github_trigger from functools import wraps
from ssl import SSLError
from github import (Github, UnknownObjectException, GithubException,
BadCredentialsException as GitHubBadCredentialsException)
from jsonschema import validate from jsonschema import validate
from app import app, github_trigger
from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException, from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
TriggerDeactivationException, TriggerStartException, TriggerDeactivationException, TriggerStartException,
EmptyRepositoryException, ValidationRequestException, EmptyRepositoryException, ValidationRequestException,
@ -13,13 +18,10 @@ from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivation
find_matching_branches) find_matching_branches)
from buildtrigger.basehandler import BuildTriggerHandler from buildtrigger.basehandler import BuildTriggerHandler
from endpoints.exception import ExternalServiceError
from util.security.ssh import generate_ssh_keypair from util.security.ssh import generate_ssh_keypair
from util.dict_wrappers import JSONPathDict, SafeDictSetter from util.dict_wrappers import JSONPathDict, SafeDictSetter
from github import (Github, UnknownObjectException, GithubException,
BadCredentialsException as GitHubBadCredentialsException)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
GITHUB_WEBHOOK_PAYLOAD_SCHEMA = { GITHUB_WEBHOOK_PAYLOAD_SCHEMA = {
@ -139,6 +141,18 @@ def get_transformed_webhook_payload(gh_payload, default_branch=None, lookup_user
return config.dict_value() return config.dict_value()
def _catch_ssl_errors(func):
@wraps(func)
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except SSLError as se:
msg = 'Request to the GitHub API failed: %s' % se.message
logger.exception(msg)
raise ExternalServiceError(msg)
return wrapper
class GithubBuildTrigger(BuildTriggerHandler): class GithubBuildTrigger(BuildTriggerHandler):
""" """
BuildTrigger for GitHub that uses the archive API and buildpacks. BuildTrigger for GitHub that uses the archive API and buildpacks.
@ -169,6 +183,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
return default_msg return default_msg
@_catch_ssl_errors
def activate(self, standard_webhook_url): def activate(self, standard_webhook_url):
config = self.config config = self.config
new_build_source = config['build_source'] new_build_source = config['build_source']
@ -216,6 +231,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
return config, {'private_key': private_key} return config, {'private_key': private_key}
@_catch_ssl_errors
def deactivate(self): def deactivate(self):
config = self.config config = self.config
gh_client = self._get_client() gh_client = self._get_client()
@ -256,6 +272,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
self.config = config self.config = config
return config return config
@_catch_ssl_errors
def list_build_sources(self): def list_build_sources(self):
gh_client = self._get_client() gh_client = self._get_client()
usr = gh_client.get_user() usr = gh_client.get_user()
@ -306,6 +323,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
entries.sort(key=lambda e: e['info']['name']) entries.sort(key=lambda e: e['info']['name'])
return entries return entries
@_catch_ssl_errors
def list_build_subdirs(self): def list_build_subdirs(self):
config = self.config config = self.config
gh_client = self._get_client() gh_client = self._get_client()
@ -331,6 +349,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
raise RepositoryReadException(message) raise RepositoryReadException(message)
@_catch_ssl_errors
def load_dockerfile_contents(self): def load_dockerfile_contents(self):
config = self.config config = self.config
gh_client = self._get_client() gh_client = self._get_client()
@ -352,6 +371,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
message = ghe.data.get('message', 'Unable to read Dockerfile: %s' % source) message = ghe.data.get('message', 'Unable to read Dockerfile: %s' % source)
raise RepositoryReadException(message) raise RepositoryReadException(message)
@_catch_ssl_errors
def list_field_values(self, field_name, limit=None): def list_field_values(self, field_name, limit=None):
if field_name == 'refs': if field_name == 'refs':
branches = self.list_field_values('branch_name') branches = self.list_field_values('branch_name')
@ -444,6 +464,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
'commit_info': commit_info 'commit_info': commit_info
} }
@_catch_ssl_errors
def manual_start(self, run_parameters=None): def manual_start(self, run_parameters=None):
config = self.config config = self.config
source = config['build_source'] source = config['build_source']
@ -474,6 +495,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
metadata = GithubBuildTrigger._build_metadata_for_commit(commit_sha, ref, repo) metadata = GithubBuildTrigger._build_metadata_for_commit(commit_sha, ref, repo)
return self.prepare_build(metadata, is_manual=True) return self.prepare_build(metadata, is_manual=True)
@_catch_ssl_errors
def lookup_user(self, username): def lookup_user(self, username):
try: try:
gh_client = self._get_client() gh_client = self._get_client()
@ -485,6 +507,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
except GithubException: except GithubException:
return None return None
@_catch_ssl_errors
def handle_trigger_request(self, request): def handle_trigger_request(self, request):
# Check the payload to see if we should skip it based on the lack of a head_commit. # Check the payload to see if we should skip it based on the lack of a head_commit.
payload = request.get_json() payload = request.get_json()

View file

@ -14,7 +14,7 @@ from buildtrigger.basehandler import BuildTriggerHandler
from util.security.ssh import generate_ssh_keypair from util.security.ssh import generate_ssh_keypair
from util.dict_wrappers import JSONPathDict, SafeDictSetter from util.dict_wrappers import JSONPathDict, SafeDictSetter
from endpoints.exception import ExternalServiceTimeout from endpoints.exception import ExternalServiceError
import gitlab import gitlab
import requests import requests
@ -78,7 +78,7 @@ def _catch_timeouts(func):
except requests.exceptions.Timeout: except requests.exceptions.Timeout:
msg = 'Request to the GitLab API timed out' msg = 'Request to the GitLab API timed out'
logger.exception(msg) logger.exception(msg)
raise ExternalServiceTimeout(msg) raise ExternalServiceError(msg)
return wrapper return wrapper

View file

@ -77,7 +77,7 @@ class ApiException(Exception):
return rv return rv
class ExternalServiceTimeout(ApiException): class ExternalServiceError(ApiException):
def __init__(self, error_description, payload=None): def __init__(self, error_description, payload=None):
ApiException.__init__(self, ApiErrorType.external_service_timeout, 520, error_description, payload) ApiException.__init__(self, ApiErrorType.external_service_timeout, 520, error_description, payload)