Catch SSL errors due to timeouts in Github calls
Fixes https://sentry.io/coreos/backend-production/issues/219378902/
This commit is contained in:
parent
ef9cb3757d
commit
9db20ff961
3 changed files with 31 additions and 8 deletions
|
@ -2,9 +2,14 @@ import logging
|
||||||
import os.path
|
import os.path
|
||||||
import base64
|
import base64
|
||||||
|
|
||||||
from app import app, github_trigger
|
from functools import wraps
|
||||||
|
from ssl import SSLError
|
||||||
|
|
||||||
|
from github import (Github, UnknownObjectException, GithubException,
|
||||||
|
BadCredentialsException as GitHubBadCredentialsException)
|
||||||
from jsonschema import validate
|
from jsonschema import validate
|
||||||
|
|
||||||
|
from app import app, github_trigger
|
||||||
from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
|
from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
|
||||||
TriggerDeactivationException, TriggerStartException,
|
TriggerDeactivationException, TriggerStartException,
|
||||||
EmptyRepositoryException, ValidationRequestException,
|
EmptyRepositoryException, ValidationRequestException,
|
||||||
|
@ -13,13 +18,10 @@ from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivation
|
||||||
find_matching_branches)
|
find_matching_branches)
|
||||||
|
|
||||||
from buildtrigger.basehandler import BuildTriggerHandler
|
from buildtrigger.basehandler import BuildTriggerHandler
|
||||||
|
from endpoints.exception import ExternalServiceError
|
||||||
from util.security.ssh import generate_ssh_keypair
|
from util.security.ssh import generate_ssh_keypair
|
||||||
from util.dict_wrappers import JSONPathDict, SafeDictSetter
|
from util.dict_wrappers import JSONPathDict, SafeDictSetter
|
||||||
|
|
||||||
from github import (Github, UnknownObjectException, GithubException,
|
|
||||||
BadCredentialsException as GitHubBadCredentialsException)
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
GITHUB_WEBHOOK_PAYLOAD_SCHEMA = {
|
GITHUB_WEBHOOK_PAYLOAD_SCHEMA = {
|
||||||
|
@ -139,6 +141,18 @@ def get_transformed_webhook_payload(gh_payload, default_branch=None, lookup_user
|
||||||
return config.dict_value()
|
return config.dict_value()
|
||||||
|
|
||||||
|
|
||||||
|
def _catch_ssl_errors(func):
|
||||||
|
@wraps(func)
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
try:
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
except SSLError as se:
|
||||||
|
msg = 'Request to the GitHub API failed: %s' % se.message
|
||||||
|
logger.exception(msg)
|
||||||
|
raise ExternalServiceError(msg)
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
class GithubBuildTrigger(BuildTriggerHandler):
|
class GithubBuildTrigger(BuildTriggerHandler):
|
||||||
"""
|
"""
|
||||||
BuildTrigger for GitHub that uses the archive API and buildpacks.
|
BuildTrigger for GitHub that uses the archive API and buildpacks.
|
||||||
|
@ -169,6 +183,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
||||||
|
|
||||||
return default_msg
|
return default_msg
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
def activate(self, standard_webhook_url):
|
def activate(self, standard_webhook_url):
|
||||||
config = self.config
|
config = self.config
|
||||||
new_build_source = config['build_source']
|
new_build_source = config['build_source']
|
||||||
|
@ -216,6 +231,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
||||||
|
|
||||||
return config, {'private_key': private_key}
|
return config, {'private_key': private_key}
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
def deactivate(self):
|
def deactivate(self):
|
||||||
config = self.config
|
config = self.config
|
||||||
gh_client = self._get_client()
|
gh_client = self._get_client()
|
||||||
|
@ -256,6 +272,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
||||||
self.config = config
|
self.config = config
|
||||||
return config
|
return config
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
def list_build_sources(self):
|
def list_build_sources(self):
|
||||||
gh_client = self._get_client()
|
gh_client = self._get_client()
|
||||||
usr = gh_client.get_user()
|
usr = gh_client.get_user()
|
||||||
|
@ -306,6 +323,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
||||||
entries.sort(key=lambda e: e['info']['name'])
|
entries.sort(key=lambda e: e['info']['name'])
|
||||||
return entries
|
return entries
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
def list_build_subdirs(self):
|
def list_build_subdirs(self):
|
||||||
config = self.config
|
config = self.config
|
||||||
gh_client = self._get_client()
|
gh_client = self._get_client()
|
||||||
|
@ -331,6 +349,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
||||||
|
|
||||||
raise RepositoryReadException(message)
|
raise RepositoryReadException(message)
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
def load_dockerfile_contents(self):
|
def load_dockerfile_contents(self):
|
||||||
config = self.config
|
config = self.config
|
||||||
gh_client = self._get_client()
|
gh_client = self._get_client()
|
||||||
|
@ -352,6 +371,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
||||||
message = ghe.data.get('message', 'Unable to read Dockerfile: %s' % source)
|
message = ghe.data.get('message', 'Unable to read Dockerfile: %s' % source)
|
||||||
raise RepositoryReadException(message)
|
raise RepositoryReadException(message)
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
def list_field_values(self, field_name, limit=None):
|
def list_field_values(self, field_name, limit=None):
|
||||||
if field_name == 'refs':
|
if field_name == 'refs':
|
||||||
branches = self.list_field_values('branch_name')
|
branches = self.list_field_values('branch_name')
|
||||||
|
@ -444,6 +464,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
||||||
'commit_info': commit_info
|
'commit_info': commit_info
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
def manual_start(self, run_parameters=None):
|
def manual_start(self, run_parameters=None):
|
||||||
config = self.config
|
config = self.config
|
||||||
source = config['build_source']
|
source = config['build_source']
|
||||||
|
@ -474,6 +495,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
||||||
metadata = GithubBuildTrigger._build_metadata_for_commit(commit_sha, ref, repo)
|
metadata = GithubBuildTrigger._build_metadata_for_commit(commit_sha, ref, repo)
|
||||||
return self.prepare_build(metadata, is_manual=True)
|
return self.prepare_build(metadata, is_manual=True)
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
def lookup_user(self, username):
|
def lookup_user(self, username):
|
||||||
try:
|
try:
|
||||||
gh_client = self._get_client()
|
gh_client = self._get_client()
|
||||||
|
@ -485,6 +507,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
||||||
except GithubException:
|
except GithubException:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
@_catch_ssl_errors
|
||||||
def handle_trigger_request(self, request):
|
def handle_trigger_request(self, request):
|
||||||
# Check the payload to see if we should skip it based on the lack of a head_commit.
|
# Check the payload to see if we should skip it based on the lack of a head_commit.
|
||||||
payload = request.get_json()
|
payload = request.get_json()
|
||||||
|
|
|
@ -14,7 +14,7 @@ from buildtrigger.basehandler import BuildTriggerHandler
|
||||||
|
|
||||||
from util.security.ssh import generate_ssh_keypair
|
from util.security.ssh import generate_ssh_keypair
|
||||||
from util.dict_wrappers import JSONPathDict, SafeDictSetter
|
from util.dict_wrappers import JSONPathDict, SafeDictSetter
|
||||||
from endpoints.exception import ExternalServiceTimeout
|
from endpoints.exception import ExternalServiceError
|
||||||
|
|
||||||
import gitlab
|
import gitlab
|
||||||
import requests
|
import requests
|
||||||
|
@ -78,7 +78,7 @@ def _catch_timeouts(func):
|
||||||
except requests.exceptions.Timeout:
|
except requests.exceptions.Timeout:
|
||||||
msg = 'Request to the GitLab API timed out'
|
msg = 'Request to the GitLab API timed out'
|
||||||
logger.exception(msg)
|
logger.exception(msg)
|
||||||
raise ExternalServiceTimeout(msg)
|
raise ExternalServiceError(msg)
|
||||||
return wrapper
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -77,7 +77,7 @@ class ApiException(Exception):
|
||||||
return rv
|
return rv
|
||||||
|
|
||||||
|
|
||||||
class ExternalServiceTimeout(ApiException):
|
class ExternalServiceError(ApiException):
|
||||||
def __init__(self, error_description, payload=None):
|
def __init__(self, error_description, payload=None):
|
||||||
ApiException.__init__(self, ApiErrorType.external_service_timeout, 520, error_description, payload)
|
ApiException.__init__(self, ApiErrorType.external_service_timeout, 520, error_description, payload)
|
||||||
|
|
||||||
|
|
Reference in a new issue