Catch SSL errors due to timeouts in Github calls

Fixes https://sentry.io/coreos/backend-production/issues/219378902/

buildtrigger/githubhandler.py

@@ -2,9 +2,14 @@ import logging
 import os.path
 import base64
 
-from app import app, github_trigger
+from functools import wraps
+from ssl import SSLError
+
+from github import (Github, UnknownObjectException, GithubException,
+                    BadCredentialsException as GitHubBadCredentialsException)
 from jsonschema import validate
 
+from app import app, github_trigger
 from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
                                       TriggerDeactivationException, TriggerStartException,
                                       EmptyRepositoryException, ValidationRequestException,
@@ -13,13 +18,10 @@ from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivation
                                       find_matching_branches)
 
 from buildtrigger.basehandler import BuildTriggerHandler
-
+from endpoints.exception import ExternalServiceError
 from util.security.ssh import generate_ssh_keypair
 from util.dict_wrappers import JSONPathDict, SafeDictSetter
 
-from github import (Github, UnknownObjectException, GithubException,
-                    BadCredentialsException as GitHubBadCredentialsException)
-
 logger = logging.getLogger(__name__)
 
 GITHUB_WEBHOOK_PAYLOAD_SCHEMA = {
@@ -139,6 +141,18 @@ def get_transformed_webhook_payload(gh_payload, default_branch=None, lookup_user
   return config.dict_value()
 
 
+def _catch_ssl_errors(func):
+  @wraps(func)
+  def wrapper(*args, **kwargs):
+    try:
+      return func(*args, **kwargs)
+    except SSLError as se:
+      msg = 'Request to the GitHub API failed: %s' % se.message
+      logger.exception(msg)
+      raise ExternalServiceError(msg)
+  return wrapper
+
+
 class GithubBuildTrigger(BuildTriggerHandler):
   """
   BuildTrigger for GitHub that uses the archive API and buildpacks.
@@ -169,6 +183,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
 
     return default_msg
 
+  @_catch_ssl_errors
   def activate(self, standard_webhook_url):
     config = self.config
     new_build_source = config['build_source']
@@ -216,6 +231,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
 
     return config, {'private_key': private_key}
 
+  @_catch_ssl_errors
   def deactivate(self):
     config = self.config
     gh_client = self._get_client()
@@ -256,6 +272,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
     self.config = config
     return config
 
+  @_catch_ssl_errors
   def list_build_sources(self):
     gh_client = self._get_client()
     usr = gh_client.get_user()
@@ -306,6 +323,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
     entries.sort(key=lambda e: e['info']['name'])
     return entries
 
+  @_catch_ssl_errors
   def list_build_subdirs(self):
     config = self.config
     gh_client = self._get_client()
@@ -331,6 +349,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
 
       raise RepositoryReadException(message)
 
+  @_catch_ssl_errors
   def load_dockerfile_contents(self):
     config = self.config
     gh_client = self._get_client()
@@ -352,6 +371,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
       message = ghe.data.get('message', 'Unable to read Dockerfile: %s' % source)
       raise RepositoryReadException(message)
 
+  @_catch_ssl_errors
   def list_field_values(self, field_name, limit=None):
     if field_name == 'refs':
       branches = self.list_field_values('branch_name')
@@ -444,6 +464,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
       'commit_info': commit_info
     }
 
+  @_catch_ssl_errors
   def manual_start(self, run_parameters=None):
     config = self.config
     source = config['build_source']
@@ -474,6 +495,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
     metadata = GithubBuildTrigger._build_metadata_for_commit(commit_sha, ref, repo)
     return self.prepare_build(metadata, is_manual=True)
 
+  @_catch_ssl_errors
   def lookup_user(self, username):
     try:
       gh_client = self._get_client()
@@ -485,6 +507,7 @@ class GithubBuildTrigger(BuildTriggerHandler):
     except GithubException:
       return None
 
+  @_catch_ssl_errors
   def handle_trigger_request(self, request):
     # Check the payload to see if we should skip it based on the lack of a head_commit.
     payload = request.get_json()

buildtrigger/gitlabhandler.py

@@ -14,7 +14,7 @@ from buildtrigger.basehandler import BuildTriggerHandler
 
 from util.security.ssh import generate_ssh_keypair
 from util.dict_wrappers import JSONPathDict, SafeDictSetter
-from endpoints.exception import ExternalServiceTimeout
+from endpoints.exception import ExternalServiceError
 
 import gitlab
 import requests
@@ -78,7 +78,7 @@ def _catch_timeouts(func):
     except requests.exceptions.Timeout:
       msg = 'Request to the GitLab API timed out'
       logger.exception(msg)
-      raise ExternalServiceTimeout(msg)
+      raise ExternalServiceError(msg)
   return wrapper
 
 

endpoints/exception.py

@@ -77,7 +77,7 @@ class ApiException(Exception):
     return rv
 
 
-class ExternalServiceTimeout(ApiException):
+class ExternalServiceError(ApiException):
   def __init__(self, error_description, payload=None):
     ApiException.__init__(self, ApiErrorType.external_service_timeout, 520, error_description, payload)