Add better exception handling and logging to the ephemeral build manager

buildman/manager/ephemeral.py

@@ -77,7 +77,7 @@ class EphemeralBuilderManager(BaseManager):
 
       try:
         etcd_result = changed_key_future.result()
-      except (ReadTimeoutError, ProtocolError):
+      except (ReadTimeoutError, ProtocolError, etcd.EtcdException):
         return
 
       change_callback(etcd_result)
@@ -233,11 +233,20 @@ class EphemeralBuilderManager(BaseManager):
       raise Return(False)
 
     logger.debug('Starting builder with executor: %s', self._executor)
-    builder_id = yield From(self._executor.start_builder(realm, token, build_uuid))
+
+    try:
+      builder_id = yield From(self._executor.start_builder(realm, token, build_uuid))
+    except:
+      logger.exception('Exception when starting builder for job: %s', build_uuid)
+      raise Return(False)
 
     # Store the builder in etcd associated with the job id
-    payload['builder_id'] = builder_id
-    yield From(self._etcd_client.write(job_key, json.dumps(payload), prevExist=True, ttl=ttl))
+    try:
+      payload['builder_id'] = builder_id
+      yield From(self._etcd_client.write(job_key, json.dumps(payload), prevExist=True, ttl=ttl))
+    except etcd.EtcdException:
+      logger.exception('Exception when writing job %s to etcd', build_uuid)
+      raise Return(False)
 
     # Store the realm spec which will allow any manager to accept this builder when it connects
     realm_spec = json.dumps({

buildman/server.py

@@ -175,7 +175,13 @@ class BuilderServer(object):
         continue
 
       logger.debug('Build job found. Checking for an avaliable worker.')
-      scheduled = yield From(self._lifecycle_manager.schedule(build_job))
+
+      try:
+        scheduled = yield From(self._lifecycle_manager.schedule(build_job))
+      except:
+        logger.exception('Exception when scheduling job')
+        scheduled = None
+
       if scheduled:
         status_handler = StatusHandler(self._build_logs, build_job.repo_build.uuid)
         status_handler.set_phase('build-scheduled')