diff --git a/buildman/server.py b/buildman/server.py
index 2ad5be99d..f4a236155 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -132,9 +132,13 @@ class BuilderServer(object):
     logger.debug('Unregistering component with realm %s and token %s',
                   component.builder_realm, component.expected_token)
 
-    self._realm_map.pop(component.builder_realm)
-    self._current_components.remove(component)
-    self._session_factory.remove(component)
+    self._realm_map.pop(component.builder_realm, None)
+
+    if component in self._current_components:
+      self._current_components.remove(component)
+
+    if component in self._session_factory:
+      self._session_factory.remove(component)
 
   def _job_heartbeat(self, build_job):
     self._queue.extend_processing(build_job.job_item, seconds_from_now=JOB_TIMEOUT_SECONDS,