Improve the builder response to being terminated or dying.

workers/dockerfilebuild.py

@@ -20,7 +20,7 @@ from collections import defaultdict
 
 from data.queue import dockerfile_build_queue
 from data import model
-from workers.worker import Worker
+from workers.worker import Worker, WorkerUnhealthyException, JobException
 from app import app, userfiles as user_files
 from util.safetar import safe_extractall
 from util.dockerfileparse import parse_dockerfile, ParsedDockerfile, serialize_dockerfile
@@ -39,6 +39,7 @@ build_logs = app.config['BUILDLOGS']
 TIMEOUT_PERIOD_MINUTES = 20
 CACHE_EXPIRATION_PERIOD_HOURS = 24
 NO_TAGS = ['<none>:<none>']
+RESERVATION_TIME = (TIMEOUT_PERIOD_MINUTES + 5) * 60
 
 
 class StatusWrapper(object):
@@ -122,7 +123,7 @@ class DockerfileBuildContext(object):
                                    'Dockerfile')
     if not os.path.exists(dockerfile_path):
       raise RuntimeError('Build job did not contain a Dockerfile.')
-    
+
     # Compute the number of steps
     with open(dockerfile_path, 'r') as dockerfileobj:
       self._parsed_dockerfile = parse_dockerfile(dockerfileobj.read())
@@ -137,9 +138,14 @@ class DockerfileBuildContext(object):
                  self._tag_names)
 
   def __enter__(self):
-    self.__cleanup_containers()
-    self.__cleanup_images()
-    self.__prune_cache()
+    try:
+      self.__cleanup_containers()
+      self.__cleanup_images()
+      self.__prune_cache()
+    except APIError:
+      message = 'Docker installation is no longer healthy.'
+      logger.exception(message)
+      raise WorkerUnhealthyException(message)
     return self
 
   def __exit__(self, exc_type, value, traceback):
@@ -209,7 +215,7 @@ class DockerfileBuildContext(object):
 
     self.__monitor_completion(pull_status, 'Downloading', self._status, 'pull_completion')
 
-  def build(self):
+  def build(self, reservation_extension_method):
     # Start the build itself.
     logger.debug('Starting build.')
 
@@ -249,6 +255,9 @@ class DockerfileBuildContext(object):
         logger.debug('Step now: %s/%s', current_step, self._num_steps)
         with self._status as status_update:
           status_update['current_command'] = current_step
+
+          # Tell the queue that we're making progress every time we advance a step
+          reservation_extension_method(RESERVATION_TIME)
         continue
       else:
         self._build_logger(status_str)
@@ -482,8 +491,9 @@ class DockerfileBuildWorker(Worker):
       log_appender('error', build_logs.PHASE)
       repository_build.phase = 'error'
       repository_build.save()
-      log_appender('Unknown mime-type: %s' % c_type, build_logs.ERROR)
-      return True
+      message = 'Unknown mime-type: %s' % c_type
+      log_appender(message, build_logs.ERROR)
+      raise JobException(message)
 
     build_dir = self._mime_processors[c_type](docker_resource)
 
@@ -496,21 +506,26 @@ class DockerfileBuildWorker(Worker):
         repository_build.save()
         build_ctxt.pull()
 
+        self.extend_processing(RESERVATION_TIME)
+
         log_appender('building', build_logs.PHASE)
         repository_build.phase = 'building'
         repository_build.save()
-        built_image = build_ctxt.build()
+        built_image = build_ctxt.build(self.extend_processing)
 
         if not built_image:
           log_appender('error', build_logs.PHASE)
           repository_build.phase = 'error'
           repository_build.save()
+
+          message = 'Unable to build dockerfile.'
           if self._timeout.is_set():
-            log_appender('Build step was terminated after %s minutes.' % TIMEOUT_PERIOD_MINUTES,
-                         build_logs.ERROR)
-          else:
-            log_appender('Unable to build dockerfile.', build_logs.ERROR)
-          return True
+            message = 'Build step was terminated after %s minutes.' % TIMEOUT_PERIOD_MINUTES
+
+          log_appender(message, build_logs.ERROR)
+          raise JobException(message)
+
+        self.extend_processing(RESERVATION_TIME)
 
         log_appender('pushing', build_logs.PHASE)
         repository_build.phase = 'pushing'
@@ -522,15 +537,17 @@ class DockerfileBuildWorker(Worker):
         repository_build.phase = 'complete'
         repository_build.save()
 
+    except WorkerUnhealthyException as exc:
+      # Need a separate handler for this so it doesn't get caught by catch all below
+      raise exc
+
     except Exception as exc:
       log_appender('error', build_logs.PHASE)
       logger.exception('Exception when processing request.')
       repository_build.phase = 'error'
       repository_build.save()
       log_appender(str(exc), build_logs.ERROR)
-      return True
-
-    return True
+      raise JobException(str(exc))
 
 
 desc = 'Worker daemon to monitor dockerfile build'
@@ -544,8 +561,8 @@ parser.add_argument('--cachegb', default=20, type=float,
 args = parser.parse_args()
 
 
-ONE_HOUR = 60*60
-worker = DockerfileBuildWorker(args.cachegb, dockerfile_build_queue, reservation_seconds=ONE_HOUR)
+worker = DockerfileBuildWorker(args.cachegb, dockerfile_build_queue,
+                               reservation_seconds=RESERVATION_TIME)
 
 if args.D:
   handler = logging.FileHandler(args.log)