Fix the metrics so they are usable for scaling the workers down and up. Switch all datetimes which touch the database from now to utcnow. Fix the worker Dockerfile.

workers/dockerfilebuild.py

@@ -1,3 +1,7 @@
+import logging.config
+
+logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False)  
+
 import logging
 import daemon
 import argparse
@@ -25,12 +29,6 @@ from util.safetar import safe_extractall
 from util.dockerfileparse import parse_dockerfile, ParsedDockerfile, serialize_dockerfile
 
 
-root_logger = logging.getLogger('')
-root_logger.setLevel(logging.DEBUG)
-
-FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - %(funcName)s - %(message)s'
-formatter = logging.Formatter(FORMAT)
-
 logger = logging.getLogger(__name__)
 
 TIMEOUT_PERIOD_MINUTES = 20
@@ -558,8 +556,6 @@ parser.add_argument('--cachegb', default=20, type=float,
                     help='Maximum cache size in gigabytes.')
 args = parser.parse_args()
 
-logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False)  
-
 worker = DockerfileBuildWorker(args.cachegb, dockerfile_build_queue,
                                reservation_seconds=RESERVATION_TIME)
 worker.start(start_status_server_port=8000)

workers/worker.py

@@ -124,6 +124,9 @@ class Worker(object):
     if not self._stop.is_set():
       logger.debug('No more work.')
 
+  def update_queue_metrics(self):
+    self._queue.update_metrics()
+
   def start(self, start_status_server_port=None):
     if start_status_server_port is not None:
       # Start a status server on a thread
@@ -140,6 +143,7 @@ class Worker(object):
     self._sched.start()
     self._sched.add_interval_job(self.poll_queue, seconds=self._poll_period_seconds,
                                  start_date=soon)
+    self._sched.add_interval_job(self.update_queue_metrics, seconds=60, start_date=soon)
     self._sched.add_interval_job(self.watchdog, seconds=self._watchdog_period_seconds)
 
     signal.signal(signal.SIGTERM, self.terminate)