From 4a4118d0e0d2939a4ca864a5f60c32e8104fdcde Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Fri, 13 Feb 2015 11:21:34 -0500
Subject: [PATCH 01/24] buildman: update metrics task

---
 buildman/server.py | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/buildman/server.py b/buildman/server.py
index 7b10995b4..ea689da3f 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -8,14 +8,15 @@ from autobahn.wamp import types
 from aiowsgi import create_server as create_wsgi_server
 from flask import Flask
 from threading import Event
+from trollius.tasks import Task
 from trollius.coroutines import From
 from datetime import timedelta
 
 from buildman.jobutil.buildjob import BuildJob, BuildJobLoadException
 from data import database
-from data.queue import WorkQueue
 from app import app
 
+# pylint: disable=invalid-name
 logger = logging.getLogger(__name__)
 
 WORK_CHECK_TIMEOUT = 10
@@ -25,16 +26,19 @@ MINIMUM_JOB_EXTENSION = timedelta(minutes=2)
 
 HEARTBEAT_PERIOD_SEC = 30
 
+# pylint: disable=too-few-public-methods
 class BuildJobResult(object):
   """ Build job result enum """
   INCOMPLETE = 'incomplete'
   COMPLETE = 'complete'
   ERROR = 'error'
 
+# pylint: disable=too-many-instance-attributes
 class BuilderServer(object):
   """ Server which handles both HTTP and WAMP requests, managing the full state of the build
       controller.
   """
+  # pylint: disable=too-many-arguments
   def __init__(self, registry_hostname, queue, build_logs, user_files, lifecycle_manager_klass,
                lifecycle_manager_config, manager_hostname):
     self._loop = None
@@ -81,8 +85,14 @@ class BuilderServer(object):
 
     logger.debug('Starting server on port %s, with controller on port %s', websocket_port,
                  controller_port)
+
+    tasks = [
+      Task(self._initialize(loop, host, websocket_port, controller_port, ssl)),
+      Task(self._metrics_updater()),
+    ]
+
     try:
-      loop.run_until_complete(self._initialize(loop, host, websocket_port, controller_port, ssl))
+      loop.run_until_complete(trollius.wait(tasks))
     except KeyboardInterrupt:
       pass
     finally:
@@ -164,7 +174,11 @@ class BuilderServer(object):
         logger.debug('All workers are busy. Requeuing.')
         self._queue.incomplete(job_item, restore_retry=True, retry_after=0)
 
-
+  @trollius.coroutine
+  def _metrics_updater(self):
+    while self._current_status == 'running':
+      yield From(trollius.sleep(30))
+      self._queue.update_metrics()
 
   @trollius.coroutine
   def _initialize(self, loop, host, websocket_port, controller_port, ssl=None):
@@ -178,5 +192,8 @@ class BuilderServer(object):
     create_wsgi_server(self._controller_app, loop=loop, host=host, port=controller_port, ssl=ssl)
     yield From(loop.create_server(transport_factory, host, websocket_port, ssl=ssl))
 
+    # Initialize the coroutine reporting metrics
+    loop.create_task(self._metrics_updater())
+
     # Initialize the work queue checker.
     yield From(self._work_checker())

From 6a3d2695740720f5b12be649b89260c5296cb591 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Fri, 13 Feb 2015 11:21:34 -0500
Subject: [PATCH 02/24] buildman: update metrics task

---
 buildman/server.py | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/buildman/server.py b/buildman/server.py
index ebbc558bb..21ce7cd89 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -8,15 +8,16 @@ from autobahn.wamp import types
 from aiowsgi import create_server as create_wsgi_server
 from flask import Flask
 from threading import Event
+from trollius.tasks import Task
 from trollius.coroutines import From
 from datetime import timedelta
 
 from buildman.jobutil.buildstatus import StatusHandler
 from buildman.jobutil.buildjob import BuildJob, BuildJobLoadException
 from data import database
-from data.queue import WorkQueue
 from app import app
 
+# pylint: disable=invalid-name
 logger = logging.getLogger(__name__)
 
 WORK_CHECK_TIMEOUT = 10
@@ -26,16 +27,19 @@ MINIMUM_JOB_EXTENSION = timedelta(minutes=2)
 
 HEARTBEAT_PERIOD_SEC = 30
 
+# pylint: disable=too-few-public-methods
 class BuildJobResult(object):
   """ Build job result enum """
   INCOMPLETE = 'incomplete'
   COMPLETE = 'complete'
   ERROR = 'error'
 
+# pylint: disable=too-many-instance-attributes
 class BuilderServer(object):
   """ Server which handles both HTTP and WAMP requests, managing the full state of the build
       controller.
   """
+  # pylint: disable=too-many-arguments
   def __init__(self, registry_hostname, queue, build_logs, user_files, lifecycle_manager_klass,
                lifecycle_manager_config, manager_hostname):
     self._loop = None
@@ -82,8 +86,14 @@ class BuilderServer(object):
 
     logger.debug('Starting server on port %s, with controller on port %s', websocket_port,
                  controller_port)
+
+    tasks = [
+      Task(self._initialize(loop, host, websocket_port, controller_port, ssl)),
+      Task(self._metrics_updater()),
+    ]
+
     try:
-      loop.run_until_complete(self._initialize(loop, host, websocket_port, controller_port, ssl))
+      loop.run_until_complete(trollius.wait(tasks))
     except KeyboardInterrupt:
       pass
     finally:
@@ -168,7 +178,11 @@ class BuilderServer(object):
         logger.debug('All workers are busy. Requeuing.')
         self._queue.incomplete(job_item, restore_retry=True, retry_after=0)
 
-
+  @trollius.coroutine
+  def _metrics_updater(self):
+    while self._current_status == 'running':
+      yield From(trollius.sleep(30))
+      self._queue.update_metrics()
 
   @trollius.coroutine
   def _initialize(self, loop, host, websocket_port, controller_port, ssl=None):
@@ -182,5 +196,8 @@ class BuilderServer(object):
     create_wsgi_server(self._controller_app, loop=loop, host=host, port=controller_port, ssl=ssl)
     yield From(loop.create_server(transport_factory, host, websocket_port, ssl=ssl))
 
+    # Initialize the coroutine reporting metrics
+    loop.create_task(self._metrics_updater())
+
     # Initialize the work queue checker.
     yield From(self._work_checker())

From 6a1dd376c206ca530b48cfc476fb1a82e7cc732c Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Sat, 14 Feb 2015 16:30:10 -0500
Subject: [PATCH 03/24] util: add cloudwatch package

This isolates the CloudWatch sending thread to it's own package where it
can be shared among any other packages that want to asynchronously send
metrics to CloudWatch.
---
 util/cloudwatch.py   | 51 +++++++++++++++++++++++++++++++++++++++
 util/queuemetrics.py | 57 +++++++-------------------------------------
 2 files changed, 60 insertions(+), 48 deletions(-)
 create mode 100644 util/cloudwatch.py

diff --git a/util/cloudwatch.py b/util/cloudwatch.py
new file mode 100644
index 000000000..344433806
--- /dev/null
+++ b/util/cloudwatch.py
@@ -0,0 +1,51 @@
+import logging
+import boto
+
+from Queue import Queue
+from threading import Thread
+
+
+logger = logging.getLogger(__name__)
+queue = None
+
+def get_queue(app):
+  """
+  Returns a queue to the CloudWatchSender. If a queue/sender do not exist, creates them.
+  """
+  if queue is None:
+    global queue
+    access_key = app.config.get('QUEUE_METRICS_AWS_ACCESS_KEY')
+    secret_key = app.config.get('QUEUE_METRICS_AWS_SECRET_KEY')
+    queue = Queue()
+    sender = CloudWatchSender(queue, access_key, secret_key)
+    sender.start()
+  else:
+    return queue
+
+
+class CloudWatchSender(Thread):
+  """
+  CloudWatchSender loops indefinitely and pulls metrics off of a queue then sends it to CloudWatch.
+  """
+  def __init__(self, request_queue, aws_access_key, aws_secret_key):
+    Thread.__init__(self)
+    self.daemon = True
+
+    self._aws_access_key = aws_access_key
+    self._aws_secret_key = aws_secret_key
+    self._put_metrics_queue = request_queue
+
+  def run(self):
+    try:
+      logger.debug('Starting CloudWatch sender process.')
+      connection = boto.connect_cloudwatch(self._aws_access_key, self._aws_secret_key)
+    except:
+      logger.exception('Failed to connect to CloudWatch.')
+
+    while True:
+      put_metric_args, kwargs = self._put_metrics_queue.get()
+      logger.debug('Got queued put metrics request.')
+      try:
+        connection.put_metric_data(*put_metric_args, **kwargs)
+      except:
+        logger.exception('Failed to write to CloudWatch')
diff --git a/util/queuemetrics.py b/util/queuemetrics.py
index 1bebfa3a6..d84ced129 100644
--- a/util/queuemetrics.py
+++ b/util/queuemetrics.py
@@ -1,8 +1,6 @@
 import logging
-import boto
 
-from Queue import Queue
-from threading import Thread
+from util.cloudwatch import get_queue
 
 
 logger = logging.getLogger(__name__)
@@ -13,8 +11,8 @@ class NullReporter(object):
     pass
 
 
-class QueueingCloudWatchReporter(object):
-  """ QueueingCloudWatchReporter reports metrics to the "SendToCloudWatch" process """
+class CloudWatchReporter(object):
+  """ CloudWatchReporter reports work queue metrics to CloudWatch """
   def __init__(self, request_queue, namespace, need_capacity_name, build_percent_name):
     self._namespace = namespace
     self._need_capacity_name = need_capacity_name
@@ -36,35 +34,10 @@ class QueueingCloudWatchReporter(object):
     self._send_to_queue(self._namespace, self._build_percent_name, building_percent,
                         unit='Percent')
 
-
-class SendToCloudWatch(Thread):
-  """ SendToCloudWatch loops indefinitely and pulls metrics off of a queue then sends it to
-  CloudWatch. """
-  def __init__(self, request_queue, aws_access_key, aws_secret_key):
-    Thread.__init__(self)
-    self.daemon = True
-
-    self._aws_access_key = aws_access_key
-    self._aws_secret_key = aws_secret_key
-    self._put_metrics_queue = request_queue
-
-  def run(self):
-    try:
-      logger.debug('Starting CloudWatch sender process.')
-      connection = boto.connect_cloudwatch(self._aws_access_key, self._aws_secret_key)
-    except:
-      logger.exception('Failed to connect to CloudWatch.')
-
-    while True:
-      put_metric_args, kwargs = self._put_metrics_queue.get()
-      logger.debug('Got queued put metrics request.')
-      try:
-        connection.put_metric_data(*put_metric_args, **kwargs)
-      except:
-        logger.exception('Failed to write to CloudWatch')
-
-
 class QueueMetrics(object):
+  """
+  QueueMetrics initializes a reporter for recording metrics of work queues.
+  """
   def __init__(self, app=None):
     self.app = app
     self.sender = None
@@ -77,29 +50,17 @@ class QueueMetrics(object):
     analytics_type = app.config.get('QUEUE_METRICS_TYPE', 'Null')
 
     if analytics_type == 'CloudWatch':
-      access_key = app.config.get('QUEUE_METRICS_AWS_ACCESS_KEY')
-      secret_key = app.config.get('QUEUE_METRICS_AWS_SECRET_KEY')
       namespace = app.config.get('QUEUE_METRICS_NAMESPACE')
       req_capacity_name = app.config.get('QUEUE_METRICS_CAPACITY_SHORTAGE_NAME')
       build_percent_name = app.config.get('QUEUE_METRICS_BUILD_PERCENT_NAME')
 
-      request_queue = Queue()
-      reporter = QueueingCloudWatchReporter(request_queue, namespace, req_capacity_name,
-                                            build_percent_name)
-      self.sender = SendToCloudWatch(request_queue, access_key, secret_key)
+      request_queue = get_queue(app)
+      reporter = CloudWatchReporter(request_queue, namespace, req_capacity_name,
+                                    build_percent_name)
     else:
       reporter = NullReporter()
 
-    # register extension with app
-    app.extensions = getattr(app, 'extensions', {})
-    app.extensions['queuemetrics'] = reporter
     return reporter
 
-  def run(self):
-    logger.debug('Asked to start CloudWatch reporter')
-    if self.sender is not None:
-      logger.debug('Starting CloudWatch reporter')
-      self.sender.start()
-
   def __getattr__(self, name):
     return getattr(self.state, name, None)

From 0e7418ffce7a55011c700498c6d496df2c6cb858 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Tue, 17 Feb 2015 10:56:09 -0500
Subject: [PATCH 04/24] buildman: add BuildMetrics and BuildReporter

---
 app.py                            |  2 +
 buildman/jobutil/buildreporter.py | 75 +++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+)
 create mode 100644 buildman/jobutil/buildreporter.py

diff --git a/app.py b/app.py
index c3b15d7aa..78243de75 100644
--- a/app.py
+++ b/app.py
@@ -32,6 +32,7 @@ from util.queuemetrics import QueueMetrics
 from util.config.provider import FileConfigProvider, TestConfigProvider
 from util.config.configutil import generate_secret_key
 from util.config.superusermanager import SuperUserManager
+from buildman.jobutil.buildreporter import BuildMetrics
 
 OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/'
 OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml'
@@ -118,6 +119,7 @@ userevents = UserEventsBuilderModule(app)
 superusers = SuperUserManager(app)
 signer = Signer(app, OVERRIDE_CONFIG_DIRECTORY)
 queue_metrics = QueueMetrics(app)
+build_metrics = BuildMetrics(app)
 
 tf = app.config['DB_TRANSACTION_FACTORY']
 
diff --git a/buildman/jobutil/buildreporter.py b/buildman/jobutil/buildreporter.py
new file mode 100644
index 000000000..0016778dc
--- /dev/null
+++ b/buildman/jobutil/buildreporter.py
@@ -0,0 +1,75 @@
+import logging
+from trollius import From
+
+from util.cloudwatch import get_queue
+
+
+# pylint: disable=invalid-name
+logger = logging.getLogger(__name__)
+
+
+# pylint: disable=too-few-public-methods
+class BuildReporter(object):
+  """
+  Base class for reporting build statuses to a metrics service.
+  """
+  def report_completion_status(self, status):
+    """
+    Method to invoke the recording of build's completion status to a metric service.
+    This must _not_ block because it is assumed to run in an event loop.
+    """
+    raise NotImplementedError
+
+
+class NullReporter(BuildReporter):
+  """
+  The /dev/null of BuildReporters.
+  """
+  def report_completion_status(self, *args):
+    pass
+
+
+# pylint: disable=too-many-instance-attributes
+class CloudWatchBuildReporter(BuildReporter):
+  """
+  Implements a BuildReporter for Amazon's CloudWatch.
+  """
+  # pylint: disable=too-many-arguments
+  def __init__(self, queue, namespace_name, completed_name, failed_name, incompleted_name):
+    self._queue = queue
+    self._namespace_name = namespace_name
+    self._completed_name = completed_name
+    self._failed_name = failed_name
+    self._incompleted_name = incompleted_name
+
+  def report_completion_status(self, status):
+    if status == 'complete':
+      status_name = self._completed_name
+    elif status == 'error':
+      status_name = self._failed_name
+    elif status == 'incomplete':
+      status_name = self._incompleted_name
+    else:
+      return
+
+    yield From(self._queue.put(self._namespace_name, status_name, 1, unit='Count'))
+
+
+class BuildMetrics(object):
+  """
+  BuildMetrics initializes a reporter for recording the status of build completions.
+  """
+  def __init__(self, app=None):
+    self._app = app
+    if app is not None:
+      reporter_type = app.config.get('BUILD_METRICS_TYPE', 'Null')
+      if reporter_type == 'CloudWatch':
+        namespace = app.config.get('BUILD_METRICS_NAMESPACE')
+        completed_name = app.config.get('BUILD_METRICS_COMPLETED_NAME')
+        failed_name = app.config.get('BUILD_METRICS_FAILED_NAME')
+        incompleted_name = app.config.get('BUILD_METRICS_INCOMPLETED_NAME')
+        request_queue = get_queue(app)
+        self._reporter = CloudWatchBuildReporter(request_queue, namespace, completed_name,
+                                                 failed_name, incompleted_name)
+      else:
+        self._reporter = NullReporter()

From 0a00453024f8e7fcbed0f8f83ef518830944a275 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Tue, 17 Feb 2015 12:20:46 -0500
Subject: [PATCH 05/24] buildreporter: rm pylint comments

---
 buildman/jobutil/buildreporter.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/buildman/jobutil/buildreporter.py b/buildman/jobutil/buildreporter.py
index 0016778dc..f8ccf1fd5 100644
--- a/buildman/jobutil/buildreporter.py
+++ b/buildman/jobutil/buildreporter.py
@@ -4,11 +4,9 @@ from trollius import From
 from util.cloudwatch import get_queue
 
 
-# pylint: disable=invalid-name
 logger = logging.getLogger(__name__)
 
 
-# pylint: disable=too-few-public-methods
 class BuildReporter(object):
   """
   Base class for reporting build statuses to a metrics service.
@@ -29,12 +27,10 @@ class NullReporter(BuildReporter):
     pass
 
 
-# pylint: disable=too-many-instance-attributes
 class CloudWatchBuildReporter(BuildReporter):
   """
   Implements a BuildReporter for Amazon's CloudWatch.
   """
-  # pylint: disable=too-many-arguments
   def __init__(self, queue, namespace_name, completed_name, failed_name, incompleted_name):
     self._queue = queue
     self._namespace_name = namespace_name

From ca0d2b17214776576b1ea5f13286a538768b0bb4 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Tue, 17 Feb 2015 12:21:22 -0500
Subject: [PATCH 06/24] buildreporter: getattr method

---
 buildman/jobutil/buildreporter.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/buildman/jobutil/buildreporter.py b/buildman/jobutil/buildreporter.py
index f8ccf1fd5..08e16576b 100644
--- a/buildman/jobutil/buildreporter.py
+++ b/buildman/jobutil/buildreporter.py
@@ -69,3 +69,6 @@ class BuildMetrics(object):
                                                  failed_name, incompleted_name)
       else:
         self._reporter = NullReporter()
+
+  def __getattr__(self, name):
+    return getattr(self._reporter, name, None)

From ffb897dfe6c81f26854be7efcb9453c1064df7c3 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Tue, 17 Feb 2015 12:22:23 -0500
Subject: [PATCH 07/24] buildman: add job status logging to managers

---
 buildman/manager/enterprise.py | 2 ++
 buildman/manager/ephemeral.py  | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/buildman/manager/enterprise.py b/buildman/manager/enterprise.py
index c56830a1c..f144fd504 100644
--- a/buildman/manager/enterprise.py
+++ b/buildman/manager/enterprise.py
@@ -1,6 +1,7 @@
 import logging
 import uuid
 
+from app import build_metrics
 from buildman.component.basecomponent import BaseComponent
 from buildman.component.buildcomponent import BuildComponent
 from buildman.manager.basemanager import BaseManager
@@ -75,6 +76,7 @@ class EnterpriseManager(BaseManager):
 
   @coroutine
   def job_completed(self, build_job, job_status, build_component):
+    build_metrics.report(job_status)
     self.job_complete_callback(build_job, job_status)
 
   def build_component_disposed(self, build_component, timed_out):
diff --git a/buildman/manager/ephemeral.py b/buildman/manager/ephemeral.py
index cfb52f8ad..123594190 100644
--- a/buildman/manager/ephemeral.py
+++ b/buildman/manager/ephemeral.py
@@ -10,6 +10,7 @@ from trollius import From, coroutine, Return, async
 from concurrent.futures import ThreadPoolExecutor
 from urllib3.exceptions import ReadTimeoutError, ProtocolError
 
+from app import build_metrics
 from buildman.manager.basemanager import BaseManager
 from buildman.manager.executor import PopenExecutor, EC2Executor
 from buildman.component.buildcomponent import BuildComponent
@@ -286,6 +287,8 @@ class EphemeralBuilderManager(BaseManager):
     job_key = self._etcd_job_key(build_job)
     yield From(self._etcd_client.delete(job_key))
 
+    build_metrics.report(job_status)
+
     self.job_complete_callback(build_job, job_status)
 
   @coroutine

From 935db5c766f6ab10b08fdf5be3af6125a978a618 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Tue, 17 Feb 2015 12:23:08 -0500
Subject: [PATCH 08/24] buildman: clarify queue metrics from job state metrics

---
 buildman/server.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/buildman/server.py b/buildman/server.py
index 21ce7cd89..e40452e3d 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -89,7 +89,7 @@ class BuilderServer(object):
 
     tasks = [
       Task(self._initialize(loop, host, websocket_port, controller_port, ssl)),
-      Task(self._metrics_updater()),
+      Task(self._queue_metrics_updater()),
     ]
 
     try:
@@ -179,7 +179,7 @@ class BuilderServer(object):
         self._queue.incomplete(job_item, restore_retry=True, retry_after=0)
 
   @trollius.coroutine
-  def _metrics_updater(self):
+  def _queue_metrics_updater(self):
     while self._current_status == 'running':
       yield From(trollius.sleep(30))
       self._queue.update_metrics()

From ef8d320c95ed93cbc3e0e8b2a311f8375e6d74a9 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Tue, 17 Feb 2015 13:13:12 -0500
Subject: [PATCH 09/24] cloudwatch: global before reading queue

Technically, it isn't necessary to use the global keyword before reading
a global value, only modifying it. However, in this case it leaves a
pretty annoying log line.
---
 util/cloudwatch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/util/cloudwatch.py b/util/cloudwatch.py
index 344433806..cf7aed392 100644
--- a/util/cloudwatch.py
+++ b/util/cloudwatch.py
@@ -12,8 +12,8 @@ def get_queue(app):
   """
   Returns a queue to the CloudWatchSender. If a queue/sender do not exist, creates them.
   """
+  global queue
   if queue is None:
-    global queue
     access_key = app.config.get('QUEUE_METRICS_AWS_ACCESS_KEY')
     secret_key = app.config.get('QUEUE_METRICS_AWS_SECRET_KEY')
     queue = Queue()

From 8df35e720e20d04c0c79231a96deea56e2eaef9d Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Tue, 17 Feb 2015 13:16:03 -0500
Subject: [PATCH 10/24] web: remove manually starting of queuemetrics

---
 web.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/web.py b/web.py
index 92b3d6758..7c945cc45 100644
--- a/web.py
+++ b/web.py
@@ -1,7 +1,7 @@
 import logging
 import logging.config
 
-from app import app as application, queue_metrics
+from app import app as application
 
 from endpoints.api import api_bp
 from endpoints.web import web
@@ -9,9 +9,6 @@ from endpoints.webhooks import webhooks
 from endpoints.realtime import realtime
 from endpoints.callbacks import callback
 
-# Start the cloudwatch reporting.
-queue_metrics.run()
-
 application.register_blueprint(web)
 application.register_blueprint(callback, url_prefix='/oauth2')
 application.register_blueprint(api_bp, url_prefix='/api')

From b8d9ef0fe9695889349b1ead1d80ecdbe7bb41de Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Tue, 17 Feb 2015 14:18:32 -0500
Subject: [PATCH 11/24] buildman: remove old create_task for queue metrics

---
 buildman/server.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/buildman/server.py b/buildman/server.py
index e40452e3d..f077f7e9f 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -196,8 +196,5 @@ class BuilderServer(object):
     create_wsgi_server(self._controller_app, loop=loop, host=host, port=controller_port, ssl=ssl)
     yield From(loop.create_server(transport_factory, host, websocket_port, ssl=ssl))
 
-    # Initialize the coroutine reporting metrics
-    loop.create_task(self._metrics_updater())
-
     # Initialize the work queue checker.
     yield From(self._work_checker())

From 25fc999d508e9c91be2ac6d16fc8472569bfa5b7 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Tue, 17 Feb 2015 15:30:09 -0500
Subject: [PATCH 12/24] buildreporter: handle app=None

---
 buildman/jobutil/buildreporter.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/buildman/jobutil/buildreporter.py b/buildman/jobutil/buildreporter.py
index 08e16576b..09a74a565 100644
--- a/buildman/jobutil/buildreporter.py
+++ b/buildman/jobutil/buildreporter.py
@@ -57,7 +57,9 @@ class BuildMetrics(object):
   """
   def __init__(self, app=None):
     self._app = app
-    if app is not None:
+    if app is None:
+      self._reporter = None
+    else:
       reporter_type = app.config.get('BUILD_METRICS_TYPE', 'Null')
       if reporter_type == 'CloudWatch':
         namespace = app.config.get('BUILD_METRICS_NAMESPACE')

From d70c95e42e89fc652b56cb262f8a7fc177ed64d3 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Tue, 17 Feb 2015 15:31:53 -0500
Subject: [PATCH 13/24] buildreporter: move reporting into server callback

---
 buildman/manager/enterprise.py | 2 --
 buildman/manager/ephemeral.py  | 3 ---
 buildman/server.py             | 4 +++-
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/buildman/manager/enterprise.py b/buildman/manager/enterprise.py
index f144fd504..c56830a1c 100644
--- a/buildman/manager/enterprise.py
+++ b/buildman/manager/enterprise.py
@@ -1,7 +1,6 @@
 import logging
 import uuid
 
-from app import build_metrics
 from buildman.component.basecomponent import BaseComponent
 from buildman.component.buildcomponent import BuildComponent
 from buildman.manager.basemanager import BaseManager
@@ -76,7 +75,6 @@ class EnterpriseManager(BaseManager):
 
   @coroutine
   def job_completed(self, build_job, job_status, build_component):
-    build_metrics.report(job_status)
     self.job_complete_callback(build_job, job_status)
 
   def build_component_disposed(self, build_component, timed_out):
diff --git a/buildman/manager/ephemeral.py b/buildman/manager/ephemeral.py
index 123594190..cfb52f8ad 100644
--- a/buildman/manager/ephemeral.py
+++ b/buildman/manager/ephemeral.py
@@ -10,7 +10,6 @@ from trollius import From, coroutine, Return, async
 from concurrent.futures import ThreadPoolExecutor
 from urllib3.exceptions import ReadTimeoutError, ProtocolError
 
-from app import build_metrics
 from buildman.manager.basemanager import BaseManager
 from buildman.manager.executor import PopenExecutor, EC2Executor
 from buildman.component.buildcomponent import BuildComponent
@@ -287,8 +286,6 @@ class EphemeralBuilderManager(BaseManager):
     job_key = self._etcd_job_key(build_job)
     yield From(self._etcd_client.delete(job_key))
 
-    build_metrics.report(job_status)
-
     self.job_complete_callback(build_job, job_status)
 
   @coroutine
diff --git a/buildman/server.py b/buildman/server.py
index f077f7e9f..9aad39b5c 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -15,7 +15,7 @@ from datetime import timedelta
 from buildman.jobutil.buildstatus import StatusHandler
 from buildman.jobutil.buildjob import BuildJob, BuildJobLoadException
 from data import database
-from app import app
+from app import app, build_metrics
 
 # pylint: disable=invalid-name
 logger = logging.getLogger(__name__)
@@ -135,6 +135,8 @@ class BuilderServer(object):
                                   minimum_extension=MINIMUM_JOB_EXTENSION)
 
   def _job_complete(self, build_job, job_status):
+    build_metrics.report(job_status)
+
     if job_status == BuildJobResult.INCOMPLETE:
       self._queue.incomplete(build_job.job_item, restore_retry=False, retry_after=30)
     else:

From 85edb651e2fe425d9d823ad27ae1b2e3099df8f0 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Tue, 17 Feb 2015 15:32:25 -0500
Subject: [PATCH 14/24] buildserver: remove pylint comments

---
 buildman/server.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/buildman/server.py b/buildman/server.py
index 9aad39b5c..900746c6e 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -17,7 +17,6 @@ from buildman.jobutil.buildjob import BuildJob, BuildJobLoadException
 from data import database
 from app import app, build_metrics
 
-# pylint: disable=invalid-name
 logger = logging.getLogger(__name__)
 
 WORK_CHECK_TIMEOUT = 10
@@ -27,19 +26,16 @@ MINIMUM_JOB_EXTENSION = timedelta(minutes=2)
 
 HEARTBEAT_PERIOD_SEC = 30
 
-# pylint: disable=too-few-public-methods
 class BuildJobResult(object):
   """ Build job result enum """
   INCOMPLETE = 'incomplete'
   COMPLETE = 'complete'
   ERROR = 'error'
 
-# pylint: disable=too-many-instance-attributes
 class BuilderServer(object):
   """ Server which handles both HTTP and WAMP requests, managing the full state of the build
       controller.
   """
-  # pylint: disable=too-many-arguments
   def __init__(self, registry_hostname, queue, build_logs, user_files, lifecycle_manager_klass,
                lifecycle_manager_config, manager_hostname):
     self._loop = None

From 0e69222af163ed6c1eb48c2afefffd443dbe96ba Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Tue, 17 Feb 2015 15:32:54 -0500
Subject: [PATCH 15/24] test: add TROLLIUSDEBUG=1 to test script

---
 local-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/local-test.sh b/local-test.sh
index a54491969..2a85148bf 100755
--- a/local-test.sh
+++ b/local-test.sh
@@ -1 +1 @@
-TEST=true python -m unittest discover
+TEST=true TROLLIUSDEBUG=1 python -m unittest discover

From 1a71925125e076e7762494ffbc8f1a392f89b76c Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Tue, 17 Feb 2015 17:02:37 -0500
Subject: [PATCH 16/24] buildreporter: remove unused logging

---
 buildman/jobutil/buildreporter.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/buildman/jobutil/buildreporter.py b/buildman/jobutil/buildreporter.py
index 09a74a565..0ff798eae 100644
--- a/buildman/jobutil/buildreporter.py
+++ b/buildman/jobutil/buildreporter.py
@@ -1,12 +1,8 @@
-import logging
 from trollius import From
 
 from util.cloudwatch import get_queue
 
 
-logger = logging.getLogger(__name__)
-
-
 class BuildReporter(object):
   """
   Base class for reporting build statuses to a metrics service.

From 5790d7d8cce8c6da4c71e17257b28e35e325fc61 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Tue, 17 Feb 2015 17:03:12 -0500
Subject: [PATCH 17/24] buildman: build_metrics call correct method

---
 buildman/server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildman/server.py b/buildman/server.py
index 900746c6e..82eb39448 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -131,7 +131,7 @@ class BuilderServer(object):
                                   minimum_extension=MINIMUM_JOB_EXTENSION)
 
   def _job_complete(self, build_job, job_status):
-    build_metrics.report(job_status)
+    build_metrics.report_completion_status(job_status)
 
     if job_status == BuildJobResult.INCOMPLETE:
       self._queue.incomplete(build_job.job_item, restore_retry=False, retry_after=30)

From f53dea46b7ba1f5753827dc98c2b8c40f7f65524 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Wed, 18 Feb 2015 14:13:36 -0500
Subject: [PATCH 18/24] buildman: address PR #11 comments

---
 buildman/enums.py                 | 12 +++++++++++
 buildman/jobutil/buildreporter.py | 17 ++++++++-------
 buildman/server.py                | 27 ++++++++++--------------
 util/cloudwatch.py                | 19 ++++++++---------
 util/queuemetrics.py              | 35 +++++++++++++------------------
 5 files changed, 55 insertions(+), 55 deletions(-)
 create mode 100644 buildman/enums.py

diff --git a/buildman/enums.py b/buildman/enums.py
new file mode 100644
index 000000000..3d38217fe
--- /dev/null
+++ b/buildman/enums.py
@@ -0,0 +1,12 @@
+class BuildJobResult(object):
+  """ Build job result enum """
+  INCOMPLETE = 'incomplete'
+  COMPLETE = 'complete'
+  ERROR = 'error'
+
+
+class BuildServerStatus(object):
+  """ Build server status enum """
+  STARTING = 'starting'
+  RUNNING = 'running'
+  SHUTDOWN = 'shutting_down'
diff --git a/buildman/jobutil/buildreporter.py b/buildman/jobutil/buildreporter.py
index 0ff798eae..b295c33ea 100644
--- a/buildman/jobutil/buildreporter.py
+++ b/buildman/jobutil/buildreporter.py
@@ -1,5 +1,6 @@
 from trollius import From
 
+from buildman.enums import BuildJobResult
 from util.cloudwatch import get_queue
 
 
@@ -28,6 +29,9 @@ class CloudWatchBuildReporter(BuildReporter):
   Implements a BuildReporter for Amazon's CloudWatch.
   """
   def __init__(self, queue, namespace_name, completed_name, failed_name, incompleted_name):
+    if None in (queue, namespace_name, completed_name, failed_name, incompleted_name):
+      raise TypeError
+
     self._queue = queue
     self._namespace_name = namespace_name
     self._completed_name = completed_name
@@ -35,11 +39,11 @@ class CloudWatchBuildReporter(BuildReporter):
     self._incompleted_name = incompleted_name
 
   def report_completion_status(self, status):
-    if status == 'complete':
+    if status == BuildJobResult.COMPLETE:
       status_name = self._completed_name
-    elif status == 'error':
+    elif status == BuildJobResult.ERROR:
       status_name = self._failed_name
-    elif status == 'incomplete':
+    elif status == BuildJobResult.INCOMPLETE:
       status_name = self._incompleted_name
     else:
       return
@@ -53,9 +57,8 @@ class BuildMetrics(object):
   """
   def __init__(self, app=None):
     self._app = app
-    if app is None:
-      self._reporter = None
-    else:
+    self._reporter = NullReporter()
+    if app is not None:
       reporter_type = app.config.get('BUILD_METRICS_TYPE', 'Null')
       if reporter_type == 'CloudWatch':
         namespace = app.config.get('BUILD_METRICS_NAMESPACE')
@@ -65,8 +68,6 @@ class BuildMetrics(object):
         request_queue = get_queue(app)
         self._reporter = CloudWatchBuildReporter(request_queue, namespace, completed_name,
                                                  failed_name, incompleted_name)
-      else:
-        self._reporter = NullReporter()
 
   def __getattr__(self, name):
     return getattr(self._reporter, name, None)
diff --git a/buildman/server.py b/buildman/server.py
index 82eb39448..8f788d6cc 100644
--- a/buildman/server.py
+++ b/buildman/server.py
@@ -12,6 +12,7 @@ from trollius.tasks import Task
 from trollius.coroutines import From
 from datetime import timedelta
 
+from buildman.enums import BuildJobResult, BuildServerStatus
 from buildman.jobutil.buildstatus import StatusHandler
 from buildman.jobutil.buildjob import BuildJob, BuildJobLoadException
 from data import database
@@ -26,12 +27,6 @@ MINIMUM_JOB_EXTENSION = timedelta(minutes=2)
 
 HEARTBEAT_PERIOD_SEC = 30
 
-class BuildJobResult(object):
-  """ Build job result enum """
-  INCOMPLETE = 'incomplete'
-  COMPLETE = 'complete'
-  ERROR = 'error'
-
 class BuilderServer(object):
   """ Server which handles both HTTP and WAMP requests, managing the full state of the build
       controller.
@@ -39,7 +34,7 @@ class BuilderServer(object):
   def __init__(self, registry_hostname, queue, build_logs, user_files, lifecycle_manager_klass,
                lifecycle_manager_config, manager_hostname):
     self._loop = None
-    self._current_status = 'starting'
+    self._current_status = BuildServerStatus.STARTING
     self._current_components = []
     self._job_count = 0
 
@@ -59,7 +54,7 @@ class BuilderServer(object):
     self._lifecycle_manager_config = lifecycle_manager_config
 
     self._shutdown_event = Event()
-    self._current_status = 'running'
+    self._current_status = BuildServerStatus.RUNNING
 
     self._register_controller()
 
@@ -83,13 +78,13 @@ class BuilderServer(object):
     logger.debug('Starting server on port %s, with controller on port %s', websocket_port,
                  controller_port)
 
-    tasks = [
+    TASKS = [
       Task(self._initialize(loop, host, websocket_port, controller_port, ssl)),
       Task(self._queue_metrics_updater()),
     ]
 
     try:
-      loop.run_until_complete(trollius.wait(tasks))
+      loop.run_until_complete(trollius.wait(TASKS))
     except KeyboardInterrupt:
       pass
     finally:
@@ -97,7 +92,7 @@ class BuilderServer(object):
 
   def close(self):
     logger.debug('Requested server shutdown')
-    self._current_status = 'shutting_down'
+    self._current_status = BuildServerStatus.SHUTDOWN
     self._lifecycle_manager.shutdown()
     self._shutdown_event.wait()
     logger.debug('Shutting down server')
@@ -131,8 +126,6 @@ class BuilderServer(object):
                                   minimum_extension=MINIMUM_JOB_EXTENSION)
 
   def _job_complete(self, build_job, job_status):
-    build_metrics.report_completion_status(job_status)
-
     if job_status == BuildJobResult.INCOMPLETE:
       self._queue.incomplete(build_job.job_item, restore_retry=False, retry_after=30)
     else:
@@ -140,12 +133,14 @@ class BuilderServer(object):
 
     self._job_count = self._job_count - 1
 
-    if self._current_status == 'shutting_down' and not self._job_count:
+    if self._current_status == BuildServerStatus.SHUTDOWN and not self._job_count:
       self._shutdown_event.set()
 
+    build_metrics.report_completion_status(job_status)
+
   @trollius.coroutine
   def _work_checker(self):
-    while self._current_status == 'running':
+    while self._current_status == BuildServerStatus.RUNNING:
       with database.CloseForLongOperation(app.config):
         yield From(trollius.sleep(WORK_CHECK_TIMEOUT))
 
@@ -178,7 +173,7 @@ class BuilderServer(object):
 
   @trollius.coroutine
   def _queue_metrics_updater(self):
-    while self._current_status == 'running':
+    while self._current_status == BuildServerStatus.RUNNING:
       yield From(trollius.sleep(30))
       self._queue.update_metrics()
 
diff --git a/util/cloudwatch.py b/util/cloudwatch.py
index cf7aed392..eca563164 100644
--- a/util/cloudwatch.py
+++ b/util/cloudwatch.py
@@ -1,27 +1,26 @@
 import logging
 import boto
+import thread
 
 from Queue import Queue
 from threading import Thread
 
 
 logger = logging.getLogger(__name__)
-queue = None
 
 def get_queue(app):
   """
   Returns a queue to the CloudWatchSender. If a queue/sender do not exist, creates them.
   """
-  global queue
-  if queue is None:
-    access_key = app.config.get('QUEUE_METRICS_AWS_ACCESS_KEY')
-    secret_key = app.config.get('QUEUE_METRICS_AWS_SECRET_KEY')
-    queue = Queue()
-    sender = CloudWatchSender(queue, access_key, secret_key)
-    sender.start()
-  else:
-    return queue
+  access_key = app.config.get('CLOUDWATCH_AWS_ACCESS_KEY')
+  secret_key = app.config.get('CLOUDWATCH_AWS_SECRET_KEY')
+  if None in (access_key, secret_key):
+    raise TypeError
 
+  queue = Queue()
+  sender = CloudWatchSender(queue, access_key, secret_key)
+  sender.start()
+  return queue
 
 class CloudWatchSender(Thread):
   """
diff --git a/util/queuemetrics.py b/util/queuemetrics.py
index d84ced129..0f7090801 100644
--- a/util/queuemetrics.py
+++ b/util/queuemetrics.py
@@ -14,6 +14,9 @@ class NullReporter(object):
 class CloudWatchReporter(object):
   """ CloudWatchReporter reports work queue metrics to CloudWatch """
   def __init__(self, request_queue, namespace, need_capacity_name, build_percent_name):
+    if None in (request_queue, namespace, need_capacity_name, build_percent_name):
+      raise TypeError
+
     self._namespace = namespace
     self._need_capacity_name = need_capacity_name
     self._build_percent_name = build_percent_name
@@ -39,28 +42,18 @@ class QueueMetrics(object):
   QueueMetrics initializes a reporter for recording metrics of work queues.
   """
   def __init__(self, app=None):
-    self.app = app
-    self.sender = None
+    self._app = app
+    self._reporter = NullReporter()
     if app is not None:
-      self.state = self.init_app(app)
-    else:
-      self.state = None
+      reporter_type = app.config.get('QUEUE_METRICS_TYPE', 'Null')
+      if reporter_type == 'CloudWatch':
+        namespace = app.config.get('QUEUE_METRICS_NAMESPACE')
+        req_capacity_name = app.config.get('QUEUE_METRICS_CAPACITY_SHORTAGE_NAME')
+        build_percent_name = app.config.get('QUEUE_METRICS_BUILD_PERCENT_NAME')
 
-  def init_app(self, app):
-    analytics_type = app.config.get('QUEUE_METRICS_TYPE', 'Null')
-
-    if analytics_type == 'CloudWatch':
-      namespace = app.config.get('QUEUE_METRICS_NAMESPACE')
-      req_capacity_name = app.config.get('QUEUE_METRICS_CAPACITY_SHORTAGE_NAME')
-      build_percent_name = app.config.get('QUEUE_METRICS_BUILD_PERCENT_NAME')
-
-      request_queue = get_queue(app)
-      reporter = CloudWatchReporter(request_queue, namespace, req_capacity_name,
-                                    build_percent_name)
-    else:
-      reporter = NullReporter()
-
-    return reporter
+        request_queue = get_queue(app)
+        self._reporter = CloudWatchReporter(request_queue, namespace, req_capacity_name,
+                                        build_percent_name)
 
   def __getattr__(self, name):
-    return getattr(self.state, name, None)
+    return getattr(self._reporter, name, None)

From 0d38e0b00b5db1cf52f9c028bf2ce07c337738b3 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Wed, 18 Feb 2015 16:05:36 -0500
Subject: [PATCH 19/24] metrics: use config['name'] to get metric conf

---
 buildman/jobutil/buildreporter.py | 11 ++++-------
 util/queuemetrics.py              |  9 +++------
 2 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/buildman/jobutil/buildreporter.py b/buildman/jobutil/buildreporter.py
index b295c33ea..992b7a06e 100644
--- a/buildman/jobutil/buildreporter.py
+++ b/buildman/jobutil/buildreporter.py
@@ -29,9 +29,6 @@ class CloudWatchBuildReporter(BuildReporter):
   Implements a BuildReporter for Amazon's CloudWatch.
   """
   def __init__(self, queue, namespace_name, completed_name, failed_name, incompleted_name):
-    if None in (queue, namespace_name, completed_name, failed_name, incompleted_name):
-      raise TypeError
-
     self._queue = queue
     self._namespace_name = namespace_name
     self._completed_name = completed_name
@@ -61,10 +58,10 @@ class BuildMetrics(object):
     if app is not None:
       reporter_type = app.config.get('BUILD_METRICS_TYPE', 'Null')
       if reporter_type == 'CloudWatch':
-        namespace = app.config.get('BUILD_METRICS_NAMESPACE')
-        completed_name = app.config.get('BUILD_METRICS_COMPLETED_NAME')
-        failed_name = app.config.get('BUILD_METRICS_FAILED_NAME')
-        incompleted_name = app.config.get('BUILD_METRICS_INCOMPLETED_NAME')
+        namespace = app.config['BUILD_METRICS_NAMESPACE']
+        completed_name = app.config['BUILD_METRICS_COMPLETED_NAME']
+        failed_name = app.config['BUILD_METRICS_FAILED_NAME']
+        incompleted_name = app.config['BUILD_METRICS_INCOMPLETED_NAME']
         request_queue = get_queue(app)
         self._reporter = CloudWatchBuildReporter(request_queue, namespace, completed_name,
                                                  failed_name, incompleted_name)
diff --git a/util/queuemetrics.py b/util/queuemetrics.py
index 0f7090801..9e0a549f4 100644
--- a/util/queuemetrics.py
+++ b/util/queuemetrics.py
@@ -14,9 +14,6 @@ class NullReporter(object):
 class CloudWatchReporter(object):
   """ CloudWatchReporter reports work queue metrics to CloudWatch """
   def __init__(self, request_queue, namespace, need_capacity_name, build_percent_name):
-    if None in (request_queue, namespace, need_capacity_name, build_percent_name):
-      raise TypeError
-
     self._namespace = namespace
     self._need_capacity_name = need_capacity_name
     self._build_percent_name = build_percent_name
@@ -47,9 +44,9 @@ class QueueMetrics(object):
     if app is not None:
       reporter_type = app.config.get('QUEUE_METRICS_TYPE', 'Null')
       if reporter_type == 'CloudWatch':
-        namespace = app.config.get('QUEUE_METRICS_NAMESPACE')
-        req_capacity_name = app.config.get('QUEUE_METRICS_CAPACITY_SHORTAGE_NAME')
-        build_percent_name = app.config.get('QUEUE_METRICS_BUILD_PERCENT_NAME')
+        namespace = app.config['QUEUE_METRICS_NAMESPACE']
+        req_capacity_name = app.config['QUEUE_METRICS_CAPACITY_SHORTAGE_NAME']
+        build_percent_name = app.config['QUEUE_METRICS_BUILD_PERCENT_NAME']
 
         request_queue = get_queue(app)
         self._reporter = CloudWatchReporter(request_queue, namespace, req_capacity_name,

From 59b794dd61646b9baa3f7070fc904c680d2f5c24 Mon Sep 17 00:00:00 2001
From: Jake Moshenko <jake.moshenko@coreos.com>
Date: Fri, 13 Feb 2015 16:28:45 -0500
Subject: [PATCH 20/24] Move the creation of images to when the JSON is
 uploaded.

---
 data/model/legacy.py  |  2 --
 endpoints/index.py    | 30 ++----------------------------
 endpoints/registry.py | 13 ++++++++++---
 3 files changed, 12 insertions(+), 33 deletions(-)

diff --git a/data/model/legacy.py b/data/model/legacy.py
index fe675767e..f04283c5a 100644
--- a/data/model/legacy.py
+++ b/data/model/legacy.py
@@ -1628,8 +1628,6 @@ def garbage_collect_repository(namespace_name, repository_name):
     logger.info('Garbage collecting storage for images: %s', to_remove)
     _garbage_collect_storage(storage_id_whitelist)
 
-  return len(to_remove)
-
 
 def _garbage_collect_storage(storage_id_whitelist):
   if len(storage_id_whitelist) == 0:
diff --git a/endpoints/index.py b/endpoints/index.py
index d1a902915..f2e1f7411 100644
--- a/endpoints/index.py
+++ b/endpoints/index.py
@@ -222,32 +222,6 @@ def create_repository(namespace, repository):
     repo = model.create_repository(namespace, repository,
                                    get_authenticated_user())
 
-  logger.debug('Determining already added images')
-  added_images = OrderedDict([(desc['id'], desc) for desc in image_descriptions])
-  new_repo_images = dict(added_images)
-
-  # Optimization: Lookup any existing images in the repository with matching docker IDs and
-  # remove them from the added dict, so we don't need to look them up one-by-one.
-  def chunks(l, n):
-    for i in xrange(0, len(l), n):
-      yield l[i:i+n]
-
-  # Note: We do this in chunks in an effort to not hit the SQL query size limit.
-  for chunk in chunks(new_repo_images.keys(), 50):
-    existing_images = model.lookup_repository_images(namespace, repository, chunk)
-    for existing in existing_images:
-      added_images.pop(existing.docker_image_id)
-
-  logger.debug('Creating/Linking necessary images')
-  username = get_authenticated_user() and get_authenticated_user().username
-  translations = {}
-  for image_description in added_images.values():
-    model.find_create_or_link_image(image_description['id'], repo, username,
-                                    translations, storage.preferred_locations[0])
-
-
-  logger.debug('Created images')
-  track_and_log('push_repo', repo)
   return make_response('Created', 201)
 
 
@@ -280,7 +254,7 @@ def update_images(namespace, repository):
       event.publish_event_data('docker-cli', user_data)
 
     logger.debug('GCing repository')
-    num_removed = model.garbage_collect_repository(namespace, repository)
+    model.garbage_collect_repository(namespace, repository)
 
     # Generate a job for each notification that has been added to this repo
     logger.debug('Adding notifications for repository')
@@ -288,8 +262,8 @@ def update_images(namespace, repository):
     updated_tags = session.get('pushed_tags', {})
     event_data = {
       'updated_tags': updated_tags,
-      'pruned_image_count': num_removed
     }
+    track_and_log('push_repo', repo)
     spawn_notification(repo, 'repo_push', event_data)
     return make_response('Updated', 204)
 
diff --git a/endpoints/registry.py b/endpoints/registry.py
index 5178f3a83..dc5069e22 100644
--- a/endpoints/registry.py
+++ b/endpoints/registry.py
@@ -9,6 +9,7 @@ from time import time
 
 from app import storage as store, image_diff_queue, app
 from auth.auth import process_auth, extract_namespace_repo_from_session
+from auth.auth_context import get_authenticated_user
 from util import checksums, changes
 from util.http import abort, exact_abort
 from auth.permissions import (ReadRepositoryPermission,
@@ -456,9 +457,15 @@ def put_image_json(namespace, repository, image_id):
   logger.debug('Looking up repo image')
   repo_image = model.get_repo_image_extended(namespace, repository, image_id)
   if not repo_image:
-    logger.debug('Image not found')
-    abort(404, 'Image %(image_id)s not found', issue='unknown-image',
-          image_id=image_id)
+    logger.debug('Image not found, creating image')
+    repo = model.get_repository(namespace, repository)
+    if repo is None:
+      abort(404, 'Repository does not exist: %(namespace)s/%(repository)s', issue='no-repo',
+            namespace=namespace, repository=repository)
+
+    username = get_authenticated_user() and get_authenticated_user().username
+    repo_image = model.find_create_or_link_image(image_id, repo, username, {},
+                                                 store.preferred_locations[0])
 
   uuid = repo_image.storage.uuid
 

From 04b06547b8a8fc5e9a7267e940ce750ce0a042f0 Mon Sep 17 00:00:00 2001
From: Jake Moshenko <jake.moshenko@coreos.com>
Date: Wed, 18 Feb 2015 16:33:28 -0500
Subject: [PATCH 21/24] Remove all of the timeouts since they were not doing
 the right thing anyway.

---
 conf/gunicorn_local.py      | 1 -
 conf/gunicorn_registry.py   | 1 -
 conf/gunicorn_verbs.py      | 1 -
 conf/gunicorn_web.py        | 1 -
 conf/proxy-server-base.conf | 4 ----
 conf/server-base.conf       | 3 ---
 6 files changed, 11 deletions(-)

diff --git a/conf/gunicorn_local.py b/conf/gunicorn_local.py
index 6987041be..49a30682d 100644
--- a/conf/gunicorn_local.py
+++ b/conf/gunicorn_local.py
@@ -1,7 +1,6 @@
 bind = '0.0.0.0:5000'
 workers = 2
 worker_class = 'gevent'
-timeout = 2000
 daemon = False
 logconfig = 'conf/logging_debug.conf'
 pythonpath = '.'
diff --git a/conf/gunicorn_registry.py b/conf/gunicorn_registry.py
index 4f7bb37f2..944608868 100644
--- a/conf/gunicorn_registry.py
+++ b/conf/gunicorn_registry.py
@@ -1,7 +1,6 @@
 bind = 'unix:/tmp/gunicorn_registry.sock'
 workers = 8
 worker_class = 'gevent'
-timeout = 2000
 logconfig = 'conf/logging.conf'
 pythonpath = '.'
 preload_app = True
diff --git a/conf/gunicorn_verbs.py b/conf/gunicorn_verbs.py
index eaf8041df..f329a8cbe 100644
--- a/conf/gunicorn_verbs.py
+++ b/conf/gunicorn_verbs.py
@@ -1,6 +1,5 @@
 bind = 'unix:/tmp/gunicorn_verbs.sock'
 workers = 4
-timeout = 2000
 logconfig = 'conf/logging.conf'
 pythonpath = '.'
 preload_app = True
diff --git a/conf/gunicorn_web.py b/conf/gunicorn_web.py
index bdfa8001a..cb9f78d24 100644
--- a/conf/gunicorn_web.py
+++ b/conf/gunicorn_web.py
@@ -1,7 +1,6 @@
 bind = 'unix:/tmp/gunicorn_web.sock'
 workers = 2
 worker_class = 'gevent'
-timeout = 30
 logconfig = 'conf/logging.conf'
 pythonpath = '.'
 preload_app = True
diff --git a/conf/proxy-server-base.conf b/conf/proxy-server-base.conf
index fb2f3f962..6230dbfd8 100644
--- a/conf/proxy-server-base.conf
+++ b/conf/proxy-server-base.conf
@@ -34,7 +34,6 @@ location /v1/repositories/ {
     proxy_request_buffering off;
 
     proxy_pass http://registry_app_server;
-    proxy_read_timeout 2000;
     proxy_temp_path /var/log/nginx/proxy_temp 1 2;
 
     client_max_body_size 20G;
@@ -48,7 +47,6 @@ location /v1/ {
     proxy_request_buffering off;
 
     proxy_pass   http://registry_app_server;
-    proxy_read_timeout 2000;
     proxy_temp_path /var/log/nginx/proxy_temp 1 2;
 
     client_max_body_size 20G;
@@ -60,7 +58,6 @@ location /c1/ {
     proxy_request_buffering off;
 
     proxy_pass   http://verbs_app_server;
-    proxy_read_timeout 2000;
     proxy_temp_path /var/log/nginx/proxy_temp 1 2;
 
     limit_req zone=api burst=5 nodelay;
@@ -80,7 +77,6 @@ location /v1/_ping {
 
 location ~ ^/b1/controller(/?)(.*) {
     proxy_pass http://build_manager_controller_server/$2;
-    proxy_read_timeout 2000;
 }
 
 location ~ ^/b1/socket(/?)(.*) {
diff --git a/conf/server-base.conf b/conf/server-base.conf
index d5b211c52..4122a99eb 100644
--- a/conf/server-base.conf
+++ b/conf/server-base.conf
@@ -35,7 +35,6 @@ location /v1/ {
     proxy_request_buffering off;
 
     proxy_pass   http://registry_app_server;
-    proxy_read_timeout 2000;
     proxy_temp_path /var/log/nginx/proxy_temp 1 2;
 
     client_max_body_size 20G;
@@ -47,7 +46,6 @@ location /c1/ {
     proxy_request_buffering off;
 
     proxy_pass   http://verbs_app_server;
-    proxy_read_timeout 2000;
     proxy_temp_path /var/log/nginx/proxy_temp 1 2;
 }
 
@@ -65,7 +63,6 @@ location /v1/_ping {
 
 location ~ ^/b1/controller(/?)(.*) {
     proxy_pass http://build_manager_controller_server/$2;
-    proxy_read_timeout 2000;
 }
 
 location ~ ^/b1/socket(/?)(.*) {

From 41108a0856ae57b3b7677322e7f0c7738bcd0364 Mon Sep 17 00:00:00 2001
From: Jake Moshenko <jake.moshenko@coreos.com>
Date: Wed, 18 Feb 2015 16:37:38 -0500
Subject: [PATCH 22/24] Allow tags to be marked as hidden. Create a hidden tag
 on every image during a push to prevent them from getting GCed.

---
 config.py                                     |   5 ++-
 data/database.py                              |   1 +
 ...1fcf9_allow_tags_to_be_marked_as_hidden.py |  26 +++++++++++
 data/model/legacy.py                          |  41 ++++++++++++------
 endpoints/registry.py                         |   4 ++
 test/data/test.db                             | Bin 729088 -> 729088 bytes
 6 files changed, 63 insertions(+), 14 deletions(-)
 create mode 100644 data/migrations/versions/4ef04c61fcf9_allow_tags_to_be_marked_as_hidden.py

diff --git a/config.py b/config.py
index 6a6c9c2e6..bedddcda7 100644
--- a/config.py
+++ b/config.py
@@ -194,4 +194,7 @@ class DefaultConfig(object):
   SYSTEM_SERVICES_PATH = "conf/init/"
 
   # Services that should not be shown in the logs view.
-  SYSTEM_SERVICE_BLACKLIST = []
\ No newline at end of file
+  SYSTEM_SERVICE_BLACKLIST = []
+
+  # Temporary tag expiration in seconds, this may actually be longer based on GC policy
+  PUSH_TEMP_TAG_EXPIRATION_S = 60 * 60
diff --git a/data/database.py b/data/database.py
index d23157c0c..162057530 100644
--- a/data/database.py
+++ b/data/database.py
@@ -469,6 +469,7 @@ class RepositoryTag(BaseModel):
   repository = ForeignKeyField(Repository)
   lifetime_start_ts = IntegerField(default=_get_epoch_timestamp)
   lifetime_end_ts = IntegerField(null=True, index=True)
+  hidden = BooleanField(default=False)
 
   class Meta:
     database = db
diff --git a/data/migrations/versions/4ef04c61fcf9_allow_tags_to_be_marked_as_hidden.py b/data/migrations/versions/4ef04c61fcf9_allow_tags_to_be_marked_as_hidden.py
new file mode 100644
index 000000000..e4fc1ea5e
--- /dev/null
+++ b/data/migrations/versions/4ef04c61fcf9_allow_tags_to_be_marked_as_hidden.py
@@ -0,0 +1,26 @@
+"""Allow tags to be marked as hidden.
+
+Revision ID: 4ef04c61fcf9
+Revises: 509d2857566f
+Create Date: 2015-02-18 16:34:16.586129
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '4ef04c61fcf9'
+down_revision = '509d2857566f'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('repositorytag', sa.Column('hidden', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()))
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('repositorytag', 'hidden')
+    ### end Alembic commands ###
diff --git a/data/model/legacy.py b/data/model/legacy.py
index f04283c5a..331bf2720 100644
--- a/data/model/legacy.py
+++ b/data/model/legacy.py
@@ -5,6 +5,7 @@ import json
 import time
 
 from datetime import datetime, timedelta, date
+from uuid import uuid4
 
 from data.database import (User, Repository, Image, AccessToken, Role, RepositoryPermission,
                            Visibility, RepositoryTag, EmailConfirmation, FederatedLogin,
@@ -1561,15 +1562,20 @@ def _tag_alive(query):
                      (RepositoryTag.lifetime_end_ts > int(time.time())))
 
 
-def list_repository_tags(namespace_name, repository_name):
-  return _tag_alive(RepositoryTag
-                    .select(RepositoryTag, Image)
-                    .join(Repository)
-                    .join(Namespace, on=(Repository.namespace_user == Namespace.id))
-                    .switch(RepositoryTag)
-                    .join(Image)
-                    .where(Repository.name == repository_name,
-                           Namespace.username == namespace_name))
+def list_repository_tags(namespace_name, repository_name, include_hidden=False):
+  query = _tag_alive(RepositoryTag
+                     .select(RepositoryTag, Image)
+                     .join(Repository)
+                     .join(Namespace, on=(Repository.namespace_user == Namespace.id))
+                     .switch(RepositoryTag)
+                     .join(Image)
+                     .where(Repository.name == repository_name,
+                            Namespace.username == namespace_name))
+
+  if not include_hidden:
+    query = query.where(RepositoryTag.hidden == False)
+
+  return query
 
 
 def _garbage_collect_tags(namespace_name, repository_name):
@@ -1786,10 +1792,8 @@ def create_or_update_tag(namespace_name, repository_name, tag_name,
       # No tag that needs to be ended
       pass
 
-    tag = RepositoryTag.create(repository=repo, image=image, name=tag_name,
-                               lifetime_start_ts=now_ts)
-
-    return tag
+    return RepositoryTag.create(repository=repo, image=image, name=tag_name,
+                                lifetime_start_ts=now_ts)
 
 
 def delete_tag(namespace_name, repository_name, tag_name):
@@ -1812,6 +1816,17 @@ def delete_tag(namespace_name, repository_name, tag_name):
     found.save()
 
 
+def create_temporary_hidden_tag(repo, image, expiration_s):
+  """ Create a tag with a defined timeline, that will not appear in the UI or CLI. Returns the name
+      of the temporary tag. """
+  now_ts = int(time.time())
+  expire_ts = now_ts + expiration_s
+  tag_name = str(uuid4())
+  RepositoryTag.create(repository=repo, image=image, name=tag_name, lifetime_start_ts=now_ts,
+                       lifetime_end_ts=expire_ts, hidden=True)
+  return tag_name
+
+
 def purge_all_repository_tags(namespace_name, repository_name):
   """ Immediately purge all repository tags without respecting the lifeline procedure """
   try:
diff --git a/endpoints/registry.py b/endpoints/registry.py
index dc5069e22..8222212a7 100644
--- a/endpoints/registry.py
+++ b/endpoints/registry.py
@@ -467,6 +467,10 @@ def put_image_json(namespace, repository, image_id):
     repo_image = model.find_create_or_link_image(image_id, repo, username, {},
                                                  store.preferred_locations[0])
 
+  # Create a temporary tag to prevent this image from getting garbage collected while the push
+  # is in progress.
+  model.create_temporary_hidden_tag(repo, repo_image, app.config['PUSH_TEMP_TAG_EXPIRATION_S'])
+
   uuid = repo_image.storage.uuid
 
   if image_id != data['id']:
diff --git a/test/data/test.db b/test/data/test.db
index 0856c2f6acf315003dfdb7e0db326e7a5db6c37a..4da80c978b4df80b01a42e5de32d324dd0ae295b 100644
GIT binary patch
delta 8193
zcmeHMd3aP+maiqLdR0kPg)B&dm=ywqS9$wFB9K%isiZ1<RhAOa_v*cZgpiPxrI7%N
z-Gj(h!WD!-%cg)r+jL?pBG96(!hoOM;HV?$eCp7&qK?zu(kOE(3zDe)*~1_6&(uHl
zeZTw8@0@$iz4!d?d&`ecS$=%V;Wg_sh#)Ib635}BBvUw}QY0?(f{N1=PX&T>P?cCE
zba*|&a>mQ=R-O6$bf|no7dC2ITJo6m*Jg*#Y}kQe!j7X+D;G}<b!_axEJvRp;&O_+
zLP;S%rt588t3NG-LQ6xX*yulQ^(O!IZCmKg5W<AS`>vJ!%V(2A3m)ylu&*wTiC!I<
z8v5+f9a!eGcNFj&KfN!sX;TlDb7IG;>L(ATgmO0fG1JKxW?A-&D?{5imtvL^ISb<Q
ze!DI7<z|G9`Dp4ZY45E1S!e_5!eT#tU}sYIbA=(>V>>YGgV#^SSN}dg)caTu7F(1q
zCT@QHa%kojKQ?NT{rxe9j&q@7TS_tUW6QEi>&3&NxUC4Yj*m%<JG1m^=-I6>({JB-
zG?9F4Zisk%2WI+9+I-8G1QojQcn`+LA6Q&ck?siHzs-*&?w~(NfB(ar(8X<~*p%)y
zUDG^SQ6XkKickBGmMz5Fx?M{n%o?ADPQ5oG0t-opGD7;ELM-mpS&POV?zM(0dv;)V
zZXwr9G#z+3^tYZKEcr_27gG-PR6fwN<L5m)w0%80bo=YN1%eexl_c;W!KyeDkZ4?B
zL>eapG|BS;Mj};C?p)m5B(=>^vzpa~Ep2n!TI7}vXIYKgRmXWmAM5uy^1R-fGDiiU
z&%69oeLf)wwN*~K%uBlJ93_>`e4m@}F|<(bE_b`=8V^k|6hX2$QA**YkV27DX=*CV
z*+qsX2`08vvm!mprt#656&6f`nJ_F*5Al(-s7g2^@IjnW0wOM|GLOq_kd{=i3kCzI
z(x#8|7v}M$UdF*ymy~k8;<Cb`Qo5dS*ZADUWo%W2%UfFECQGVngsLh>v8$98DoU$K
z*5l`DX*Mt4>0<{X8xa`A3!H#fe65SiP<U0Kf|7_!;Y2Y~ki{iJ5ph`&0<iW~M&&s4
z+}Aq3`@4A&BE|O4E5iiyN?j2-$)v&Hm!~HfL6k{Czy*O&aE9VxN)*g1NXS7lAgXGB
zAzL~-+8Y!VQlCLd86=g_(N>U870=^pn<^c3ZQ{amuD!mjv)nbOtv$PZL1iJ?RUh!N
zvGdhMc@5cwBozA{&St5(g+i^@b)$#kV(bLTa3mkjcM1k^CEgkr$x#$9sW>4r0uD0`
z-~tigaG4N-0vV7MlAzjDNm1JdV=Aa_R27epE9;Pj_KxiNjg+&irM$3!5Xy;$U_rH7
z)7a{9%%P+f^O8!wrK#Rs-r0td|E^0Kiifsy3@?gQIGzmSV4k4<{dp2;k|6?ufD^O`
z#|lddxENlt6wOd7DFu0wRy&)!)V79TgQ^V1WN$7Z9eHwXW5CBZEb>Xtj-u?w^7h42
z-lF`ra*1hnF-`N9#8w9Uo{sqiOR5B!X)d1M+`{(f$u?L&+D-`q3q`&9cU?lGtddla
zq6jEDN8pe@5*HMS#f6{*>yu(6P9)o;W~F6*XZy|MxaKx>RJAOq3ixYUo!)kFq1{!!
zz+P0<B`Zo^xhvS(HgCRDqN@0MIauuV7ZEiri~}`a*Co0JgP>q@FtAs{$(axd$q{ea
z92^}WRF;%*jtf9cj7s1lr?R+621P*>c}WyFbQVhQFLdX-D~dV3+UMg+owW{!r?!@^
z_tX%DCA8qmuO#Zl(i(q#1;zTAN>6QGVR=b@KI1DWb=Fe_mA=7kd+X+az$jWIL^S;y
zT|xs-!LlLy(-DO8gy(2nQeaJ!L7A5cisD#>4z{W7b8krwHK)43Q{Y+X;G7cE+St`n
z!L@gEme9@`aei~9uZE-M@bg>jY}4E_motyEyK5TDTG5N&=rV>XMA=zRfChs;hZuqs
zOG~UO$hg9C5CccT-sA-eS0$QI36YREfmqNfH#GzXm&ts0r>8+(G{3sUSyVk|u7iVR
z<LvbEonmobQB9@aQ|0Mum6mt}S5s@-VtXUwoKs$l=6$P6A6mMUog!I=?5|FW4yuFS
zQXMtGvWz4Ka9)N`7#R{Ht2CS%B(KOcC5kL5Dr#4UBsZy9N=N_k>hYI(oh<3BtgCSm
zyqB+{J*?a7^0QTi{ye&}vbvP=I~jsz>m2!HT|t2;P=uGMBP%^Fwo0Uh!tbAA5F;Vb
z42x#m&?U?zga8f8oxlk-2up?)Ra^`gih-K3EUid{qz(l&m?nptCcJsUqQa^L1+@Za
zZ(taPC~tEs0$Jp6vqc2!AWC>|Ll?8qS>tS1^6M)KRrKNw9XV7d(k`&jSA@QAZs=lC
z^r^Nd`rvP(wflCvNpwNP3XArCeexRzkA8H_uAf_QyL%*ZoPN4Cygn1rgW3A@89%X>
zKU&JZPqOu&PrL0L{p^0X-l)-PS6D_x>Hqmubr0{7z@{5yi`3t4#(+#aNT$>ONV+DC
zT>C$suWwuQwiJD<?YF-kmGY*wyC@RotkvJ{8jBd@`DattJyKxkhlp<TEY=l<CuaJw
zgbn2@b?C78(S-Hx)L$>l^w#Fp`km!255*PJ9vA1yXWZ@zLa1^$${8==;@y6a+r^-l
zuIk4l*HwM_M83YD-ceiZuB&%eG1YWA;cyhXtLr^ppIaa*O3Q`n3RfOi;O_&I&$BD(
z{st*PmkNe^1I^30EQ7+Jf(#qv6-g!oASo8KOqG=R4b2^Ddq+phJhl1eNhYQc#8gt8
zN`p|O1(s!dQzG88W&9M{hlq$?BJ$|$3@RWCq|DMd%?3ekNhFQKc?@z(kpxkY6-iMT
zbZKp59yZu_88m5K<b-;P>X)cY3QoW;?f#oV2`MBuFl470=!EQw2(3xu)0$QQhbpKn
zN3$eI)gXvgik3h$axzYlYLF!ug_VNn$#sz>mhUKdn(Vu=F7glAQ@+E(w7PNnJkt+*
z?KgQT@`%mzQ)Z{j)cZ_7A+Ww1XCs%`x&tE5Gb%I(=rC}blXy@TkU5%TDOMmPg{2w2
zE?b9bv_2CYE@qif7)A-oxF~=`W|(lZqXHBdZ~>B)NyxFN(`Re@4YV4qS&=|m2fzcT
z20%_jS#jt~EUt*EB1s00U<Hoq`}SgF&$bMZ-Gm_XB2I#efKJK_xWok+oDi4*!LXn(
z2nrovYRJt{2}TtZnZ%(f2OtcI!zG!Z@qo$_5@-NM4#+6wegmE%lM=;&Zo?%|OQ2XJ
z1(zjK##!jt&=4sZR2^!)-@s<D6evy*7O-k5;3NYKE=bVPX<8!T7Q#q@fP`MZ-!L_U
z<{(L;M1W-G!knCf3%tO>nkPs(C=#S92hhZ2hO`WdQ5YehfKN&Z&qx6^8K5a#3<Npo
z&!RvFgQ$6#A$KOPij2g=jYE`G8E1kbhYKVl;tItCSXp5yfedmHMvW;|tMSc&>zdTu
z(J3|SXppNsh%eY?6jU%kaZthV&0Jw*(5nKih#WF>8)jsv46E=#l7dR`ur~sV4BCwd
z;!tXlkR?$Tc@Z^sL)5ZD^Hh+Aa)~OW7;bjB2;wR%#wAJ-C>lgEI^1oTk^wUf!lQ6+
z2yzU}A8rr=Tr1&mCJ+#KsDjEpwA|3U+<1SKbzsE{cA8~Hjz$kZY}}Z^g1Cp8gDt4G
zaDpJ|0Z`vT9xhH2MS%DZpu9E4J4afVuQMi~<!g-0$bnz1HD;kt*BG-#4h&gDq8X1E
zog+uVY$AI8k?%X|ABbLS96Qot7+(acUTe%8`3TthMq*di8f!*kKvg&r+qKS^I|A$b
zbe-{VS@-ZfM&G5+(|-Tt7)o>0KOQ#!<6+Ye40pp_euy3Wf$>|0jl+hSp|f<z+AwUG
z9yuKO!^7&3dBm`BYNWaT9}f>daldEXd&#)lc28a%TVGbmd29T|6}3er1!A#SK~;L{
zeSVQ}mlPM5mvCZPq060DQYMzVxB_o}CET%!91gEgSwd9SW8W_fXBPyxC{cYkt{P7y
z&$wL=y)oF<?4Pua`i@P<Dw+p-5%gFB^gSB;40h=jh!AOn9l|aRE<A`;<XKMa`^zCL
z=D594W2!KnGS`?tjan9Mh<?;E-r8zCZJTV{h&^pgFsw1eMLrmDS-)NPH{En?Bd9Y@
z+xJkb=|E)E1Mw?H8Slu<)U8tY<t{X7jcIcm6h&=DOV&lC_mwO%9lfJsx5jqG_O|V$
z?HSuv+XJ>1+dVd?&2CGw8LVGg-?zSQJ!0*#{=(X3oo#hl8S6N!!E)7d!Sa&jNy`Sy
z5=+41u`rf9EfLXwj{ZaR(dgaazbv|YbS9=5r`1MiuqG+2IY{AWQQnUt&7&ub!1=`k
z`1lbxw{QT@7=g2$19<ufoM|7x)B5r5?;O*u0|Tie42YI~JRt?sjJ+jLaef#!B_t1r
zg?R(;UBh9%VE`UC9Ok;`4B&ST$62)>9zAwMEL30sPa1)f(g023a2n>m0eH-Cn4UcV
z-+^hmZ=Dh~vmcKgjcF44XS$%RMX54}rx~AsX%cUl{Qw#pf9sQFrv4}6hCexo#tuhI
zjPNWLKdO5;Twv&jV}_$MBEx8`ZOB;{XD7Ibe$09kW9<YxfWb@4f$+QF6&pQ(Mc>3|
zJ7EuF#@Hypx^K=iu^)^#4|hF)CYXj2*xn({X=t7^W`F^^$x7LY(F2%q2n#1KMh~70
z35F5owthG^64S_ok%GNt0F1cR9@B;G<MqRX8bC*Z>Pyg?U;&3cg`FGXK-VSqLj#+s
zAMD*}`dxW(Jy+-OmF6*(VmZZ$ej-b2&}doQq-$&s%x!6Fn=$mZ!5(OtU+Bp%EO5iC
z5huraiwiijMx)W$E@*6jwq1a2`-W{&ydKw%q}0XfannfCBj%#i|NqwTfAg&Y^<<m<
z*z`qNwRP{dT-r>Szr@C`MkZ$MTkYr?VTMmDk;M4Ksa;#qBGSAA8~^3QCN^c}bd*e)
zd$9Dat9<0!UI#iw0g3j(mL<C6vrd#rn@cgQ>2Oxm8?#JkDQ!kru5r(wyyoA;qp=Kd
zYByLo9l7Wuw1o+CwyeL-%>5QeGg#o9)?T!ywwX*qAG79rjOyOqwy%1}AQuOe;^iqH
zR>n;`kG69_iT~TVB{PW6_oG}MC~d#`WJSW)F`uC4d7#YsB<WAp|33a1Bndz{A$j9M
z_TJ!LWE3IHgy|o@IPRtQo<I#Ea3<^X=VUehWj#730%v3|S7*F2<^wcqDsV=3_M92j
zy6_5mcPemBeQJGbRm$c^(aId)oV;gEQ_hhMa7V}m&gg9&&yU)7z6Y(&1y1hM%#EIp
zwx329bAfZ(AHSGT`?t!+(XzXNGjfBw-uOntQgr=p;LPo4-s#<?EkwJf0jGFw@tKOx
z_pd_5(}6Q_X6T{x6_5MSf$6}Be~|X`8978v?+o*o_hA#>$t&iR6Q|L&oxo|-Z;Cg)
zdG0r8(Jz5B?&=pGj=j9<JW757IJrHh63_CzThXZ}fYVYQeJb{s?>~()cLC?5?l*ot
zHL>D8v~(B5iN$0&qKeN?Mq_sa=j0cZj-0n9<e)9Pfs;9U_Ehm7(=MSIdw_F#kEqng
z^uB~X-UFPunbqlD-trA{?FCNu`G>3B!pwbW`(EI*t<@b&c;(z`l)Dc&CnjEgQphN}
z3q8LNIPH_K{$j=xD<4MElfap@_WFtO_Dxn~d<r;Ye{-xOiQJuw8lD2q+<UWbcvs#1
z7&`V8aOO-7&aL{xf4_`o?FY`(ieF{gcV}_v-TmerqbFR$F>XWKv=!;jsQxc+E3(3$
z-ct8upd$y&O&Arw<$U3t7iOc02f@p@Rhte>^7i=9>Vx2A%p-qYmv*0j6S{a1yd?Fm
zO&YIW%RtMX1}{m+&ptc8!}m41{&YASSN^Igxnsf>wCfp&Jlgic-k1;HT8WAe0q2<a
z-}x}@&BxB81Bb#H9)09q@uBSJdJmfoF~~reYtg;O&6}`j?uGTE5}U50>&JmTEoga}
zn7H^LT5<x|Q;#irFZ=HI=AhJ*z&<v3{^Eq0ugpWQo&@%!td{8+yVyE3>38NbW1o8G
zMEbwJLd$*!0TYvZ?T+Rf7Buc<ptMR`V&c<s52MFl4pUx#;jQdVhObcmDWFV#V@&dt
zl`qUj|9Q&nk2gD}{M|Y`>Qr^pz_&^bBn<uPf;j^%dBr>qi=7sGHgR>54{d)1oTOVL
z_K|Tr$D!O;!O0y*rZl8xx894Me-)ghU2?riytO3*NxugtvFF~oo>cqTHDr7Z;vW4`
z-e)P6lfOm{uR&J0X=@j_zrfF-W3K_{xN$|gi6=L2L$h87PWx+*jLuuWu?oHWI&hA@
z@!shaTKgxo@(tk3U9Bneo}hQ4iKl@xeP3(@dHLf=wE8q~M!e+a4SO0&(Z$oinfY)}
z9X~7P1GMZ6aK<G(v_7ftXaHS51Du*GTgGUM&wJ6Xe+5o^#uu?Ow%+AI#b<$YV(Nuw
zg=}dGI&c;^$DMkgp7hN2CB5g&4F-6(peO;Eg*O`<ypjUp&BOZ$cx@swswi<X!76f~
z?~(K7{iY(y&>MA6)Pqq^N1czlVLTW;F}gB(N%StPBKmane_QUd6j>G+G0PUqNy`;$
zy!CEt(7MhLYkki8fg#6cw$ZkkQ6}3e+cSO3E}7Sv(X&_0llm-I%zGoyXSwEdxOAAe
z*|7<k*JyTLRy^8uS5yxcVS6hk^3aLVs5m*wkMVn7e5dsC0umiajw;3CoCOaieQi(v
EA1@4Kng9R*

delta 8177
zcmeI1d3+RAw#R!(cUO1P-Gn48Mv?%5BtRv#FCipk@2uTPx|2=-wW_N+WFa9dVGTiL
zMr2VNxq=W-TtNgC67?yfh#UIQSwNH#X5P4sGB`TVZ9v}XBn(N^`8bb1-ajw@<n#I6
zy5Do|Ik)aPw|e!FS*wrCI<WqZc$y?dT4G7uPx*Z~!;vg5$_k6Cf-LfsDEoap5jb!M
z!n9xHp49cdHZ$P7vlE+o@=_x;>!K2P>&~4RlXl{K!Q=}c2HNlH#hAaiw-rX7ITg4r
zP=gtZ@CDe32R{j{2$W#aZ`3WD{t2EFcqf1`_M1a%i#Gf)C9rsNCq_K{)uEgi4-xq5
z=AD=!=ZSkwH_d!EaQEH4*toT?y&8L4_lm&mEj8GjsQQDRTjZ3$jx8nF3~WMu>bq0$
zz?WMPrdw2XNw5C#?!cX>6T@#Fb2_7c<;8&go}E~l`b>{|V|{d>@19;P?ejAYo}z6R
z1M|1mU}@7s6*rqbEpT{i35MnWnh8Z-M<8Mw!uUSt{*udS^8!z9>%>yVoJr68bM3kS
zaqmtn<JVu@>izOB6@gRt_F_|OCwR#_JdVK1?KN2BgI~<P{u5Vu;OzDi?7GyM`{Mgg
zz7Sw`ph(NEN#kti5}#fXV$pas^yC{uLa>0eKR%%E&BM~Seq34gK=ucLvfiE8q?@8A
zCdEy=5csyY7mN6n=bh-P@WVIv?)+8nPHlhhPTd}FH^It0!O$cwhynrG5*b`#{Uk2>
z2#yd*m7r8v?pWH?D7DtAGn>>U&8@Yq&2n?QQ0^&a$<iFMw5;4wQ_Po(96?moxbnOu
znMJt;)l~vtT}tMZ7E>h!HG+^^<0`DF$ScSysi-c>DzDC_2-p}z35AnF9Lc6KOe#Sp
z6CBHM^w<tf&(u)6#zSj*te6HfV_2r%&-zqGR4JU5Wgchz6oJbs?Z*|Fm0@cMEJsPG
z%&w29$#e1LIXo#k%ZX|?QI?gLBNh{tf}3PY1cDUuO85dghbqZ;la8{SEDz}_<2-p)
zyn|z0c|vX;Kj_&g#~6|pDb(|wE+U?VKS^N-To7m)XM6&S%OpwQ5~nhLmF6ggkkHfL
z>G<xe>!JyY<AUpoH^aIn_ZX7Q8Vvq2^|Y+eB(D&-%1|n-%dg@R1^JK#!Y?p>S|SO#
zxud<UPEjHB@st!#Qt>P!)i0y+YxAnJ%7yamVzxD#D(j#d`~`ehX$8HIaJEz_Rb#uB
zxm;yL{^BCGroEA=YHFrX%SGL|0l(-}R!C;S62%Aoio?LKiNpM2c}n6H9#$!fGR{yO
zyvD(|+Q;*%%2KSPN>ZyTDQfGGPmA(WI>{!Xu-wHgt!>S0tLL4`9nRLa(&i;g9kt2j
z&7PK;CFLH$PgvYtnKhm+zC61f#r{Kg-Bo``fuLw&AWo*CI6?jg;>3~!p+GQjzn})=
zB>Higqd45pN_;T+vP2UdO`U3Moxe_1hJ0#tEzQaCE^A<l<z#o)Qns+&M;1ESyrQO#
zik!xJADioHDrg-mwX|l-^}g)VEDuqVlik$J4#deG%pb&wPZmg#B3X3dAG)Y|zpOAm
zMS|@|k+2a3O2B1V6mibaGBU{s63@!5Qj^lWsH5$l!Kv+7R_tp~*n%Q=aVsUWix;|^
znw_0u1KqH=vofW<tC+6w6tppX&XTM}b>&pCgW;E=ri;3Xj(<AAa2zX&!Qf0cKybui
zF9%Pv5=C+YTZh9LiD7V2;Rf~;@1qm}Qc0l`kop=|g<EuUR8hGrzmO(8*>0xVU6@4{
z=2Uvht7-~ME4+nOnT*TBi=HaCkX4*5x{Gov3W|i1DlxaD(mU$q7>EwT5iCoij7z$x
zI+Y_RpP>41TBhLir{Un22?cy&DN2z^n)0bWf2-QIa9D8iI=!AQ(%D>>+fdxz=;57p
ze2b%_Gn4UlHFsramNxi{T5_977eRNlI9yIA1gt0-J%33Te>FoSK~kK6z5pNmtO!|A
z2omT03TzdY;BlGeC7h)c7+^`7V2Q;Ya$}uuXqza@a{W~`ZQjKNUPnu%+E(cI=XG|}
zxk-;hY~<^T^YfPR<zDX+GH+28?WH=iOIuSY)bN9D>Xog_CG!l!3j^7S(?NFdVcFq$
zfnzv^#~~3S&ZyvuM5qc5`#>RSmG?{Vrm3CnlH91yRN4oQSErM&F7xJ<<yU6rQ@O<z
zZi?r!$t=n#R8_jOd1t=6v?wcwpowasv?!a%t>z0USFSgwtfH`zXUfYRKTZ_*NDDk6
zpt+ZIQ42+yB4io%h$1WgV4@V*(0&Hz89zai3?WFQaK)*iFlClf&ZP~NRctb!?_gx5
zsJ7T!oSa$P&et!_&i5{Kw3jZUJPlP#xoV++EUc^ddFwmS^Otqxl{|6DG{+Gn*MI4<
zZcLp1e*5Hp_?KwwzSe3ImK)Mz)&8$fe$&v=j}9m67v^5;J7Som&(H?<=R|Z%ihk<c
zUzp3EO=bVbDf%zcuQ^7`R~G3_8m+d+8Xl_u_e0e^vPuF?H$*e&uQg%-(+<IO`kx6`
z(&)Ya^YQwjS#OWiKVxrsDm?BTTX((z)~wZE>mCak;{3az>mJ=;=!Zbp8Z}*BaAD@3
z3)q!-O}=ti{A|GbcIhvcCX_K%m7W47-&N!$xk}RIt>H=5BWBYzxjB{Yik#BIn$pr5
zH+tcMej0LI&^r?|vrEbf%9-js((7`OZmOK&p-~`<@(Cupg00E*mK3`(%Zk1IFy)Js
zG66a@ALnCmRpOvFsJx7eewxJ<&d<R4&8mb#w5p4nQzd0lT~oW-*52OSpf(LZ$%Isz
zNM)c=gi6SfeQ_aY?D4+@yOs#)BMi^o5Ks6iQWQuy@knSMp`nBZ2kJSlD6~YW0;528
zgU)R<WMV^gS3pUd4AZMgZlDzq;&1}~rmP(96nHeyO$ma?FkDZF)~xYp%{{=NBw1xh
zNx|WGg&L?pIg)*xANSKFblS2X>Na|0lcCUh6#;dk|MDio?^0%6)q+^vBz>m&r>zc5
zdBJedUi?c|r%TX#%)h{~{>vu}%k15}DEkOnz-j1ggY7AFrhXdYMXGQ&U=`@TM7=IW
zhiSAPGaN1!SeC+4f`t1h68c7wWpRP>L)qgQnUY0?P#IOHPtguc&}y_6A5RK?=<aY?
zf@Hu-p;~i(9%lLc1Z$*4o|Aq3Kb$r6ZjV<<hGiHLnn00(%C3kiE|TDvUxHW)0w?-t
z5*=A#OoNK$;~5Dqpo-)R_5eO8Z$2Kb3nb?gSyhHyODJxo5sw!MT9VWtNs+)|71AR^
z8;=W|kCXirudpP8T2>m_cpv4Hd8pOU@9_-8f)H@YN76V)s!Xum;bnqDZ>%(?#;ZO>
zp+sOpxq~YL$w0-Yq1O5&iSSDz?}JsL#8t*A@hT_z1s3{!zaS{V{Rn<=aLa+HGqkF5
zEYHiRX_YZ;KB!=aA#q4DFj+bnCne|tAwgp(Q3Av+#H2A#)@nSpa9xv{+B>8s9VE#w
zDUb$EfTb}q&*6fk%DBS7>j<|k8hTk|>^9Dg=lp((l}Q-qWEQ+uc!;hKIs|CiWS(YN
z61vFxZt$8RNQtKja0A>9u8+bc0U8`u;Z#xbQKHHU=s>q|R=g;y(6`GR&H)d)Ww<d3
zG&CF>w9F(gDx&CT*RD49tv0O;wZV>uYao>jDW(V!-G002u6T(iC1~CRoRHx?f(sUd
z%L*h?WH^W>&nt>gLYeDL*N>j<++>PEtJj;D(F4EUXqt)6uQ$ycJ#b})0nOcD${9Th
z(-P6M8-8qPU?6OxDSGsb5xfvovC)(;`VpAx8LfS`(NsBF161B<ZTBWq+9<96{3g?Z
z((aLQ42#icYDb+M!@?Z;uZJ!Fdf5CE!`;K^*iVcP8#av?X5z?^t{X=T)1!wCKRv8o
z*+C&A#;MWM_5XT!<ca&E*?s3s57_6Cp2Dn3fhx$O%N*G?WyN`wVo|jx+X)rXlauKn
zE2tuZrLsynr_1BX&Ud-09b`^cW_eDQqqMTZ<;H%T7$R8|c#$UiFJCad7(4e`E%c4S
zwx#@{*Qoz+2IguC)*{Fd)L05`PAK|u?A$Pk5E+E+$IcCHJa~%a1fK5ydOtSiNb*{Z
z*=0UvskEFAT@_{w+iaa?Yq7m)pJBfX+h-z;j~G)7y9@^XGx|u~JndTO&dRm@^)2ST
zhR`)#J>jPD2?@Ge@rUD<n6>DQEX>wl*kyihyz98e{<-}<`!V~I_MP_I?45R>y~Hlq
zr`gBaF55o0owmJf+i%-uyVcfeTVN}+F}4^RX8oJ>J?qQX$F28R*H{~^9&5Ta!8#`F
zyRbin9S6G|Vck&)Ce0+RHbm1X1zQbLSRU&B^HW(-(?`nu(m{FJNSRwQD94YK*^WVZ
z>PVSs8<eM9mAivB>6Sr#@>M-36U_s1WSmJ8ebqy8(V!YTLKPZ<YG!222$ioJR40v4
zx!OVXdQiKsf?0Jyj*1@X1?3x*uNx_o(ja8w2sCrkpgLiMO5ZT3jvt{?^Fi%?Dl>eX
zNfQ-xV{vP<Qf>(@BO=PAnRwN+K`}gX*mI@kf#)JdJU1kc9U&H);K{iW;UmOc<A6BE
z)IHqJTtiR|w-4Fr<lswW$be+KBC*hO4ocQ55|d2OgHqTPiB2Yx2c^&<DY&+YK`A4`
zGTf8y_CYn$JQSO@cB!_msdmD^v*FkkB9%;x8<b4Lh-36YVx)28T>F3;ZWuP#Iw(P3
z@XttNy5QW|5pWqC7V1HD4BY9WwC2H-4454Y7qf{$W$-l_(E4_nk2(vSgvVJ?l*<-7
zJ(Zk?%9^P)YP2kF*44N97B;uG&b{*aAlcWvsFL=Sm(&z_iGpeu=Me;pMx)W$PigF@
z?4QH?c-)>8smHaWk-7*yZXP}Lpe29u|NnUKfAix3>P@lKV3zE}OST(~$y*4^msr&8
zJ-5Up9V$oP6P8ZQe)gftapLu3Q5R|1i6wme?l=4gDMwH&W$DHEosKCbAN;Td9isp;
zYs0FD^cxSXMhUc~1dF(>d`)!q=}*xL+Jdl251R6mR_}ftMKi!84ZHJm?psBmtqd@)
z>*OM)g??Cq=CZ&HpYh)D3E1&Z(MPOho@uc!eT@F_ROH|QX<BYRX<2GLhIVj(O#iee
z)f;i}FDQ)%q;T_@BxkJFh@RyE8M9^LZ)W@^emar_K*qU>GVwWF0y2r<re<@?+nN)<
zxee8cz|=l==tiBFj6sJ*V2+>mU!TM+%m_ydQh^y;n_-^$_1{jT_fvrx^4+05#&5`9
zp*6FCsckN3)ahu164QWbJs7(sBIU*gv@Q*p<m#@yxg{OXp|fegoRjldoadEw0kmok
zFp0K>zs>&LvA?5>bAXxgr>CE)Uc4PgyVHT0bUxjb`s6w@D#!rljOF`I&tCeKjP_;#
zb4>5uTgT@#nEU2hzPt%b`-<-NxR)29?{@(+V%7PB(M83tp|1OYIV<{>DRYj*$|&}J
zV5a?c7wbvPJAjVe4@^_~*V8TWU$Q7+H!vri(Y=!N){f<9#cuF($`jw*M%;DeD-`_z
zFyrH%H&L5spF~?90Os_1!oj}!2NKPF5SW(TkHpzNj0vER9t38>=?~ZNZ#>+G91j6g
zv**z5n!o;#h;}>#%!!ZsE2f@Ld=I5P49xIwb<y|_6YfFJJ`7ATWAg>qw6}YZ^awDS
z(qj)4%#M5#nH~jZlyi+?((n6>sP0i<PPo1H`YH2|Za{}01!m$l_HE(RpD&;Vdw^+r
z@t9ya6B&)(-(%T1ZkA%GN}qOLM9<Wm(1D-QdS(WHOPl;)JUY17(uj@U+_7!4KGcE|
z9|J4Gx4$<zUpjgct$Pft*w!ps9O<IoMrR)bE7M<@@)_qq1!&biurkGe*J&b&zKAaF
z1J5VS+OuWyyHgBk_v7F>e<XHMY1p^NQNey-YQNKO*Bv!j(cb;Q#OyIdsAEBA-vNtp
zjCKJs5tixW3k{zbvO?|+nWSH(Ptv`ro2*@?xu{uiXXZpS?+BPqZCdhO*<0@?q3@4?
z>8R$-D`RxqA4gp;f@yP4@d0aT?;R-iB`}@*M2a`_hK}Xv*h^rV`S$ipg^5T1ff9~d
zN=+}!icL&Ebpu*)6p(`3=qx>h{T@ZX99+%5%@>LFJ8nW-Uk)PsR;;ydjr<QZ_gF9w
z`|o!YytMxZ^wBX(O=R@F7eAT)aPkLbjf1~TYV>*xJ#oqskGfv5Oft?unX`9oL_1yq
z6Vac}iAjj?+=0?w1ruzZV}0S>_q>dreHBckpEEq`H8#~F=`}Dh`x&CYV*4N7LZ;Wj
zYgS1-TKGa^4yt<{0%jGXYRBF@>1%ZObzn|&^XW|1Q_rIXZvZoXO{|r^{?4=L{WpMV
zoad~<KAL?FtvL?N$wKPh<etx-Ly2z!Q~dj#2g;^W^U=CDfr&qr(U$T3my6KZH-Tv_
z{rH=)_wAjIR=owxgbnr=`5VXC(8afaIkmk0zsWXv9oqe$z%)Gj&exb>y%!an0H%HW
zs-CF#$M>MUCx96vpBOu1oo8a-+m<?GJlw%q)i1+$Fj*plpGY`d6yYlcC-NjOQ3OL$
za{q>tmObWtv#~Ff3Y`~vOX$APlcvW)FNY<Dm4z+CTw%Mz-VFP1Ym7DDgjp9`w_0Da
zerAib&9V87V{MykPa9|3{%E(@Y5U{$`Sx4;SDmwLvINfGm(+jbXO@*A$o82f5uH!7
z#KR@SvOO6SiypGOxI+%KJ0`Rjo21<3Pnz!EhzepuA>=g&A1Zv3h(>#3LrXAr{(4{G
I!I#(l4^|11ng9R*


From ec01373240af0bf9ba8f14c3a0c2271f40b5db82 Mon Sep 17 00:00:00 2001
From: Jake Moshenko <jake.moshenko@coreos.com>
Date: Wed, 18 Feb 2015 17:06:41 -0500
Subject: [PATCH 23/24] Rename the config variable for temp tag expiration per
 the pull request feedback.

---
 config.py             | 2 +-
 endpoints/registry.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/config.py b/config.py
index bedddcda7..4bb44fca2 100644
--- a/config.py
+++ b/config.py
@@ -197,4 +197,4 @@ class DefaultConfig(object):
   SYSTEM_SERVICE_BLACKLIST = []
 
   # Temporary tag expiration in seconds, this may actually be longer based on GC policy
-  PUSH_TEMP_TAG_EXPIRATION_S = 60 * 60
+  PUSH_TEMP_TAG_EXPIRATION_SEC = 60 * 60
diff --git a/endpoints/registry.py b/endpoints/registry.py
index 8222212a7..c901eed5b 100644
--- a/endpoints/registry.py
+++ b/endpoints/registry.py
@@ -469,7 +469,7 @@ def put_image_json(namespace, repository, image_id):
 
   # Create a temporary tag to prevent this image from getting garbage collected while the push
   # is in progress.
-  model.create_temporary_hidden_tag(repo, repo_image, app.config['PUSH_TEMP_TAG_EXPIRATION_S'])
+  model.create_temporary_hidden_tag(repo, repo_image, app.config['PUSH_TEMP_TAG_EXPIRATION_SEC'])
 
   uuid = repo_image.storage.uuid
 

From 9ab3554226186d35f71e79b1d0b1f88487e62417 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Wed, 18 Feb 2015 17:11:45 -0500
Subject: [PATCH 24/24] buildreporter: does not execute in a coroutine!

---
 buildman/jobutil/buildreporter.py | 6 ++++--
 util/cloudwatch.py                | 7 ++-----
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/buildman/jobutil/buildreporter.py b/buildman/jobutil/buildreporter.py
index 992b7a06e..16dd0ca5b 100644
--- a/buildman/jobutil/buildreporter.py
+++ b/buildman/jobutil/buildreporter.py
@@ -11,7 +11,6 @@ class BuildReporter(object):
   def report_completion_status(self, status):
     """
     Method to invoke the recording of build's completion status to a metric service.
-    This must _not_ block because it is assumed to run in an event loop.
     """
     raise NotImplementedError
 
@@ -35,6 +34,9 @@ class CloudWatchBuildReporter(BuildReporter):
     self._failed_name = failed_name
     self._incompleted_name = incompleted_name
 
+  def _send_to_queue(self, *args, **kwargs):
+    self._queue.put((args, kwargs))
+
   def report_completion_status(self, status):
     if status == BuildJobResult.COMPLETE:
       status_name = self._completed_name
@@ -45,7 +47,7 @@ class CloudWatchBuildReporter(BuildReporter):
     else:
       return
 
-    yield From(self._queue.put(self._namespace_name, status_name, 1, unit='Count'))
+    self._send_to_queue(self._namespace_name, status_name, 1, unit='Count')
 
 
 class BuildMetrics(object):
diff --git a/util/cloudwatch.py b/util/cloudwatch.py
index eca563164..b75dadf31 100644
--- a/util/cloudwatch.py
+++ b/util/cloudwatch.py
@@ -1,6 +1,5 @@
 import logging
 import boto
-import thread
 
 from Queue import Queue
 from threading import Thread
@@ -12,10 +11,8 @@ def get_queue(app):
   """
   Returns a queue to the CloudWatchSender. If a queue/sender do not exist, creates them.
   """
-  access_key = app.config.get('CLOUDWATCH_AWS_ACCESS_KEY')
-  secret_key = app.config.get('CLOUDWATCH_AWS_SECRET_KEY')
-  if None in (access_key, secret_key):
-    raise TypeError
+  access_key = app.config['CLOUDWATCH_AWS_ACCESS_KEY']
+  secret_key = app.config['CLOUDWATCH_AWS_SECRET_KEY']
 
   queue = Queue()
   sender = CloudWatchSender(queue, access_key, secret_key)