Switch a few of the buildman methods to coroutines in order to support network calls in methods. Add a test for the ephemeral build manager.

buildman/manager/executor.py

@@ -7,6 +7,10 @@ import requests
 import cachetools
 
 from jinja2 import FileSystemLoader, Environment
+from trollius import coroutine, From, Return, get_event_loop
+from functools import partial
+
+from buildman.asyncutil import AsyncWrapper
 
 
 logger = logging.getLogger(__name__)
@@ -32,12 +36,14 @@ class BuilderExecutor(object):
   """ Interface which can be plugged into the EphemeralNodeManager to provide a strategy for
       starting and stopping builders.
   """
+  @coroutine
   def start_builder(self, realm, token):
     """ Create a builder with the specified config. Returns a unique id which can be used to manage
         the builder.
     """
     raise NotImplementedError
 
+  @coroutine
   def stop_builder(self, builder_id):
     """ Stop a builder which is currently running.
     """
@@ -74,14 +80,18 @@ class EC2Executor(BuilderExecutor):
   """
   COREOS_STACK_URL = 'http://%s.release.core-os.net/amd64-usr/current/coreos_production_ami_hvm.txt'
 
+  def __init__(self, *args, **kwargs):
+    self._loop = get_event_loop()
+    super(EC2Executor, self).__init__(*args, **kwargs)
+
   def _get_conn(self):
     """ Creates an ec2 connection which can be used to manage instances.
     """
-    return boto.ec2.connect_to_region(
+    return AsyncWrapper(boto.ec2.connect_to_region(
         self.executor_config['EC2_REGION'],
         aws_access_key_id=self.executor_config['AWS_ACCESS_KEY'],
         aws_secret_access_key=self.executor_config['AWS_SECRET_KEY'],
-    )
+    ))
 
   @classmethod
   @cachetools.ttl_cache(ttl=ONE_HOUR)
@@ -92,25 +102,24 @@ class EC2Executor(BuilderExecutor):
     stack_amis = dict([stack.split('=') for stack in stack_list_string.split('|')])
     return stack_amis[ec2_region]
 
+  @coroutine
   def start_builder(self, realm, token):
     region = self.executor_config['EC2_REGION']
     channel = self.executor_config.get('COREOS_CHANNEL', 'stable')
-    coreos_ami = self._get_coreos_ami(region, channel)
+    get_ami_callable = partial(self._get_coreos_ami, region, channel)
+    coreos_ami = yield From(self._loop.run_in_executor(None, get_ami_callable))
     user_data = self.generate_cloud_config(realm, token, channel, self.manager_public_ip)
 
     logger.debug('Generated cloud config: %s', user_data)
 
     ec2_conn = self._get_conn()
-    # class FakeReservation(object):
-    #   def __init__(self):
-    #     self.instances = None
-    # reservation = FakeReservation()
-    reservation = ec2_conn.run_instances(
+    reservation = yield ec2_conn.run_instances(
         coreos_ami,
         instance_type=self.executor_config['EC2_INSTANCE_TYPE'],
         security_groups=self.executor_config['EC2_SECURITY_GROUP_IDS'],
         key_name=self.executor_config.get('EC2_KEY_NAME', None),
         user_data=user_data,
+        instance_initiated_shutdown_behavior='terminate',
     )
 
     if not reservation.instances:
@@ -124,12 +133,13 @@ class EC2Executor(BuilderExecutor):
         'Realm': realm,
         'Token': token,
     })
-    return launched.id
+    raise Return(launched.id)
 
+  @coroutine
   def stop_builder(self, builder_id):
     ec2_conn = self._get_conn()
-    stopped_instance_ids = [si.id for si in ec2_conn.stop_instances([builder_id], force=True)]
-    if builder_id not in stopped_instance_ids:
+    stopped_instances = yield ec2_conn.stop_instances([builder_id], force=True)
+    if builder_id not in [si.id for si in stopped_instances]:
       raise ExecutorException('Unable to stop instance: %s' % builder_id)
 
 class PopenExecutor(BuilderExecutor):
@@ -142,6 +152,7 @@ class PopenExecutor(BuilderExecutor):
 
   """ Executor which uses Popen to fork a quay-builder process.
   """
+  @coroutine
   def start_builder(self, realm, token):
     # Now start a machine for this job, adding the machine id to the etcd information
     logger.debug('Forking process for build')
@@ -162,9 +173,9 @@ class PopenExecutor(BuilderExecutor):
     builder_id = str(uuid.uuid4())
     self._jobs[builder_id] = (spawned, logpipe)
     logger.debug('Builder spawned with id: %s', builder_id)
-    return builder_id
-
+    raise Return(builder_id)
 
+  @coroutine
   def stop_builder(self, builder_id):
     if builder_id not in self._jobs:
       raise ExecutorException('Builder id not being tracked by executor.')