Finish the implementation of local userfiles. Strip charsets from mimetypes in the build worker. Add canonical name ordering to the build queue. Port all queues to the canonical naming version.

data/queue.py

@@ -8,17 +8,26 @@ transaction_factory = app.config['DB_TRANSACTION_FACTORY']
 
 
 class WorkQueue(object):
-  def __init__(self, queue_name):
+  def __init__(self, queue_name, canonical_name_match_list=None):
     self.queue_name = queue_name
 
-  def put(self, message, available_after=0, retries_remaining=5):
+    if canonical_name_match_list is None:
+      self.canonical_name_match_list = []
+    else:
+      self.canonical_name_match_list = canonical_name_match_list
+
+  @staticmethod
+  def _canonical_name(name_list):
+    return '/'.join(name_list) + '/'
+
+  def put(self, canonical_name_list, message, available_after=0, retries_remaining=5):
     """
     Put an item, if it shouldn't be processed for some number of seconds,
     specify that amount as available_after.
     """
 
     params = {
-      'queue_name': self.queue_name,
+      'queue_name': self._canonical_name([self.queue_name] + canonical_name_list),
       'body': message,
       'retries_remaining': retries_remaining,
     }
@@ -35,16 +44,25 @@ class WorkQueue(object):
     minutes.
     """
     now = datetime.now()
-    available_or_expired = ((QueueItem.available == True) |
-                            (QueueItem.processing_expires <= now))
+
+    name_match_query = '%s%%' % self._canonical_name([self.queue_name] +
+                                                     self.canonical_name_match_list)
 
     with transaction_factory(db):
-      avail = QueueItem.select().where(QueueItem.queue_name == self.queue_name,
-                                       QueueItem.available_after <= now,
-                                       available_or_expired,
-                                       QueueItem.retries_remaining > 0)
+      running = (QueueItem
+        .select(QueueItem.queue_name)
+        .where(QueueItem.available == False,
+               QueueItem.processing_expires > now,
+               QueueItem.queue_name ** name_match_query))
 
-      found = list(avail.limit(1).order_by(QueueItem.available_after))
+      avail = QueueItem.select().where(QueueItem.queue_name ** name_match_query,
+                                       QueueItem.available_after <= now,
+                                       ((QueueItem.available == True) |
+                                        (QueueItem.processing_expires <= now)),
+                                       QueueItem.retries_remaining > 0,
+                                       ~(QueueItem.queue_name << running))
+
+      found = list(avail.limit(1).order_by(QueueItem.id))
 
       if found:
         item = found[0]
@@ -57,16 +75,24 @@ class WorkQueue(object):
 
       return None
 
-  def complete(self, completed_item):
+  @staticmethod
+  def complete(completed_item):
     completed_item.delete_instance()
 
-  def incomplete(self, incomplete_item, retry_after=300):
+  @staticmethod
+  def incomplete(incomplete_item, retry_after=300):
     retry_date = datetime.now() + timedelta(seconds=retry_after)
     incomplete_item.available_after = retry_date
     incomplete_item.available = True
     incomplete_item.save()
 
+  @staticmethod
+  def extend_processing(queue_item, seconds_from_now):
+    new_expiration = datetime.now() + timedelta(seconds=seconds_from_now)
+    queue_item.processing_expires = new_expiration
+    queue_item.save()
+
 
 image_diff_queue = WorkQueue('imagediff')
-dockerfile_build_queue = WorkQueue('dockerfilebuild3')
+dockerfile_build_queue = WorkQueue('dockerfilebuild')
 webhook_queue = WorkQueue('webhook')