Refactor queue locking to not use select for update

The test suggests this works.

fixes #622

data/queue.py

@@ -50,11 +50,12 @@ class WorkQueue(object):
                    QueueItem.queue_name ** name_match_query))
 
   def _available_jobs(self, now, name_match_query):
-    return (QueueItem
-            .select()
-            .where(QueueItem.queue_name ** name_match_query, QueueItem.available_after <= now,
+    return self._available_jobs_where(QueueItem.select(), now, name_match_query)
+
+  def _available_jobs_where(self, query, now, name_match_query):
+    return query.where(QueueItem.queue_name ** name_match_query, QueueItem.available_after <= now,
                    ((QueueItem.available == True) | (QueueItem.processing_expires <= now)),
-                   QueueItem.retries_remaining > 0))
+                   QueueItem.retries_remaining > 0)
 
   def _available_jobs_not_running(self, now, name_match_query, running_query):
     return (self
@@ -145,25 +146,30 @@ class WorkQueue(object):
 
     item = None
     try:
-      db_item_candidate = avail.order_by(QueueItem.id).get()
-
-      with self._transaction_factory(db):
-        still_available_query = (db_for_update(self
-                                               ._available_jobs(now, name_match_query)
-                                               .where(QueueItem.id == db_item_candidate.id)))
-
-        db_item = still_available_query.get()
-        db_item.available = False
-        db_item.processing_expires = now + timedelta(seconds=processing_time)
-        db_item.retries_remaining -= 1
-        db_item.save()
+      # The previous solution to this used a select for update in a
+      # transaction to prevent multiple instances from processing the
+      # same queue item. This suffered performance problems. This solution
+      # instead has instances attempt to update the potential queue item to be
+      # unavailable. However, since their update clause is restricted to items
+      # that are available=False, only one instance's update will succeed, and
+      # it will have a changed row count of 1. Instances that have 0 changed
+      # rows know that another instance is already handling that item.
+      db_item = avail.order_by(QueueItem.id).get()
+      changed_query = (QueueItem.update(
+          available=False,
+          processing_expires=now + timedelta(seconds=processing_time),
+          retries_remaining=QueueItem.retries_remaining-1,
+        )
+        .where(QueueItem.id == db_item.id))
 
+      changed_query = self._available_jobs_where(changed_query, now, name_match_query)
+      changed = changed_query.execute()
+      if changed == 1:
         item = AttrDict({
           'id': db_item.id,
           'body': db_item.body,
-          'retries_remaining': db_item.retries_remaining
+          'retries_remaining': db_item.retries_remaining - 1,
         })
-
         self._currently_processing = True
     except QueueItem.DoesNotExist:
       self._currently_processing = False