Handle internal errors in a better fashion: If a build would be marked as internal error, only do so if there are retries remaining. Otherwise, we mark it as failed (since it won't be rebuilt anyway)

data/database.py

@@ -476,6 +476,15 @@ class BUILD_PHASE(object):
   COMPLETE = 'complete'
 
 
+class QueueItem(BaseModel):
+  queue_name = CharField(index=True, max_length=1024)
+  body = TextField()
+  available_after = DateTimeField(default=datetime.utcnow, index=True)
+  available = BooleanField(default=True, index=True)
+  processing_expires = DateTimeField(null=True, index=True)
+  retries_remaining = IntegerField(default=5)
+
+
 class RepositoryBuild(BaseModel):
   uuid = CharField(default=uuid_generator, index=True)
   repository = ForeignKeyField(Repository, index=True)
@@ -488,15 +497,7 @@ class RepositoryBuild(BaseModel):
   trigger = ForeignKeyField(RepositoryBuildTrigger, null=True, index=True)
   pull_robot = QuayUserField(null=True, related_name='buildpullrobot')
   logs_archived = BooleanField(default=False)
-
-
-class QueueItem(BaseModel):
-  queue_name = CharField(index=True, max_length=1024)
-  body = TextField()
-  available_after = DateTimeField(default=datetime.utcnow, index=True)
-  available = BooleanField(default=True, index=True)
-  processing_expires = DateTimeField(null=True, index=True)
-  retries_remaining = IntegerField(default=5)
+  queue_item = ForeignKeyField(QueueItem, null=True, index=True)
 
 
 class LogEntryKind(BaseModel):

data/migrations/versions/14fe12ade3df_add_build_queue_item_reference_to_the_.py

@@ -0,0 +1,30 @@
+"""Add build queue item reference to the repositorybuild table
+
+Revision ID: 14fe12ade3df
+Revises: 5ad999136045
+Create Date: 2015-02-12 16:11:57.814645
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '14fe12ade3df'
+down_revision = '5ad999136045'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('repositorybuild', sa.Column('queue_item_id', sa.Integer(), nullable=True))
+    op.create_index('repositorybuild_queue_item_id', 'repositorybuild', ['queue_item_id'], unique=False)
+    op.create_foreign_key(op.f('fk_repositorybuild_queue_item_id_queueitem'), 'repositorybuild', 'queueitem', ['queue_item_id'], ['id'])
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(op.f('fk_repositorybuild_queue_item_id_queueitem'), 'repositorybuild', type_='foreignkey')
+    op.drop_index('repositorybuild_queue_item_id', table_name='repositorybuild')
+    op.drop_column('repositorybuild', 'queue_item_id')
+    ### end Alembic commands ###

data/queue.py

@@ -60,8 +60,8 @@ class WorkQueue(object):
       running_query = self._running_jobs(now, name_match_query)
       running_count = running_query.distinct().count()
 
-      avialable_query = self._available_jobs_not_running(now, name_match_query, running_query)
-      available_count = avialable_query.select(QueueItem.queue_name).distinct().count()
+      available_query = self._available_jobs_not_running(now, name_match_query, running_query)
+      available_count = available_query.select(QueueItem.queue_name).distinct().count()
 
     self._reporter(self._currently_processing, running_count, running_count + available_count)
 
@@ -81,7 +81,7 @@ class WorkQueue(object):
     params['available_after'] = available_date
 
     with self._transaction_factory(db):
-      QueueItem.create(**params)
+      return QueueItem.create(**params)
 
   def get(self, processing_time=300):
     """
@@ -114,6 +114,7 @@ class WorkQueue(object):
         item = AttrDict({
           'id': db_item.id,
           'body': db_item.body,
+          'retries_remaining': db_item.retries_remaining
         })
 
         self._currently_processing = True
@@ -141,6 +142,7 @@ class WorkQueue(object):
 
       incomplete_item_obj.save()
       self._currently_processing = False
+      return incomplete_item_obj.retries_remaining > 0
 
   def extend_processing(self, item, seconds_from_now, minimum_extension=MINIMUM_EXTENSION):
     with self._transaction_factory(db):