Merge remote-tracking branch 'upstream/phase4-11-07-2015' into python-registry-v2

data/database.py

@@ -472,9 +472,6 @@ class RepositoryBuildTrigger(BaseModel):
   pull_robot = QuayUserField(allows_robots=True, null=True, related_name='triggerpullrobot',
                              robot_null_delete=True)
 
-  # TODO(jschorr): Remove this column once we verify the backfill has succeeded.
-  used_legacy_github = BooleanField(null=True, default=False)
-
 
 class EmailConfirmation(BaseModel):
   code = CharField(default=random_string_generator(), unique=True, index=True)
@@ -487,11 +484,11 @@ class EmailConfirmation(BaseModel):
 
 class ImageStorage(BaseModel):
   uuid = CharField(default=uuid_generator, index=True, unique=True)
-  checksum = CharField(null=True)
   image_size = BigIntegerField(null=True)
   uncompressed_size = BigIntegerField(null=True)
   uploading = BooleanField(default=True, null=True)
   cas_path = BooleanField(default=True)
+  content_checksum = CharField(null=True, index=True)
 
 
 class ImageStorageTransformation(BaseModel):
@@ -573,6 +570,11 @@ class Image(BaseModel):
   command = TextField(null=True)
   aggregate_size = BigIntegerField(null=True)
   v1_json_metadata = TextField(null=True)
+  v1_checksum = CharField(null=True)
+
+  security_indexed = BooleanField(default=False)
+  security_indexed_engine = IntegerField(default=-1)
+  parent = ForeignKeyField('self', index=True, null=True, related_name='children')
 
   class Meta:
     database = db
@@ -580,6 +582,8 @@ class Image(BaseModel):
     indexes = (
       # we don't really want duplicates
       (('repository', 'docker_image_id'), True),
+
+      (('security_indexed_engine', 'security_indexed'), False),
     )
 
 
@@ -746,6 +750,7 @@ class RepositoryNotification(BaseModel):
   method = ForeignKeyField(ExternalNotificationMethod)
   title = CharField(null=True)
   config_json = TextField()
+  event_config_json = TextField(default='{}')
 
 
 class RepositoryAuthorizedEmail(BaseModel):

data/migrations/versions/22af01f81722_backfill_parent_id_and_v1_checksums.py

@@ -0,0 +1,21 @@
+"""Backfill parent id and v1 checksums
+
+Revision ID: 22af01f81722
+Revises: 2827d36939e4
+Create Date: 2015-11-05 16:24:43.679323
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '22af01f81722'
+down_revision = '2827d36939e4'
+
+from util.migrate.backfill_v1_checksums import backfill_checksums
+from util.migrate.backfill_parent_id import backfill_parent_id
+
+def upgrade(tables):
+    backfill_parent_id()
+    backfill_checksums()
+
+def downgrade(tables):
+    pass

data/migrations/versions/2827d36939e4_separate_v1_and_v2_checksums.py

@@ -0,0 +1,30 @@
+"""Separate v1 and v2 checksums.
+
+Revision ID: 2827d36939e4
+Revises: 73669db7e12
+Create Date: 2015-11-04 16:29:48.905775
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '2827d36939e4'
+down_revision = '5cdc2d819c5'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('image', sa.Column('v1_checksum', sa.String(length=255), nullable=True))
+    op.add_column('imagestorage', sa.Column('content_checksum', sa.String(length=255), nullable=True))
+    op.create_index('imagestorage_content_checksum', 'imagestorage', ['content_checksum'], unique=False)
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('imagestorage_content_checksum', table_name='imagestorage')
+    op.drop_column('imagestorage', 'content_checksum')
+    op.drop_column('image', 'v1_checksum')
+    ### end Alembic commands ###

data/migrations/versions/35f538da62_switch_manifest_text_to_a_longtext.py

@@ -15,28 +15,13 @@ import sqlalchemy as sa
 
 from sqlalchemy.types import TypeDecorator, Text
 from sqlalchemy.dialects.mysql import LONGTEXT
-import uuid
 
-class EngineLongText(TypeDecorator):
-    """Platform-independent LongText type.
-
-    Uses MySQL's LONGTEXT type, otherwise uses
-    Text, because other engines are not as limited
-    as MySQL.
-
-    """
-    impl = Text
-
-    def load_dialect_impl(self, dialect):
-        if dialect.name == 'mysql':
-            return dialect.type_descriptor(LONGTEXT())
-        else:
-            return dialect.type_descriptor(Text())
+from util.migrate import UTF8LongText
 
 def upgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###
     op.drop_column(u'tagmanifest', 'json_data')
-    op.add_column(u'tagmanifest', sa.Column('json_data', EngineLongText(), nullable=False))
+    op.add_column(u'tagmanifest', sa.Column('json_data', UTF8LongText(), nullable=False))
     ### end Alembic commands ###
 
 

data/migrations/versions/50925110da8c_add_event_specific_config.py

@@ -0,0 +1,27 @@
+"""Add event-specific config
+
+Revision ID: 50925110da8c
+Revises: 2fb9492c20cc
+Create Date: 2015-10-13 18:03:14.859839
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '50925110da8c'
+down_revision = '57dad559ff2d'
+
+from alembic import op
+import sqlalchemy as sa
+from util.migrate import UTF8LongText
+
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('repositorynotification', sa.Column('event_config_json', UTF8LongText, nullable=False))
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('repositorynotification', 'event_config_json')
+    ### end Alembic commands ###

data/migrations/versions/545794454f49_migrate_image_data_back_to_image_table.py

@@ -13,14 +13,15 @@ down_revision = '3a3bb77e17d5'
 from alembic import op
 import sqlalchemy as sa
 
+from util.migrate import UTF8LongText
 
 def upgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###
     op.add_column('image', sa.Column('aggregate_size', sa.BigInteger(), nullable=True))
     op.add_column('image', sa.Column('command', sa.Text(), nullable=True))
-    op.add_column('image', sa.Column('comment', sa.Text(), nullable=True))
+    op.add_column('image', sa.Column('comment', UTF8LongText(), nullable=True))
     op.add_column('image', sa.Column('created', sa.DateTime(), nullable=True))
-    op.add_column('image', sa.Column('v1_json_metadata', sa.Text(), nullable=True))
+    op.add_column('image', sa.Column('v1_json_metadata', UTF8LongText(), nullable=True))
     ### end Alembic commands ###
 
 

data/migrations/versions/57dad559ff2d_add_support_for_quay_s_security_indexer.py

@@ -0,0 +1,32 @@
+"""add support for quay's security indexer
+Revision ID: 57dad559ff2d
+Revises: 154f2befdfbe
+Create Date: 2015-07-13 16:51:41.669249
+"""
+
+# revision identifiers, used by Alembic.
+revision = '57dad559ff2d'
+down_revision = '73669db7e12'
+
+from alembic import op
+import sqlalchemy as sa
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('image', sa.Column('parent_id', sa.Integer(), nullable=True))
+    op.add_column('image', sa.Column('security_indexed', sa.Boolean(), nullable=False, default=False, server_default=sa.sql.expression.false()))
+    op.add_column('image', sa.Column('security_indexed_engine', sa.Integer(), nullable=False, default=-1, server_default="-1"))
+    op.create_index('image_parent_id', 'image', ['parent_id'], unique=False)
+    op.create_foreign_key(op.f('fk_image_parent_id_image'), 'image', 'image', ['parent_id'], ['id'])
+    ### end Alembic commands ###
+    op.create_index('image_security_indexed_engine_security_indexed', 'image', ['security_indexed_engine', 'security_indexed'])
+
+def downgrade(tables):
+   ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('image_security_indexed_engine_security_indexed', 'image')
+    op.drop_constraint(op.f('fk_image_parent_id_image'), 'image', type_='foreignkey')
+    op.drop_index('image_parent_id', table_name='image')
+    op.drop_column('image', 'security_indexed')
+    op.drop_column('image', 'security_indexed_engine')
+    op.drop_column('image', 'parent_id')
+    ### end Alembic commands ###

data/migrations/versions/5cdc2d819c5_add_vulnerability_found_event.py

@@ -0,0 +1,41 @@
+"""Add vulnerability_found event
+
+Revision ID: 5cdc2d819c5
+Revises: 50925110da8c
+Create Date: 2015-10-13 18:05:32.157858
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '5cdc2d819c5'
+down_revision = '50925110da8c'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+
+def upgrade(tables):
+    op.bulk_insert(tables.externalnotificationevent,
+    [
+        {'id':6, 'name':'vulnerability_found'},
+    ])
+
+    op.bulk_insert(tables.notificationkind,
+    [
+        {'id':11, 'name':'vulnerability_found'},
+    ])
+
+
+def downgrade(tables):
+    op.execute(
+        (tables.externalnotificationevent.delete()
+            .where(tables.externalnotificationevent.c.name == op.inline_literal('vulnerability_found')))
+
+    )
+
+    op.execute(
+        (tables.notificationkind.delete()
+            .where(tables.notificationkind.c.name == op.inline_literal('vulnerability_found')))
+
+    )

data/migrations/versions/73669db7e12_remove_legacy_github_column.py

@@ -0,0 +1,25 @@
+"""Remove legacy github column
+
+Revision ID: 73669db7e12
+Revises: 35f538da62
+Create Date: 2015-11-04 16:18:18.107314
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '73669db7e12'
+down_revision = '35f538da62'
+
+from alembic import op
+import sqlalchemy as sa
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('repositorybuildtrigger', 'used_legacy_github')
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('repositorybuildtrigger', sa.Column('used_legacy_github', sa.Boolean(), nullable=True))
+    ### end Alembic commands ###

data/migrations/versions/b0e4a7dd82e_drop_v1_checksums_from_imagestorage.py

@@ -0,0 +1,25 @@
+"""Drop v1 checksums from imagestorage
+
+Revision ID: b0e4a7dd82e
+Revises: 22af01f81722
+Create Date: 2015-11-06 16:45:31.517503
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'b0e4a7dd82e'
+down_revision = '22af01f81722'
+
+from alembic import op
+import sqlalchemy as sa
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('imagestorage', 'checksum')
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('imagestorage', sa.Column('checksum', sa.String(length=255), nullable=True))
+    ### end Alembic commands ###

data/model/blob.py

@@ -17,7 +17,8 @@ def get_repo_blob_by_digest(namespace, repo_name, blob_digest):
                     .join(Repository)
                     .join(Namespace, on=(Namespace.id == Repository.namespace_user))
                     .where(Repository.name == repo_name, Namespace.username == namespace,
-                           ImageStorage.checksum == blob_digest, ImageStorage.uploading == False))
+                           ImageStorage.content_checksum == blob_digest,
+                           ImageStorage.uploading == False))
   if not placements:
     raise BlobDoesNotExist('Blob does not exist with digest: {0}'.format(blob_digest))
 
@@ -35,13 +36,14 @@ def store_blob_record_and_temp_link(namespace, repo_name, blob_digest, location_
   with db_transaction():
     repo = _basequery.get_existing_repository(namespace, repo_name)
     try:
-      storage = ImageStorage.get(checksum=blob_digest)
+      storage = ImageStorage.get(content_checksum=blob_digest)
       storage.image_size = byte_count
       storage.save()
 
       ImageStoragePlacement.get(storage=storage, location=location_obj)
     except ImageStorage.DoesNotExist:
-      storage = ImageStorage.create(checksum=blob_digest, uploading=False, image_size=byte_count)
+      storage = ImageStorage.create(content_checksum=blob_digest, uploading=False,
+                                    image_size=byte_count)
       ImageStoragePlacement.create(storage=storage, location=location_obj)
     except ImageStoragePlacement.DoesNotExist:
       ImageStoragePlacement.create(storage=storage, location=location_obj)

data/model/image.py

@@ -281,10 +281,7 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
     except Image.DoesNotExist:
       raise DataModelException('No image with specified id and repository')
 
-    # We cleanup any old checksum in case it's a retry after a fail
-    fetched.storage.checksum = None
     fetched.created = datetime.now()
-
     if created_date_str is not None:
       try:
         fetched.created = dateutil.parser.parse(created_date_str).replace(tzinfo=None)
@@ -292,12 +289,17 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
         # parse raises different exceptions, so we cannot use a specific kind of handler here.
         pass
 
+    # We cleanup any old checksum in case it's a retry after a fail
+    fetched.v1_checksum = None
+    fetched.storage.content_checksum = None
+
     fetched.comment = comment
     fetched.command = command
     fetched.v1_json_metadata = v1_json_metadata
 
     if parent:
       fetched.ancestors = '%s%s/' % (parent.ancestors, parent.id)
+      fetched.parent = parent
 
     fetched.save()
     return fetched
@@ -363,7 +365,8 @@ def get_repo_image_by_storage_checksum(namespace, repository_name, storage_check
             .join(Repository)
             .join(Namespace, on=(Namespace.id == Repository.namespace_user))
             .where(Repository.name == repository_name, Namespace.username == namespace,
-                   ImageStorage.checksum == storage_checksum, ImageStorage.uploading == False)
+                   ImageStorage.content_checksum == storage_checksum,
+                   ImageStorage.uploading == False)
             .get())
   except Image.DoesNotExist:
     msg = 'Image with storage checksum {0} does not exist in repo {1}/{2}'.format(storage_checksum,

data/model/notification.py

@@ -113,12 +113,13 @@ def delete_matching_notifications(target, kind_name, **kwargs):
     notification.delete_instance()
 
 
-def create_repo_notification(repo, event_name, method_name, config, title=None):
+def create_repo_notification(repo, event_name, method_name, method_config, event_config, title=None):
   event = ExternalNotificationEvent.get(ExternalNotificationEvent.name == event_name)
   method = ExternalNotificationMethod.get(ExternalNotificationMethod.name == method_name)
 
   return RepositoryNotification.create(repository=repo, event=event, method=method,
-                                       config_json=json.dumps(config), title=title)
+                                       config_json=json.dumps(method_config), title=title,
+                                       event_config_json=json.dumps(event_config))
 
 
 def get_repo_notification(uuid):

data/model/storage.py

@@ -209,33 +209,18 @@ def get_storage_by_uuid(storage_uuid):
     raise InvalidImageException('No storage found with uuid: %s', storage_uuid)
 
 
-def get_repo_storage_by_checksum(namespace, repository_name, checksum):
-  def filter_to_repo_and_checksum(query):
-    return (query
-            .join(Image)
-            .join(Repository)
-            .join(Namespace, on=(Repository.namespace_user == Namespace.id))
-            .where(Repository.name == repository_name, Namespace.username == namespace,
-                   ImageStorage.checksum == checksum))
-
-  try:
-    return _get_storage(filter_to_repo_and_checksum)
-  except InvalidImageException:
-    raise InvalidImageException('No storage found with checksum {0}'.format(checksum))
-
-
 def get_layer_path(storage_record):
   """ Returns the path in the storage engine to the layer data referenced by the storage row. """
   store = config.store
   if not storage_record.cas_path:
     return store.v1_image_layer_path(storage_record.uuid)
 
-  return store.blob_path(storage_record.checksum)
+  return store.blob_path(storage_record.content_checksum)
 
-def lookup_repo_storages_by_checksum(repo, checksums):
+def lookup_repo_storages_by_content_checksum(repo, checksums):
   """ Looks up repository storages (without placements) matching the given repository
       and checksum. """
   return (ImageStorage
           .select()
           .join(Image)
-          .where(Image.repository == repo, ImageStorage.checksum << checksums))
+          .where(Image.repository == repo, ImageStorage.content_checksum << checksums))

data/queue.py

@@ -50,11 +50,12 @@ class WorkQueue(object):
                    QueueItem.queue_name ** name_match_query))
 
   def _available_jobs(self, now, name_match_query):
-    return (QueueItem
-            .select()
-            .where(QueueItem.queue_name ** name_match_query, QueueItem.available_after <= now,
+    return self._available_jobs_where(QueueItem.select(), now, name_match_query)
+
+  def _available_jobs_where(self, query, now, name_match_query):
+    return query.where(QueueItem.queue_name ** name_match_query, QueueItem.available_after <= now,
                    ((QueueItem.available == True) | (QueueItem.processing_expires <= now)),
-                   QueueItem.retries_remaining > 0))
+                   QueueItem.retries_remaining > 0)
 
   def _available_jobs_not_running(self, now, name_match_query, running_query):
     return (self
@@ -145,25 +146,30 @@ class WorkQueue(object):
 
     item = None
     try:
-      db_item_candidate = avail.order_by(QueueItem.id).get()
-
-      with self._transaction_factory(db):
-        still_available_query = (db_for_update(self
-                                               ._available_jobs(now, name_match_query)
-                                               .where(QueueItem.id == db_item_candidate.id)))
-
-        db_item = still_available_query.get()
-        db_item.available = False
-        db_item.processing_expires = now + timedelta(seconds=processing_time)
-        db_item.retries_remaining -= 1
-        db_item.save()
+      # The previous solution to this used a select for update in a
+      # transaction to prevent multiple instances from processing the
+      # same queue item. This suffered performance problems. This solution
+      # instead has instances attempt to update the potential queue item to be
+      # unavailable. However, since their update clause is restricted to items
+      # that are available=False, only one instance's update will succeed, and
+      # it will have a changed row count of 1. Instances that have 0 changed
+      # rows know that another instance is already handling that item.
+      db_item = avail.order_by(QueueItem.id).get()
+      changed_query = (QueueItem.update(
+          available=False,
+          processing_expires=now + timedelta(seconds=processing_time),
+          retries_remaining=QueueItem.retries_remaining-1,
+        )
+        .where(QueueItem.id == db_item.id))
 
+      changed_query = self._available_jobs_where(changed_query, now, name_match_query)
+      changed = changed_query.execute()
+      if changed == 1:
         item = AttrDict({
           'id': db_item.id,
           'body': db_item.body,
-          'retries_remaining': db_item.retries_remaining
+          'retries_remaining': db_item.retries_remaining - 1,
         })
-
         self._currently_processing = True
     except QueueItem.DoesNotExist:
       self._currently_processing = False