Add quay releases

.dockerignore

@@ -4,10 +4,12 @@ tools
 test/data/registry
 venv
 .git
+!.git/HEAD
+!.git/refs
 .gitignore
 Bobfile
 README.md
 requirements-nover.txt
 run-local.sh
 .DS_Store
-*.pyc
+*.pyc

.gitignore

@@ -10,3 +10,4 @@ node_modules
 static/ldn
 static/fonts
 stack_local
+GIT_HEAD

Dockerfile

@@ -43,6 +43,7 @@ ADD conf/init/doupdatelimits.sh /etc/my_init.d/
 ADD conf/init/copy_syslog_config.sh /etc/my_init.d/
 ADD conf/init/runmigration.sh /etc/my_init.d/
 ADD conf/init/syslog-ng.conf /etc/syslog-ng/
+ADD conf/init/zz_release.sh /etc/my_init.d/
 
 ADD conf/init/service/ /etc/service/
 
@@ -53,6 +54,11 @@ RUN mkdir static/fonts static/ldn
 RUN venv/bin/python -m external_libraries
 RUN mkdir /usr/local/nginx/logs/
 
+# We exclude everything except .git/{HEAD,refs} in .dockerignore so we don't
+# leak data into the image layer.
+RUN cat ".git/$( cat .git/HEAD | cut -d' ' -f 2 )" > GIT_HEAD
+RUN rm -fr .git
+
 # Run the tests
 RUN TEST=true venv/bin/python -m unittest discover -f
 RUN TEST=true venv/bin/python -m test.registry_tests -f

conf/init/zz_release.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+source venv/bin/activate
+
+export PYTHONPATH=.
+
+python /release.py

data/database.py

@@ -756,6 +756,33 @@ class RepositoryAuthorizedEmail(BaseModel):
     )
 
 
+class QuayService(BaseModel):
+  name = CharField(index=True, unique=True)
+
+
+class QuayRegion(BaseModel):
+  name = CharField(index=True, unique=True)
+
+
+class QuayRelease(BaseModel):
+  service = ForeignKeyField(QuayService)
+  version = CharField()
+  region = ForeignKeyField(QuayRegion)
+  reverted = BooleanField(default=False)
+  created = DateTimeField(default=datetime.now, index=True)
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      # unique release per region
+      (('service', 'version', 'region'), True),
+
+      # get recent releases
+      (('service', 'region', 'created'), False),
+    )
+
+
 
 all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, Visibility,
               RepositoryTag, EmailConfirmation, FederatedLogin, LoginService, QueueItem,
@@ -766,4 +793,5 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission,
               ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification,
               RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage,
               TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind,
-              AccessTokenKind, Star, RepositoryActionCount, TagManifest, UserRegion]
+              AccessTokenKind, Star, RepositoryActionCount, TagManifest, UserRegion,
+              QuayService, QuayRegion, QuayRelease]

data/migrations/env.py

@@ -1,8 +1,11 @@
 from __future__ import with_statement
 
+import logging
 import os
 
 from alembic import context
+from alembic.revision import ResolutionError
+from alembic.util import CommandError
 from sqlalchemy import engine_from_config, pool
 from logging.config import fileConfig
 from urllib import unquote, quote
@@ -11,6 +14,7 @@ from peewee import SqliteDatabase
 from data.database import all_models, db
 from app import app
 from data.model.sqlalchemybridge import gen_sqlalchemy_metadata
+from release import GIT_HEAD, REGION, SERVICE
 from util.morecollections import AttrDict
 
 config = context.config
@@ -21,6 +25,8 @@ config.set_main_option('sqlalchemy.url', unquote(app.config['DB_URI']))
 if config.config_file_name:
     fileConfig(config.config_file_name)
 
+logger = logging.getLogger(__name__)
+
 # add your model's MetaData object here
 # for 'autogenerate' support
 # from myapp import mymodel
@@ -77,7 +83,23 @@ def run_migrations_online():
 
     try:
         with context.begin_transaction():
-            context.run_migrations(tables=tables)
+            try:
+                context.run_migrations(tables=tables)
+            except (CommandError, ResolutionError) as ex:
+                if 'No such revision' not in str(ex):
+                    raise
+
+                if not REGION or not GIT_HEAD:
+                    raise
+
+                from data.model.release import get_recent_releases
+
+                # ignore revision error if we're running the previous release
+                releases = list(get_recent_releases(SERVICE, REGION).offset(1).limit(1))
+                if releases and releases[0].version == GIT_HEAD:
+                    logger.warn('Skipping database migration because revision not found')
+                else:
+                    raise
     finally:
         connection.close()
 

data/migrations/versions/1c0f6ede8992_quay_releases.py

@@ -0,0 +1,55 @@
+"""Quay releases
+
+Revision ID: 1c0f6ede8992
+Revises: 545794454f49
+Create Date: 2015-09-15 15:46:09.784607
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '1c0f6ede8992'
+down_revision = '545794454f49'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('quayregion',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(length=255), nullable=False),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_quayregion'))
+    )
+    op.create_index('quayregion_name', 'quayregion', ['name'], unique=True)
+    op.create_table('quayservice',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(length=255), nullable=False),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_quayservice'))
+    )
+    op.create_index('quayservice_name', 'quayservice', ['name'], unique=True)
+    op.create_table('quayrelease',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('service_id', sa.Integer(), nullable=False),
+    sa.Column('version', sa.String(length=255), nullable=False),
+    sa.Column('region_id', sa.Integer(), nullable=False),
+    sa.Column('reverted', sa.Boolean(), nullable=False),
+    sa.Column('created', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['region_id'], ['quayregion.id'], name=op.f('fk_quayrelease_region_id_quayregion')),
+    sa.ForeignKeyConstraint(['service_id'], ['quayservice.id'], name=op.f('fk_quayrelease_service_id_quayservice')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_quayrelease'))
+    )
+    op.create_index('quayrelease_created', 'quayrelease', ['created'], unique=False)
+    op.create_index('quayrelease_region_id', 'quayrelease', ['region_id'], unique=False)
+    op.create_index('quayrelease_service_id', 'quayrelease', ['service_id'], unique=False)
+    op.create_index('quayrelease_service_id_region_id_created', 'quayrelease', ['service_id', 'region_id', 'created'], unique=False)
+    op.create_index('quayrelease_service_id_version_region_id', 'quayrelease', ['service_id', 'version', 'region_id'], unique=True)
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('quayrelease')
+    op.drop_table('quayservice')
+    op.drop_table('quayregion')
+    ### end Alembic commands ###

data/model/release.py

@@ -0,0 +1,23 @@
+from data.database import QuayRelease, QuayRegion, QuayService
+
+
+def set_region_release(service_name, region_name, version):
+  service, _ = QuayService.create_or_get(name=service_name)
+  region, _ = QuayRegion.create_or_get(name=region_name)
+
+  return QuayRelease.create_or_get(service=service, version=version, region=region)
+
+
+def get_recent_releases(service_name, region_name):
+  return (QuayRelease
+    .select(QuayRelease)
+    .join(QuayService)
+    .switch(QuayRelease)
+    .join(QuayRegion)
+    .where(
+      QuayService.name == service_name,
+      QuayRegion.name == region_name,
+      QuayRelease.reverted == False,
+    )
+    .order_by(QuayRelease.created.desc())
+  )

release.py

@@ -0,0 +1,26 @@
+import os
+
+
+_GIT_HEAD_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'GIT_HEAD')
+
+SERVICE = 'quay'
+GIT_HEAD = None
+REGION = os.environ.get('QUAY_REGION')
+
+
+# Load git head if available
+if os.path.isfile(_GIT_HEAD_PATH):
+  with open(_GIT_HEAD_PATH) as f:
+    GIT_HEAD = f.read().strip()
+
+
+def main():
+  from app import app
+  from data.model.release import set_region_release
+
+  if REGION and GIT_HEAD:
+    set_region_release(SERVICE, REGION, GIT_HEAD)
+
+
+if __name__ == '__main__':
+  main()