From 386c017d9975e23c08a9ef2ee3ce35f985a58b14 Mon Sep 17 00:00:00 2001 From: Silas Sewell Date: Wed, 16 Sep 2015 11:44:58 -0400 Subject: [PATCH] Add quay releases --- .dockerignore | 4 +- .gitignore | 1 + Dockerfile | 6 ++ conf/init/zz_release.sh | 8 +++ data/database.py | 30 +++++++++- data/migrations/env.py | 24 +++++++- .../versions/1c0f6ede8992_quay_releases.py | 55 +++++++++++++++++++ data/model/release.py | 23 ++++++++ release.py | 26 +++++++++ 9 files changed, 174 insertions(+), 3 deletions(-) create mode 100755 conf/init/zz_release.sh create mode 100644 data/migrations/versions/1c0f6ede8992_quay_releases.py create mode 100644 data/model/release.py create mode 100644 release.py diff --git a/.dockerignore b/.dockerignore index fb1e0c080..15479142e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,10 +4,12 @@ tools test/data/registry venv .git +!.git/HEAD +!.git/refs .gitignore Bobfile README.md requirements-nover.txt run-local.sh .DS_Store -*.pyc \ No newline at end of file +*.pyc diff --git a/.gitignore b/.gitignore index 00e24caf7..9354014ce 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ node_modules static/ldn static/fonts stack_local +GIT_HEAD diff --git a/Dockerfile b/Dockerfile index ef834f2b6..135a262cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,6 +43,7 @@ ADD conf/init/doupdatelimits.sh /etc/my_init.d/ ADD conf/init/copy_syslog_config.sh /etc/my_init.d/ ADD conf/init/runmigration.sh /etc/my_init.d/ ADD conf/init/syslog-ng.conf /etc/syslog-ng/ +ADD conf/init/zz_release.sh /etc/my_init.d/ ADD conf/init/service/ /etc/service/ @@ -53,6 +54,11 @@ RUN mkdir static/fonts static/ldn RUN venv/bin/python -m external_libraries RUN mkdir /usr/local/nginx/logs/ +# We exclude everything except .git/{HEAD,refs} in .dockerignore so we don't +# leak data into the image layer. +RUN cat ".git/$( cat .git/HEAD | cut -d' ' -f 2 )" > GIT_HEAD +RUN rm -fr .git + # Run the tests RUN TEST=true venv/bin/python -m unittest discover -f RUN TEST=true venv/bin/python -m test.registry_tests -f diff --git a/conf/init/zz_release.sh b/conf/init/zz_release.sh new file mode 100755 index 000000000..152494cff --- /dev/null +++ b/conf/init/zz_release.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +source venv/bin/activate + +export PYTHONPATH=. + +python /release.py diff --git a/data/database.py b/data/database.py index 660976bcf..2a991f111 100644 --- a/data/database.py +++ b/data/database.py @@ -756,6 +756,33 @@ class RepositoryAuthorizedEmail(BaseModel): ) +class QuayService(BaseModel): + name = CharField(index=True, unique=True) + + +class QuayRegion(BaseModel): + name = CharField(index=True, unique=True) + + +class QuayRelease(BaseModel): + service = ForeignKeyField(QuayService) + version = CharField() + region = ForeignKeyField(QuayRegion) + reverted = BooleanField(default=False) + created = DateTimeField(default=datetime.now, index=True) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + # unique release per region + (('service', 'version', 'region'), True), + + # get recent releases + (('service', 'region', 'created'), False), + ) + + all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, Visibility, RepositoryTag, EmailConfirmation, FederatedLogin, LoginService, QueueItem, @@ -766,4 +793,5 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification, RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage, TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind, - AccessTokenKind, Star, RepositoryActionCount, TagManifest, UserRegion] + AccessTokenKind, Star, RepositoryActionCount, TagManifest, UserRegion, + QuayService, QuayRegion, QuayRelease] diff --git a/data/migrations/env.py b/data/migrations/env.py index 108c4c496..c53421f50 100644 --- a/data/migrations/env.py +++ b/data/migrations/env.py @@ -1,8 +1,11 @@ from __future__ import with_statement +import logging import os from alembic import context +from alembic.revision import ResolutionError +from alembic.util import CommandError from sqlalchemy import engine_from_config, pool from logging.config import fileConfig from urllib import unquote, quote @@ -11,6 +14,7 @@ from peewee import SqliteDatabase from data.database import all_models, db from app import app from data.model.sqlalchemybridge import gen_sqlalchemy_metadata +from release import GIT_HEAD, REGION, SERVICE from util.morecollections import AttrDict config = context.config @@ -21,6 +25,8 @@ config.set_main_option('sqlalchemy.url', unquote(app.config['DB_URI'])) if config.config_file_name: fileConfig(config.config_file_name) +logger = logging.getLogger(__name__) + # add your model's MetaData object here # for 'autogenerate' support # from myapp import mymodel @@ -77,7 +83,23 @@ def run_migrations_online(): try: with context.begin_transaction(): - context.run_migrations(tables=tables) + try: + context.run_migrations(tables=tables) + except (CommandError, ResolutionError) as ex: + if 'No such revision' not in str(ex): + raise + + if not REGION or not GIT_HEAD: + raise + + from data.model.release import get_recent_releases + + # ignore revision error if we're running the previous release + releases = list(get_recent_releases(SERVICE, REGION).offset(1).limit(1)) + if releases and releases[0].version == GIT_HEAD: + logger.warn('Skipping database migration because revision not found') + else: + raise finally: connection.close() diff --git a/data/migrations/versions/1c0f6ede8992_quay_releases.py b/data/migrations/versions/1c0f6ede8992_quay_releases.py new file mode 100644 index 000000000..92583881d --- /dev/null +++ b/data/migrations/versions/1c0f6ede8992_quay_releases.py @@ -0,0 +1,55 @@ +"""Quay releases + +Revision ID: 1c0f6ede8992 +Revises: 545794454f49 +Create Date: 2015-09-15 15:46:09.784607 + +""" + +# revision identifiers, used by Alembic. +revision = '1c0f6ede8992' +down_revision = '545794454f49' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('quayregion', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_quayregion')) + ) + op.create_index('quayregion_name', 'quayregion', ['name'], unique=True) + op.create_table('quayservice', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_quayservice')) + ) + op.create_index('quayservice_name', 'quayservice', ['name'], unique=True) + op.create_table('quayrelease', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('service_id', sa.Integer(), nullable=False), + sa.Column('version', sa.String(length=255), nullable=False), + sa.Column('region_id', sa.Integer(), nullable=False), + sa.Column('reverted', sa.Boolean(), nullable=False), + sa.Column('created', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['region_id'], ['quayregion.id'], name=op.f('fk_quayrelease_region_id_quayregion')), + sa.ForeignKeyConstraint(['service_id'], ['quayservice.id'], name=op.f('fk_quayrelease_service_id_quayservice')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_quayrelease')) + ) + op.create_index('quayrelease_created', 'quayrelease', ['created'], unique=False) + op.create_index('quayrelease_region_id', 'quayrelease', ['region_id'], unique=False) + op.create_index('quayrelease_service_id', 'quayrelease', ['service_id'], unique=False) + op.create_index('quayrelease_service_id_region_id_created', 'quayrelease', ['service_id', 'region_id', 'created'], unique=False) + op.create_index('quayrelease_service_id_version_region_id', 'quayrelease', ['service_id', 'version', 'region_id'], unique=True) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('quayrelease') + op.drop_table('quayservice') + op.drop_table('quayregion') + ### end Alembic commands ### diff --git a/data/model/release.py b/data/model/release.py new file mode 100644 index 000000000..883ae146e --- /dev/null +++ b/data/model/release.py @@ -0,0 +1,23 @@ +from data.database import QuayRelease, QuayRegion, QuayService + + +def set_region_release(service_name, region_name, version): + service, _ = QuayService.create_or_get(name=service_name) + region, _ = QuayRegion.create_or_get(name=region_name) + + return QuayRelease.create_or_get(service=service, version=version, region=region) + + +def get_recent_releases(service_name, region_name): + return (QuayRelease + .select(QuayRelease) + .join(QuayService) + .switch(QuayRelease) + .join(QuayRegion) + .where( + QuayService.name == service_name, + QuayRegion.name == region_name, + QuayRelease.reverted == False, + ) + .order_by(QuayRelease.created.desc()) + ) diff --git a/release.py b/release.py new file mode 100644 index 000000000..91a46f796 --- /dev/null +++ b/release.py @@ -0,0 +1,26 @@ +import os + + +_GIT_HEAD_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'GIT_HEAD') + +SERVICE = 'quay' +GIT_HEAD = None +REGION = os.environ.get('QUAY_REGION') + + +# Load git head if available +if os.path.isfile(_GIT_HEAD_PATH): + with open(_GIT_HEAD_PATH) as f: + GIT_HEAD = f.read().strip() + + +def main(): + from app import app + from data.model.release import set_region_release + + if REGION and GIT_HEAD: + set_region_release(SERVICE, REGION, GIT_HEAD) + + +if __name__ == '__main__': + main()