vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into ephemeral

Browse source
This commit is contained in:
Jake Moshenko 2015-01-21 13:39:27 -05:00
commit 44f7ab53a2
46 changed files with 664 additions and 263 deletions
Download patch file Download diff file Expand all files Collapse all files

23
data/database.py
View file

@ -70,6 +70,14 @@ read_slave = Proxy()
db_random_func = CallableProxy()
def validate_database_url(url, connect_timeout=5):
driver = _db_from_url(url, {
'connect_timeout': connect_timeout
})
driver.connect()
driver.close()
def _db_from_url(url, db_kwargs):
parsed_url = make_url(url)
@ -82,6 +90,10 @@ def _db_from_url(url, db_kwargs):
if parsed_url.password:
db_kwargs['password'] = parsed_url.password
# Note: sqlite does not support connect_timeout.
if parsed_url.drivername == 'sqlite' and 'connect_timeout' in db_kwargs:
del db_kwargs['connect_timeout']
return SCHEME_DRIVERS[parsed_url.drivername](parsed_url.database, **db_kwargs)
@ -122,8 +134,9 @@ def close_db_filter(_):
class QuayUserField(ForeignKeyField):
def __init__(self, allows_robots=False, *args, **kwargs):
def __init__(self, allows_robots=False, robot_null_delete=False, *args, **kwargs):
self.allows_robots = allows_robots
self.robot_null_delete = robot_null_delete
if not 'rel_model' in kwargs:
kwargs['rel_model'] = User
@ -157,7 +170,11 @@ class User(BaseModel):
for query, fk in self.dependencies(search_nullable=True):
if isinstance(fk, QuayUserField) and fk.allows_robots:
model = fk.model_class
model.delete().where(query).execute()
if fk.robot_null_delete:
model.update(**{fk.name: None}).where(query).execute()
else:
model.delete().where(query).execute()
# Delete the instance itself.
super(User, self).delete_instance(recursive=False, delete_nullable=False)
@ -459,7 +476,7 @@ class LogEntry(BaseModel):
kind = ForeignKeyField(LogEntryKind, index=True)
account = QuayUserField(index=True, related_name='account')
performer = QuayUserField(allows_robots=True, index=True, null=True,
related_name='performer')
related_name='performer', robot_null_delete=True)
repository = ForeignKeyField(Repository, index=True, null=True)
datetime = DateTimeField(default=datetime.now, index=True)
ip = CharField(null=True)

35
data/migrations/migration.sh
View file

@ -2,13 +2,14 @@ set -e
DOCKER_IP=`echo $DOCKER_HOST | sed 's/tcp:\/\///' | sed 's/:.*//'`
MYSQL_CONFIG_OVERRIDE="{\"DB_URI\":\"mysql+pymysql://root:password@$DOCKER_IP/genschema\"}"
PERCONA_CONFIG_OVERRIDE="{\"DB_URI\":\"mysql+pymysql://root@$DOCKER_IP/genschema\"}"
PGSQL_CONFIG_OVERRIDE="{\"DB_URI\":\"postgresql://postgres@$DOCKER_IP/genschema\"}"
up_mysql() {
# Run a SQL database on port 3306 inside of Docker.
docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mysql
# Sleep for 5s to get MySQL get started.
# Sleep for 10s to get MySQL get started.
echo 'Sleeping for 10...'
sleep 10
@ -25,12 +26,12 @@ up_mariadb() {
# Run a SQL database on port 3306 inside of Docker.
docker run --name mariadb -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mariadb
# Sleep for 5s to get MySQL get started.
# Sleep for 10s to get MySQL get started.
echo 'Sleeping for 10...'
sleep 10
# Add the database to mysql.
docker run --rm --link mariadb:mysql mariadb sh -c 'echo "create database genschema" | mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -ppassword'
docker run --rm --link mariadb:mariadb mariadb sh -c 'echo "create database genschema" | mysql -h"$MARIADB_PORT_3306_TCP_ADDR" -P"$MARIADB_PORT_3306_TCP_PORT" -uroot -ppassword'
}
down_mariadb() {
@ -38,6 +39,23 @@ down_mariadb() {
docker rm mariadb
}
up_percona() {
# Run a SQL database on port 3306 inside of Docker.
docker run --name percona -p 3306:3306 -d dockerfile/percona
# Sleep for 10s
echo 'Sleeping for 10...'
sleep 10
# Add the daabase to mysql.
docker run --rm --link percona:percona dockerfile/percona sh -c 'echo "create database genschema" | mysql -h $PERCONA_PORT_3306_TCP_ADDR'
}
down_percona() {
docker kill percona
docker rm percona
}
up_postgres() {
# Run a SQL database on port 5432 inside of Docker.
docker run --name postgres -p 5432:5432 -d postgres
@ -93,12 +111,23 @@ down_mysql
# Test via MariaDB.
echo '> Starting MariaDB'
up_mariadb
echo '> Testing Migration (mariadb)'
set +e
test_migrate $MYSQL_CONFIG_OVERRIDE
set -e
down_mariadb
# Test via Percona.
echo '> Starting Percona'
up_percona
echo '> Testing Migration (percona)'
set +e
test_migrate $PERCONA_CONFIG_OVERRIDE
set -e
down_percona
# Test via Postgres.
echo '> Starting Postgres'
up_postgres

25
data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py Normal file
View file

@ -0,0 +1,25 @@
"""mysql max index lengths
Revision ID: 228d1af6af1c
Revises: 5b84373e5db
Create Date: 2015-01-06 14:35:24.651424
"""
# revision identifiers, used by Alembic.
revision = '228d1af6af1c'
down_revision = '5b84373e5db'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
def upgrade(tables):
op.drop_index('queueitem_queue_name', table_name='queueitem')
op.create_index('queueitem_queue_name', 'queueitem', ['queue_name'], unique=False, mysql_length=767)
op.drop_index('image_ancestors', table_name='image')
op.create_index('image_ancestors', 'image', ['ancestors'], unique=False, mysql_length=767)
def downgrade(tables):
pass

4
data/migrations/versions/5a07499ce53f_set_up_initial_database.py
View file

@ -53,7 +53,7 @@ def upgrade(tables):
op.create_index('queueitem_available', 'queueitem', ['available'], unique=False)
op.create_index('queueitem_available_after', 'queueitem', ['available_after'], unique=False)
op.create_index('queueitem_processing_expires', 'queueitem', ['processing_expires'], unique=False)
op.create_index('queueitem_queue_name', 'queueitem', ['queue_name'], unique=False)
op.create_index('queueitem_queue_name', 'queueitem', ['queue_name'], unique=False, mysql_length=767)
op.create_table('role',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
@ -376,7 +376,7 @@ def upgrade(tables):
sa.ForeignKeyConstraint(['storage_id'], ['imagestorage.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('image_ancestors', 'image', ['ancestors'], unique=False)
op.create_index('image_ancestors', 'image', ['ancestors'], unique=False, mysql_length=767)
op.create_index('image_repository_id', 'image', ['repository_id'], unique=False)
op.create_index('image_repository_id_docker_image_id', 'image', ['repository_id', 'docker_image_id'], unique=True)
op.create_index('image_storage_id', 'image', ['storage_id'], unique=False)

17
data/model/legacy.py
View file

@ -14,7 +14,7 @@ from data.database import (User, Repository, Image, AccessToken, Role, Repositor
ExternalNotificationEvent, ExternalNotificationMethod,
RepositoryNotification, RepositoryAuthorizedEmail, TeamMemberInvite,
DerivedImageStorage, ImageStorageTransformation, random_string_generator,
db, BUILD_PHASE, QuayUserField)
db, BUILD_PHASE, QuayUserField, validate_database_url)
from peewee import JOIN_LEFT_OUTER, fn
from util.validation import (validate_username, validate_email, validate_password,
INVALID_PASSWORD_MESSAGE)
@ -2257,11 +2257,20 @@ def delete_user(user):
# TODO: also delete any repository data associated
def check_health():
def check_health(app_config):
# Attempt to connect to the database first. If the DB is not responding,
# using the validate_database_url will timeout quickly, as opposed to
# making a normal connect which will just hang (thus breaking the health
# check).
try:
validate_database_url(app_config['DB_URI'], connect_timeout=3)
except Exception:
logger.exception('Could not connect to the database')
return False
# We will connect to the db, check that it contains some log entry kinds
try:
found_count = LogEntryKind.select().count()
return found_count > 0
return bool(list(LogEntryKind.select().limit(1)))
except:
return False