Merge branch 'master' into star
This commit is contained in:
Jimmy Zelinskie
commit
917dd6b674
229 changed files with 10807 additions and 3003 deletions
108
data/database.py
108
data/database.py
|
|
@ -1,14 +1,15 @@
|
|||
import string
|
||||
import logging
|
||||
import uuid
|
||||
import time
|
||||
|
||||
from random import SystemRandom
|
||||
from datetime import datetime
|
||||
from peewee import (Proxy, MySQLDatabase, SqliteDatabase, PostgresqlDatabase, fn, CharField,
|
||||
BooleanField, IntegerField, DateTimeField, ForeignKeyField, TextField,
|
||||
BigIntegerField)
|
||||
from peewee import *
|
||||
from data.read_slave import ReadSlaveModel
|
||||
from sqlalchemy.engine.url import make_url
|
||||
|
||||
from data.read_slave import ReadSlaveModel
|
||||
from util.names import urn_generator
|
||||
|
||||
|
||||
|
|
@ -31,6 +32,16 @@ SCHEME_RANDOM_FUNCTION = {
|
|||
'postgresql+psycopg2': fn.Random,
|
||||
}
|
||||
|
||||
def real_for_update(query):
|
||||
return query.for_update()
|
||||
|
||||
def null_for_update(query):
|
||||
return query
|
||||
|
||||
SCHEME_SPECIALIZED_FOR_UPDATE = {
|
||||
'sqlite': null_for_update,
|
||||
}
|
||||
|
||||
class CallableProxy(Proxy):
|
||||
def __call__(self, *args, **kwargs):
|
||||
if self.obj is None:
|
||||
|
|
@ -70,6 +81,15 @@ class UseThenDisconnect(object):
|
|||
db = Proxy()
|
||||
read_slave = Proxy()
|
||||
db_random_func = CallableProxy()
|
||||
db_for_update = CallableProxy()
|
||||
|
||||
|
||||
def validate_database_url(url, connect_timeout=5):
|
||||
driver = _db_from_url(url, {
|
||||
'connect_timeout': connect_timeout
|
||||
})
|
||||
driver.connect()
|
||||
driver.close()
|
||||
|
||||
|
||||
def _db_from_url(url, db_kwargs):
|
||||
|
|
@ -84,6 +104,10 @@ def _db_from_url(url, db_kwargs):
|
|||
if parsed_url.password:
|
||||
db_kwargs['password'] = parsed_url.password
|
||||
|
||||
# Note: sqlite does not support connect_timeout.
|
||||
if parsed_url.drivername == 'sqlite' and 'connect_timeout' in db_kwargs:
|
||||
del db_kwargs['connect_timeout']
|
||||
|
||||
return SCHEME_DRIVERS[parsed_url.drivername](parsed_url.database, **db_kwargs)
|
||||
|
||||
|
||||
|
|
@ -95,6 +119,8 @@ def configure(config_object):
|
|||
|
||||
parsed_write_uri = make_url(write_db_uri)
|
||||
db_random_func.initialize(SCHEME_RANDOM_FUNCTION[parsed_write_uri.drivername])
|
||||
db_for_update.initialize(SCHEME_SPECIALIZED_FOR_UPDATE.get(parsed_write_uri.drivername,
|
||||
real_for_update))
|
||||
|
||||
read_slave_uri = config_object.get('DB_READ_SLAVE_URI', None)
|
||||
if read_slave_uri is not None:
|
||||
|
|
@ -113,6 +139,9 @@ def uuid_generator():
|
|||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
_get_epoch_timestamp = lambda: int(time.time())
|
||||
|
||||
|
||||
def close_db_filter(_):
|
||||
if not db.is_closed():
|
||||
logger.debug('Disconnecting from database.')
|
||||
|
|
@ -124,8 +153,9 @@ def close_db_filter(_):
|
|||
|
||||
|
||||
class QuayUserField(ForeignKeyField):
|
||||
def __init__(self, allows_robots=False, *args, **kwargs):
|
||||
def __init__(self, allows_robots=False, robot_null_delete=False, *args, **kwargs):
|
||||
self.allows_robots = allows_robots
|
||||
self.robot_null_delete = robot_null_delete
|
||||
if not 'rel_model' in kwargs:
|
||||
kwargs['rel_model'] = User
|
||||
|
||||
|
|
@ -151,6 +181,7 @@ class User(BaseModel):
|
|||
invoice_email = BooleanField(default=False)
|
||||
invalid_login_attempts = IntegerField(default=0)
|
||||
last_invalid_login = DateTimeField(default=datetime.utcnow)
|
||||
removed_tag_expiration_s = IntegerField(default=1209600) # Two weeks
|
||||
|
||||
def delete_instance(self, recursive=False, delete_nullable=False):
|
||||
# If we are deleting a robot account, only execute the subset of queries necessary.
|
||||
|
|
@ -159,7 +190,11 @@ class User(BaseModel):
|
|||
for query, fk in self.dependencies(search_nullable=True):
|
||||
if isinstance(fk, QuayUserField) and fk.allows_robots:
|
||||
model = fk.model_class
|
||||
model.delete().where(query).execute()
|
||||
|
||||
if fk.robot_null_delete:
|
||||
model.update(**{fk.name: None}).where(query).execute()
|
||||
else:
|
||||
model.delete().where(query).execute()
|
||||
|
||||
# Delete the instance itself.
|
||||
super(User, self).delete_instance(recursive=False, delete_nullable=False)
|
||||
|
|
@ -319,6 +354,10 @@ class PermissionPrototype(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
class AccessTokenKind(BaseModel):
|
||||
name = CharField(unique=True, index=True)
|
||||
|
||||
|
||||
class AccessToken(BaseModel):
|
||||
friendly_name = CharField(null=True)
|
||||
code = CharField(default=random_string_generator(length=64), unique=True,
|
||||
|
|
@ -327,6 +366,7 @@ class AccessToken(BaseModel):
|
|||
created = DateTimeField(default=datetime.now)
|
||||
role = ForeignKeyField(Role)
|
||||
temporary = BooleanField(default=True)
|
||||
kind = ForeignKeyField(AccessTokenKind, null=True)
|
||||
|
||||
|
||||
class BuildTriggerService(BaseModel):
|
||||
|
|
@ -368,6 +408,24 @@ class ImageStorageTransformation(BaseModel):
|
|||
name = CharField(index=True, unique=True)
|
||||
|
||||
|
||||
class ImageStorageSignatureKind(BaseModel):
|
||||
name = CharField(index=True, unique=True)
|
||||
|
||||
|
||||
class ImageStorageSignature(BaseModel):
|
||||
storage = ForeignKeyField(ImageStorage, index=True)
|
||||
kind = ForeignKeyField(ImageStorageSignatureKind)
|
||||
signature = TextField(null=True)
|
||||
uploading = BooleanField(default=True, null=True)
|
||||
|
||||
class Meta:
|
||||
database = db
|
||||
read_slaves = (read_slave,)
|
||||
indexes = (
|
||||
(('kind', 'storage'), True),
|
||||
)
|
||||
|
||||
|
||||
class DerivedImageStorage(BaseModel):
|
||||
source = ForeignKeyField(ImageStorage, null=True, related_name='source')
|
||||
derivative = ForeignKeyField(ImageStorage, related_name='derivative')
|
||||
|
|
@ -424,12 +482,15 @@ class RepositoryTag(BaseModel):
|
|||
name = CharField()
|
||||
image = ForeignKeyField(Image)
|
||||
repository = ForeignKeyField(Repository)
|
||||
lifetime_start_ts = IntegerField(default=_get_epoch_timestamp)
|
||||
lifetime_end_ts = IntegerField(null=True, index=True)
|
||||
hidden = BooleanField(default=False)
|
||||
|
||||
class Meta:
|
||||
database = db
|
||||
read_slaves = (read_slave,)
|
||||
indexes = (
|
||||
(('repository', 'name'), True),
|
||||
(('repository', 'name'), False),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -441,23 +502,10 @@ class BUILD_PHASE(object):
|
|||
PULLING = 'pulling'
|
||||
BUILDING = 'building'
|
||||
PUSHING = 'pushing'
|
||||
WAITING = 'waiting'
|
||||
COMPLETE = 'complete'
|
||||
|
||||
|
||||
class RepositoryBuild(BaseModel):
|
||||
uuid = CharField(default=uuid_generator, index=True)
|
||||
repository = ForeignKeyField(Repository, index=True)
|
||||
access_token = ForeignKeyField(AccessToken)
|
||||
resource_key = CharField(index=True)
|
||||
job_config = TextField()
|
||||
phase = CharField(default='waiting')
|
||||
started = DateTimeField(default=datetime.now)
|
||||
display_name = CharField()
|
||||
trigger = ForeignKeyField(RepositoryBuildTrigger, null=True, index=True)
|
||||
pull_robot = QuayUserField(null=True, related_name='buildpullrobot')
|
||||
logs_archived = BooleanField(default=False)
|
||||
|
||||
|
||||
class QueueItem(BaseModel):
|
||||
queue_name = CharField(index=True, max_length=1024)
|
||||
body = TextField()
|
||||
|
|
@ -467,6 +515,21 @@ class QueueItem(BaseModel):
|
|||
retries_remaining = IntegerField(default=5)
|
||||
|
||||
|
||||
class RepositoryBuild(BaseModel):
|
||||
uuid = CharField(default=uuid_generator, index=True)
|
||||
repository = ForeignKeyField(Repository, index=True)
|
||||
access_token = ForeignKeyField(AccessToken)
|
||||
resource_key = CharField(index=True)
|
||||
job_config = TextField()
|
||||
phase = CharField(default=BUILD_PHASE.WAITING)
|
||||
started = DateTimeField(default=datetime.now)
|
||||
display_name = CharField()
|
||||
trigger = ForeignKeyField(RepositoryBuildTrigger, null=True, index=True)
|
||||
pull_robot = QuayUserField(null=True, related_name='buildpullrobot')
|
||||
logs_archived = BooleanField(default=False)
|
||||
queue_item = ForeignKeyField(QueueItem, null=True, index=True)
|
||||
|
||||
|
||||
class LogEntryKind(BaseModel):
|
||||
name = CharField(index=True, unique=True)
|
||||
|
||||
|
|
@ -475,7 +538,7 @@ class LogEntry(BaseModel):
|
|||
kind = ForeignKeyField(LogEntryKind, index=True)
|
||||
account = QuayUserField(index=True, related_name='account')
|
||||
performer = QuayUserField(allows_robots=True, index=True, null=True,
|
||||
related_name='performer')
|
||||
related_name='performer', robot_null_delete=True)
|
||||
repository = ForeignKeyField(Repository, index=True, null=True)
|
||||
datetime = DateTimeField(default=datetime.now, index=True)
|
||||
ip = CharField(null=True)
|
||||
|
|
@ -566,4 +629,5 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission,
|
|||
Notification, ImageStorageLocation, ImageStoragePlacement,
|
||||
ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification,
|
||||
RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage,
|
||||
TeamMemberInvite, Star]
|
||||
TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind,
|
||||
AccessTokenKind, Star]
|
||||
|
|
|
|||
Reference in a new issue