vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Upgrade Peewee to latest 3.x

Browse source 
This requires a number of small changes in the data model code, as well as additional testing.
This commit is contained in:
Brad Ison 2018-04-06 13:48:01 -04:00 committed by Joseph Schorr
parent 70b7ee4654
commit d3d9cca182
26 changed files with 220 additions and 193 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
buildman/jobutil/buildjob.py
View file

@ -194,12 +194,19 @@ class BuildJobNotifier(object):
with UseThenDisconnect(app.config):
tags = self.build_config.get('docker_tags', ['latest'])
trigger = self.repo_build.trigger
# TODO(bison): This is weird hack. Figure this out.
if trigger is not None and trigger.id is not None:
trigger_kind = trigger.service.name
else:
trigger_kind = None
event_data = {
'build_id': self.repo_build.uuid,
'build_name': self.repo_build.display_name,
'docker_tags': tags,
'trigger_id': trigger.uuid if trigger is not None else None,
'trigger_kind': trigger.service.name if trigger is not None else None,
'trigger_kind': trigger_kind,
'trigger_metadata': self.build_config.get('trigger_metadata', {})
}

99
data/database.py
View file

@ -17,7 +17,7 @@ import toposort
from enum import Enum
from peewee import *
from playhouse.shortcuts import RetryOperationalError
from peewee import __exception_wrapper__, Function
from playhouse.pool import PooledMySQLDatabase, PooledPostgresqlDatabase, PooledSqliteDatabase
from sqlalchemy.engine.url import make_url
@ -121,27 +121,27 @@ def delete_instance_filtered(instance, model_class, delete_nullable, skip_transi
# We only want to skip transitive deletes, which are done using subqueries in the form of
# DELETE FROM <table> in <subquery>. If an op is not using a subquery, we allow it to be
# applied directly.
if fk.model_class not in skip_transitive_deletes or query.op != 'in':
if fk.model not in skip_transitive_deletes or query.op.lower() != 'in':
filtered_ops.append((query, fk))
if query.op == 'in':
dependencies[fk.model_class.__name__].add(query.rhs.model_class.__name__)
if query.op.lower() == 'in':
dependencies[fk.model.__name__].add(query.rhs.model.__name__)
elif query.op == '=':
dependencies[fk.model_class.__name__].add(model_class.__name__)
dependencies[fk.model.__name__].add(model_class.__name__)
else:
raise RuntimeError('Unknown operator in recursive repository delete query')
sorted_models = list(reversed(toposort.toposort_flatten(dependencies)))
def sorted_model_key(query_fk_tuple):
cmp_query, cmp_fk = query_fk_tuple
if cmp_query.op == 'in':
if cmp_query.op.lower() == 'in':
return -1
return sorted_models.index(cmp_fk.model_class.__name__)
return sorted_models.index(cmp_fk.model.__name__)
filtered_ops.sort(key=sorted_model_key)
with db_transaction():
for query, fk in filtered_ops:
_model = fk.model_class
_model = fk.model
if fk.null and not delete_nullable:
_model.update(**{fk.name: None}).where(query).execute()
else:
@ -162,6 +162,24 @@ class CallableProxy(Proxy):
return self.obj(*args, **kwargs)
class RetryOperationalError(object):
def execute_sql(self, sql, params=None, commit=True):
try:
cursor = super(RetryOperationalError, self).execute_sql(sql, params, commit)
except OperationalError:
if not self.is_closed():
self.close()
with __exception_wrapper__:
cursor = self.cursor()
cursor.execute(sql, params or ())
if commit and not self.in_transaction():
self.commit()
return cursor
class CloseForLongOperation(object):
""" Helper object which disconnects the database then reconnects after the nested operation
completes.
@ -214,11 +232,11 @@ class TupleSelector(object):
@classmethod
def tuple_reference_key(cls, field):
""" Returns a string key for referencing a field in a TupleSelector. """
if field._node_type == 'func':
if isinstance(field, Function):
return field.name + ','.join([cls.tuple_reference_key(arg) for arg in field.arguments])
if field._node_type == 'field':
return field.name + ':' + field.model_class.__name__
if isinstance(field, Field):
return field.name + ':' + field.model.__name__
raise Exception('Unknown field type %s in TupleSelector' % field._node_type)
@ -268,6 +286,9 @@ def _db_from_url(url, db_kwargs, connect_timeout=DEFAULT_DB_CONNECT_TIMEOUT):
if parsed_url.password:
db_kwargs['password'] = parsed_url.password
# Remove threadlocals. It used to be required.
db_kwargs.pop('threadlocals', None)
# Note: sqlite does not support connect_timeout.
if parsed_url.drivername != 'sqlite':
db_kwargs['connect_timeout'] = db_kwargs.get('connect_timeout', connect_timeout)
@ -285,8 +306,9 @@ def _db_from_url(url, db_kwargs, connect_timeout=DEFAULT_DB_CONNECT_TIMEOUT):
db_kwargs.pop('stale_timeout', None)
db_kwargs.pop('max_connections', None)
wrapped_driver = _wrap_for_retry(driver)
return wrapped_driver(parsed_url.database, **db_kwargs)
# wrapped_driver = _wrap_for_retry(driver)
# return wrapped_driver(parsed_url.database, **db_kwargs)
return driver(parsed_url.database, **db_kwargs)
def configure(config_object):
@ -351,20 +373,20 @@ class QuayUserField(ForeignKeyField):
def __init__(self, allows_robots=False, robot_null_delete=False, *args, **kwargs):
self.allows_robots = allows_robots
self.robot_null_delete = robot_null_delete
if 'rel_model' not in kwargs:
kwargs['rel_model'] = User
if 'model' not in kwargs:
kwargs['model'] = User
super(QuayUserField, self).__init__(*args, **kwargs)
class EnumField(ForeignKeyField):
""" Create a cached python Enum from an EnumTable """
def __init__(self, rel_model, enum_key_field='name', *args, **kwargs):
def __init__(self, model, enum_key_field='name', *args, **kwargs):
"""
rel_model is the EnumTable model-class (see ForeignKeyField)
model is the EnumTable model-class (see ForeignKeyField)
enum_key_field is the field from the EnumTable to use as the enum name
"""
self.enum_key_field = enum_key_field
super(EnumField, self).__init__(rel_model, *args, **kwargs)
super(EnumField, self).__init__(model, *args, **kwargs)
@property
@lru_cache(maxsize=1)
@ -412,7 +434,7 @@ class BaseModel(ReadSlaveModel):
if name.endswith('_id'):
field_name = name[0:len(name) - 3]
if field_name in self._meta.fields:
return self._data.get(field_name)
return self.__data__.get(field_name)
return super(BaseModel, self).__getattribute__(name)
@ -449,7 +471,7 @@ class User(BaseModel):
# For all the model dependencies, only delete those that allow robots.
for query, fk in reversed(list(self.dependencies(search_nullable=True))):
if isinstance(fk, QuayUserField) and fk.allows_robots:
_model = fk.model_class
_model = fk.model
if fk.robot_null_delete:
_model.update(**{fk.name: None}).where(query).execute()
@ -551,7 +573,7 @@ class TeamMemberInvite(BaseModel):
user = QuayUserField(index=True, null=True)
email = CharField(null=True)
team = ForeignKeyField(Team)
inviter = ForeignKeyField(User, related_name='inviter')
inviter = ForeignKeyField(User, backref='inviter')
invite_token = CharField(default=urn_generator(['teaminvite']))
@ -664,13 +686,13 @@ class RepositoryPermission(BaseModel):
class PermissionPrototype(BaseModel):
org = QuayUserField(index=True, related_name='orgpermissionproto')
org = QuayUserField(index=True, backref='orgpermissionproto')
uuid = CharField(default=uuid_generator)
activating_user = QuayUserField(allows_robots=True, index=True, null=True,
related_name='userpermissionproto')
delegate_user = QuayUserField(allows_robots=True, related_name='receivingpermission',
backref='userpermissionproto')
delegate_user = QuayUserField(allows_robots=True, backref='receivingpermission',
null=True)
delegate_team = ForeignKeyField(Team, related_name='receivingpermission',
delegate_team = ForeignKeyField(Team, backref='receivingpermission',
null=True)
role = ForeignKeyField(Role)
@ -714,7 +736,7 @@ class RepositoryBuildTrigger(BaseModel):
private_key = TextField(null=True)
config = TextField(default='{}')
write_token = ForeignKeyField(AccessToken, null=True)
pull_robot = QuayUserField(allows_robots=True, null=True, related_name='triggerpullrobot',
pull_robot = QuayUserField(allows_robots=True, null=True, backref='triggerpullrobot',
robot_null_delete=True)
enabled = BooleanField(default=True)
disabled_reason = EnumField(DisableReason, null=True)
@ -789,9 +811,6 @@ class UserRegion(BaseModel):
)
_ImageProxy = Proxy()
class Image(BaseModel):
# This class is intentionally denormalized. Even though images are supposed
# to be globally unique we can't treat them as such for permissions and
@ -816,7 +835,7 @@ class Image(BaseModel):
security_indexed_engine = IntegerField(default=IMAGE_NOT_SCANNED_ENGINE_VERSION, index=True)
# We use a proxy here instead of 'self' in order to disable the foreign key constraint
parent = ForeignKeyField(_ImageProxy, null=True, related_name='children')
parent = DeferredForeignKey('Image', null=True, backref='children')
class Meta:
database = db
@ -835,9 +854,6 @@ class Image(BaseModel):
return map(int, self.ancestors.split('/')[1:-1])
_ImageProxy.initialize(Image)
class DerivedStorageForImage(BaseModel):
source_image = ForeignKeyField(Image)
derivative = ForeignKeyField(ImageStorage)
@ -942,7 +958,7 @@ class RepositoryBuild(BaseModel):
started = DateTimeField(default=datetime.now, index=True)
display_name = CharField()
trigger = ForeignKeyField(RepositoryBuildTrigger, null=True)
pull_robot = QuayUserField(null=True, related_name='buildpullrobot', allows_robots=True,
pull_robot = QuayUserField(null=True, backref='buildpullrobot', allows_robots=True,
robot_null_delete=True)
logs_archived = BooleanField(default=False)
queue_id = CharField(null=True, index=True)
@ -962,9 +978,9 @@ class LogEntryKind(BaseModel):
class LogEntry(BaseModel):
kind = ForeignKeyField(LogEntryKind)
account = IntegerField(index=True, db_column='account_id')
performer = IntegerField(index=True, null=True, db_column='performer_id')
repository = IntegerField(index=True, null=True, db_column='repository_id')
account = IntegerField(index=True, column_name='account_id')
performer = IntegerField(index=True, null=True, column_name='performer_id')
repository = IntegerField(index=True, null=True, column_name='repository_id')
datetime = DateTimeField(default=datetime.now, index=True)
ip = CharField(null=True)
metadata_json = TextField(default='{}')
@ -1024,7 +1040,7 @@ class OAuthApplication(BaseModel):
name = CharField()
description = TextField(default='')
avatar_email = CharField(null=True, db_column='gravatar_email')
avatar_email = CharField(null=True, column_name='gravatar_email')
class OAuthAuthorizationCode(BaseModel):
@ -1163,15 +1179,12 @@ class ServiceKeyApprovalType(Enum):
AUTOMATIC = 'Automatic'
_ServiceKeyApproverProxy = Proxy()
class ServiceKeyApproval(BaseModel):
approver = ForeignKeyField(_ServiceKeyApproverProxy, null=True)
approver = QuayUserField(null=True)
approval_type = CharField(index=True)
approved_date = DateTimeField(default=datetime.utcnow)
notes = TextField(default='')
_ServiceKeyApproverProxy.initialize(User)
class ServiceKey(BaseModel):
name = CharField()
@ -1309,7 +1322,7 @@ class ApprTag(BaseModel):
reverted = BooleanField(default=False)
protected = BooleanField(default=False)
tag_kind = EnumField(ApprTagKind)
linked_tag = ForeignKeyField('self', null=True, related_name='tag_parents')
linked_tag = ForeignKeyField('self', null=True, backref='tag_parents')
class Meta:
database = db

2
data/fields.py
View file

@ -2,7 +2,7 @@ import base64
import resumablehashlib
import json
from peewee import TextField, CharField, Clause
from peewee import TextField, CharField
from data.text import prefix_search

7
data/model/_basequery.py
View file

@ -94,9 +94,7 @@ def filter_to_repos_for_user(query, user_id=None, namespace=None, repo_kind='ima
queries = []
if include_public:
queries.append(query
.clone()
.where(Repository.visibility == get_public_repo_visibility()))
queries.append(query.where(Repository.visibility == get_public_repo_visibility()))
if user_id is not None:
AdminTeam = Team.alias()
@ -104,13 +102,11 @@ def filter_to_repos_for_user(query, user_id=None, namespace=None, repo_kind='ima
# Add repositories in which the user has permission.
queries.append(query
.clone()
.switch(RepositoryPermission)
.where(RepositoryPermission.user == user_id))
# Add repositories in which the user is a member of a team that has permission.
queries.append(query
.clone()
.switch(RepositoryPermission)
.join(Team)
.join(TeamMember)
@ -118,7 +114,6 @@ def filter_to_repos_for_user(query, user_id=None, namespace=None, repo_kind='ima
# Add repositories under namespaces in which the user is the org admin.
queries.append(query
.clone()
.switch(Repository)
.join(AdminTeam, on=(Repository.namespace_user == AdminTeam.organization))
.join(AdminTeamMember, on=(AdminTeam.id == AdminTeamMember.team))

11
data/model/build.py
View file

@ -2,7 +2,7 @@ import json
import os
from datetime import timedelta, datetime
from peewee import JOIN_LEFT_OUTER
from peewee import JOIN
import features
from data.database import (BuildTriggerService, RepositoryBuildTrigger, Repository, Namespace, User,
@ -50,7 +50,7 @@ def get_build_trigger(trigger_uuid):
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(RepositoryBuildTrigger)
.join(User)
.join(User, on=(RepositoryBuildTrigger.connected_user == User.id))
.where(RepositoryBuildTrigger.uuid == trigger_uuid)
.get())
except RepositoryBuildTrigger.DoesNotExist:
@ -94,10 +94,10 @@ def _get_build_base_query():
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(RepositoryBuild)
.join(User, JOIN_LEFT_OUTER)
.join(User, JOIN.LEFT_OUTER)
.switch(RepositoryBuild)
.join(RepositoryBuildTrigger, JOIN_LEFT_OUTER)
.join(BuildTriggerService, JOIN_LEFT_OUTER)
.join(RepositoryBuildTrigger, JOIN.LEFT_OUTER)
.join(BuildTriggerService, JOIN.LEFT_OUTER)
.order_by(RepositoryBuild.started.desc()))
@ -308,4 +308,3 @@ def update_trigger_disable_status(trigger, final_phase):
else:
# Save the trigger changes.
trigger.save()

8
data/model/image.py
View file

@ -6,7 +6,7 @@ from collections import defaultdict
from datetime import datetime
import dateutil.parser
from peewee import JOIN_LEFT_OUTER, IntegrityError, fn
from peewee import JOIN, IntegrityError, fn
from data.model import (DataModelException, db_transaction, _basequery, storage,
InvalidImageException)
@ -273,7 +273,7 @@ def find_create_or_link_image(docker_image_id, repo_obj, username, translations,
.join(ImageStorage)
.switch(Image)
.join(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER)
.join(RepositoryPermission, JOIN.LEFT_OUTER)
.switch(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(ImageStorage.uploading == False,
@ -445,8 +445,8 @@ def get_image_with_storage_and_parent_base():
.select(Image, ImageStorage, Parent, ParentImageStorage)
.join(ImageStorage)
.switch(Image)
.join(Parent, JOIN_LEFT_OUTER, on=(Image.parent == Parent.id))
.join(ParentImageStorage, JOIN_LEFT_OUTER, on=(ParentImageStorage.id == Parent.storage)))
.join(Parent, JOIN.LEFT_OUTER, on=(Image.parent == Parent.id))
.join(ParentImageStorage, JOIN.LEFT_OUTER, on=(ParentImageStorage.id == Parent.storage)))
def set_secscan_status(image, indexed, version):

6
data/model/log.py
View file

@ -2,7 +2,7 @@ import json
import logging
from calendar import timegm
from peewee import JOIN_LEFT_OUTER, fn, PeeweeException
from peewee import JOIN, fn, PeeweeException
from datetime import datetime, timedelta
from cachetools import lru_cache
@ -82,11 +82,11 @@ def get_logs_query(start_time, end_time, performer=None, repository=None, namesp
query = _logs_query(selections, start_time, end_time, performer, repository, namespace, ignore,
model=model)
query = (query.switch(model).join(Performer, JOIN_LEFT_OUTER,
query = (query.switch(model).join(Performer, JOIN.LEFT_OUTER,
on=(model.performer == Performer.id).alias('performer')))
if namespace is None and repository is None:
query = (query.switch(model).join(Account, JOIN_LEFT_OUTER,
query = (query.switch(model).join(Account, JOIN.LEFT_OUTER,
on=(model.account == Account.id).alias('account')))
return query

14
data/model/notification.py
View file

@ -37,7 +37,16 @@ def lookup_notifications_by_path_prefix(prefix):
def list_notifications(user, kind_name=None, id_filter=None, include_dismissed=False,
page=None, limit=None):
base_query = Notification.select().join(NotificationKind)
base_query = (Notification
.select(Notification.id,
Notification.uuid,
Notification.kind,
Notification.metadata_json,
Notification.dismissed,
Notification.lookup_path,
Notification.created.alias('cd'),
Notification.target)
.join(NotificationKind))
if kind_name is not None:
base_query = base_query.where(NotificationKind.name == kind_name)
@ -73,7 +82,8 @@ def list_notifications(user, kind_name=None, id_filter=None, include_dismissed=F
elif limit:
query = query.limit(limit)
return query.order_by(base_query.c.created.desc())
from peewee import SQL
return query.order_by(SQL('cd desc'))
def delete_all_notifications_by_path_prefix(prefix):

10
data/model/permission.py
View file

@ -1,4 +1,4 @@
from peewee import JOIN_LEFT_OUTER
from peewee import JOIN
from data.database import (RepositoryPermission, User, Repository, Visibility, Role, TeamMember,
PermissionPrototype, Team, TeamRole, Namespace)
@ -112,13 +112,13 @@ def get_prototype_permissions(org):
query = (PermissionPrototype
.select()
.where(PermissionPrototype.org == org)
.join(ActivatingUser, JOIN_LEFT_OUTER,
.join(ActivatingUser, JOIN.LEFT_OUTER,
on=(ActivatingUser.id == PermissionPrototype.activating_user))
.join(DelegateUser, JOIN_LEFT_OUTER,
.join(DelegateUser, JOIN.LEFT_OUTER,
on=(DelegateUser.id == PermissionPrototype.delegate_user))
.join(Team, JOIN_LEFT_OUTER,
.join(Team, JOIN.LEFT_OUTER,
on=(Team.id == PermissionPrototype.delegate_team))
.join(Role, JOIN_LEFT_OUTER, on=(Role.id == PermissionPrototype.role)))
.join(Role, JOIN.LEFT_OUTER, on=(Role.id == PermissionPrototype.role)))
return query

7
data/model/repository.py
View file

@ -3,8 +3,7 @@ import random
from enum import Enum
from datetime import timedelta, datetime
from peewee import JOIN_LEFT_OUTER, fn, SQL, IntegrityError
from playhouse.shortcuts import case
from peewee import Case, JOIN, fn, SQL, IntegrityError
from cachetools import ttl_cache
from data.model import (
@ -406,7 +405,7 @@ def get_visible_repositories(username, namespace=None, kind_filter='image', incl
user_id = None
if username:
# Note: We only need the permissions table if we will filter based on a user's permissions.
query = query.switch(Repository).distinct().join(RepositoryPermission, JOIN_LEFT_OUTER)
query = query.switch(Repository).distinct().join(RepositoryPermission, JOIN.LEFT_OUTER)
found_namespace = _get_namespace_user(username)
if not found_namespace:
return Repository.select(Repository.id.alias('rid')).where(Repository.id == -1)
@ -552,7 +551,7 @@ def _get_sorted_matching_repositories(lookup_value, repo_kind='image', include_p
if SEARCH_FIELDS.description.name in search_fields:
clause = Repository.description.match(lookup_value) | clause
cases = [(Repository.name.match(lookup_value), 100 * RepositorySearchScore.score),]
computed_score = case(None, cases, RepositorySearchScore.score).alias('score')
computed_score = Case(None, cases, RepositorySearchScore.score).alias('score')
select_fields.append(computed_score)
query = (Repository.select(*select_fields)

4
data/model/service_keys.py
View file

@ -2,7 +2,7 @@ import re
from calendar import timegm
from datetime import datetime, timedelta
from peewee import JOIN_LEFT_OUTER
from peewee import JOIN
from Crypto.PublicKey import RSA
from jwkest.jwk import RSAKey
@ -165,7 +165,7 @@ def approve_service_key(kid, approver, approval_type, notes=''):
def _list_service_keys_query(kid=None, service=None, approved_only=True, alive_only=True,
approval_type=None):
query = ServiceKey.select().join(ServiceKeyApproval, JOIN_LEFT_OUTER)
query = ServiceKey.select().join(ServiceKeyApproval, JOIN.LEFT_OUTER)
if approved_only:
query = query.where(~(ServiceKey.approval >> None))

13
data/model/sqlalchemybridge.py
View file

@ -50,8 +50,7 @@ def gen_sqlalchemy_metadata(peewee_model_list):
alchemy_type = Integer
all_indexes.add(((field.name, ), field.unique))
if not field.deferred:
target_name = '%s.%s' % (field.to_field.model_class._meta.db_table,
field.to_field.db_column)
target_name = '%s.%s' % (field.rel_model._meta.table_name, field.rel_field.column_name)
col_args.append(ForeignKey(target_name))
elif isinstance(field, BigIntegerField):
alchemy_type = BigInteger
@ -74,19 +73,19 @@ def gen_sqlalchemy_metadata(peewee_model_list):
if field.unique or field.index:
all_indexes.add(((field.name, ), field.unique))
new_col = Column(field.db_column, alchemy_type, *col_args, **col_kwargs)
new_col = Column(field.column_name, alchemy_type, *col_args, **col_kwargs)
columns.append(new_col)
new_table = Table(meta.db_table, metadata, *columns)
new_table = Table(meta.table_name, metadata, *columns)
for col_prop_names, unique in all_indexes:
col_names = [meta.fields[prop_name].db_column for prop_name in col_prop_names]
index_name = '%s_%s' % (meta.db_table, '_'.join(col_names))
col_names = [meta.fields[prop_name].column_name for prop_name in col_prop_names]
index_name = '%s_%s' % (meta.table_name, '_'.join(col_names))
col_refs = [getattr(new_table.c, col_name) for col_name in col_names]
Index(index_name, *col_refs, unique=unique)
for col_field_name in fulltext_indexes:
index_name = '%s_%s__fulltext' % (meta.db_table, col_field_name)
index_name = '%s_%s__fulltext' % (meta.table_name, col_field_name)
col_ref = getattr(new_table.c, col_field_name)
Index(index_name, col_ref, postgresql_ops={col_field_name: 'gin_trgm_ops'},
postgresql_using='gin',

15
data/model/tag.py
View file

@ -4,7 +4,7 @@ import time
from calendar import timegm
from uuid import uuid4
from peewee import IntegrityError, JOIN_LEFT_OUTER, fn
from peewee import IntegrityError, JOIN, fn
from data.model import (image, db_transaction, DataModelException, _basequery,
InvalidManifestException, TagAlreadyCreatedException, StaleTagException,
config)
@ -44,8 +44,8 @@ def get_tags_images_eligible_for_scan(clair_version):
.join(Image, on=(RepositoryTag.image == Image.id))
.join(ImageStorage, on=(Image.storage == ImageStorage.id))
.switch(Image)
.join(Parent, JOIN_LEFT_OUTER, on=(Image.parent == Parent.id))
.join(ParentImageStorage, JOIN_LEFT_OUTER, on=(ParentImageStorage.id == Parent.storage))
.join(Parent, JOIN.LEFT_OUTER, on=(Image.parent == Parent.id))
.join(ParentImageStorage, JOIN.LEFT_OUTER, on=(ParentImageStorage.id == Parent.storage))
.where(RepositoryTag.hidden == False)
.where(Image.security_indexed_engine < clair_version))
@ -71,7 +71,7 @@ def filter_tags_have_repository_event(query, event):
lifetime_start_ts.
"""
query = filter_has_repository_event(query, event)
query = query.switch(Image).join(ImageStorage)
query = query.switch(RepositoryTag).join(Image).join(ImageStorage)
query = query.switch(RepositoryTag).order_by(RepositoryTag.lifetime_start_ts.desc())
return query
@ -146,12 +146,13 @@ def get_matching_tags_for_images(image_pairs, filter_images=None, filter_tags=No
# Collect IDs of the tags found for each query.
tags = {}
for query in sharded_queries:
ImageAlias = Image.alias()
tag_query = (_tag_alive(RepositoryTag
.select(*(selections or []))
.distinct()
.join(Image)
.join(ImageAlias)
.where(RepositoryTag.hidden == False)
.where(Image.id << query)
.where(ImageAlias.id << query)
.switch(RepositoryTag)))
if filter_tags is not None:
@ -210,7 +211,7 @@ def list_active_repo_tags(repo):
.join(Image)
.where(RepositoryTag.repository == repo, RepositoryTag.hidden == False)
.switch(RepositoryTag)
.join(TagManifest, JOIN_LEFT_OUTER))
.join(TagManifest, JOIN.LEFT_OUTER))
return query

11
data/model/team.py
View file

@ -148,12 +148,11 @@ def add_or_invite_to_team(inviter, team, user_obj=None, email=None, requires_inv
def get_matching_user_teams(team_prefix, user_obj, limit=10):
team_prefix_search = prefix_search(Team.name, team_prefix)
query = (Team
.select()
.select(Team.id.distinct(), Team)
.join(User)
.switch(Team)
.join(TeamMember)
.where(TeamMember.user == user_obj, team_prefix_search)
.distinct(Team.id)
.limit(limit))
return query
@ -179,12 +178,11 @@ def get_matching_admined_teams(team_prefix, user_obj, limit=10):
.where(TeamRole.name == 'admin'))
query = (Team
.select()
.select(Team.id.distinct(), Team)
.join(User)
.switch(Team)
.join(TeamMember)
.where(team_prefix_search, Team.organization << (admined_orgs))
.distinct(Team.id)
.limit(limit))
return query
@ -260,8 +258,9 @@ def get_user_teams_within_org(username, organization):
def list_organization_members_by_teams(organization):
query = (TeamMember
.select(Team, User)
.annotate(Team)
.annotate(User)
.join(Team)
.switch(TeamMember)
.join(User)
.where(Team.organization == organization))
return query

4
data/model/test/test_basequery.py
View file

@ -1,6 +1,6 @@
import pytest
from peewee import JOIN_LEFT_OUTER
from peewee import JOIN
from playhouse.test_utils import assert_query_count
from data.database import Repository, RepositoryPermission, TeamMember, Namespace
@ -87,7 +87,7 @@ def test_filter_repositories(username, include_public, filter_to_namespace, repo
.distinct()
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER))
.join(RepositoryPermission, JOIN.LEFT_OUTER))
with assert_query_count(1):
found = list(filter_to_repos_for_user(query, user.id,

1
data/model/test/test_repository.py
View file

@ -8,6 +8,7 @@ from data.model.repository import create_repository, purge_repository, is_empty
from data.model.repository import get_filtered_matching_repositories
from test.fixtures import *
def test_duplicate_repository_different_kinds(initialized_db):
# Create an image repo.
create_repository('devtable', 'somenewrepo', None, repo_kind='image')

4
data/model/token.py
View file

@ -1,6 +1,6 @@
import logging
from peewee import JOIN_LEFT_OUTER
from peewee import JOIN
from data.database import (AccessToken, AccessTokenKind, Repository, Namespace, Role,
RepositoryBuildTrigger, LogEntryKind)
@ -38,7 +38,7 @@ def get_repository_delegate_tokens(namespace_name, repository_name):
.switch(AccessToken)
.join(Role)
.switch(AccessToken)
.join(RepositoryBuildTrigger, JOIN_LEFT_OUTER)
.join(RepositoryBuildTrigger, JOIN.LEFT_OUTER)
.where(Repository.name == repository_name, Namespace.username == namespace_name,
AccessToken.temporary == False, RepositoryBuildTrigger.uuid >> None))

37
data/model/user.py
View file

@ -4,7 +4,7 @@ import json
import uuid
from flask_login import UserMixin
from peewee import JOIN_LEFT_OUTER, IntegrityError, fn
from peewee import JOIN, IntegrityError, fn
from uuid import uuid4
from datetime import datetime, timedelta
from enum import Enum
@ -397,15 +397,18 @@ def _list_entity_robots(entity_name, include_metadata=True):
""" Return the list of robots for the specified entity. This MUST return a query, not a
materialized list so that callers can use db_for_update.
"""
query = (User
.select(User, FederatedLogin)
.join(FederatedLogin)
.where(User.robot == True, User.username ** (entity_name + '+%')))
if include_metadata:
query = (query.switch(User)
.join(RobotAccountMetadata, JOIN_LEFT_OUTER)
.select(User, FederatedLogin, RobotAccountMetadata))
query = (User
.select(User, FederatedLogin, RobotAccountMetadata)
.join(FederatedLogin)
.switch(User)
.join(RobotAccountMetadata, JOIN.LEFT_OUTER)
.where(User.robot == True, User.username ** (entity_name + '+%')))
else:
query = (User
.select(User, FederatedLogin)
.join(FederatedLogin)
.where(User.robot == True, User.username ** (entity_name + '+%')))
return query
@ -417,12 +420,12 @@ def list_entity_robot_permission_teams(entity_name, limit=None, include_permissi
RobotAccountMetadata.description, RobotAccountMetadata.unstructured_json]
if include_permissions:
query = (query
.join(RepositoryPermission, JOIN_LEFT_OUTER,
.join(RepositoryPermission, JOIN.LEFT_OUTER,
on=(RepositoryPermission.user == FederatedLogin.user))
.join(Repository, JOIN_LEFT_OUTER)
.join(Repository, JOIN.LEFT_OUTER)
.switch(User)
.join(TeamMember, JOIN_LEFT_OUTER)
.join(Team, JOIN_LEFT_OUTER))
.join(TeamMember, JOIN.LEFT_OUTER)
.join(Team, JOIN.LEFT_OUTER))
fields.append(Repository.name)
fields.append(Team.name)
@ -684,7 +687,7 @@ def get_matching_user_namespaces(namespace_prefix, username, limit=10):
.select()
.distinct()
.join(Repository, on=(Repository.namespace_user == Namespace.id))
.join(RepositoryPermission, JOIN_LEFT_OUTER)
.join(RepositoryPermission, JOIN.LEFT_OUTER)
.where(namespace_search))
return _basequery.filter_to_repos_for_user(base_query, namespace_user_id).limit(limit)
@ -710,8 +713,8 @@ def get_matching_users(username_prefix, robot_namespace=None, organization=None,
if organization:
query = (query
.select(User.id, User.username, User.email, User.robot, fn.Sum(Team.id))
.join(TeamMember, JOIN_LEFT_OUTER)
.join(Team, JOIN_LEFT_OUTER, on=((Team.id == TeamMember.team) &
.join(TeamMember, JOIN.LEFT_OUTER)
.join(Team, JOIN.LEFT_OUTER, on=((Team.id == TeamMember.team) &
(Team.organization == organization)))
.order_by(User.robot.desc()))
@ -790,7 +793,7 @@ def verify_user(username_or_email, password):
def get_all_repo_users(namespace_name, repository_name):
return (RepositoryPermission
.select(User.username, User.email, User.robot, Role.name, RepositoryPermission)
.select(User, Role, RepositoryPermission)
.join(User)
.switch(RepositoryPermission)
.join(Role)

11
data/text.py
View file

@ -1,4 +1,4 @@
from peewee import Clause, SQL, fn, TextField, Field
from peewee import NodeList, SQL, fn, TextField, Field
def _escape_wildcard(search_query):
""" Escapes the wildcards found in the given search query so that they are treated as *characters*
@ -16,7 +16,7 @@ def prefix_search(field, prefix_query):
""" Returns the wildcard match for searching for the given prefix query. """
# Escape the known wildcard characters.
prefix_query = _escape_wildcard(prefix_query)
return Field.__pow__(field, Clause(prefix_query + '%', SQL("ESCAPE '!'")))
return Field.__pow__(field, NodeList((prefix_query + '%', SQL("ESCAPE '!'"))))
def match_mysql(field, search_query):
@ -29,14 +29,13 @@ def match_mysql(field, search_query):
# queries of the form `*` to raise a parsing error. If found, simply filter out.
search_query = search_query.replace('*', '')
return Clause(fn.MATCH(SQL("`%s`" % field.name)), fn.AGAINST(SQL('%s', search_query)),
parens=True)
return NodeList((fn.MATCH(SQL("`%s`" % field.name)), fn.AGAINST(SQL('%s', [search_query]))),
parens=True)
def match_like(field, search_query):
""" Generates a full-text match query using an ILIKE operation, which is needed for SQLite and
Postgres.
"""
escaped_query = _escape_wildcard(search_query)
clause = Clause('%' + escaped_query + '%', SQL("ESCAPE '!'"))
clause = NodeList(('%' + escaped_query + '%', SQL("ESCAPE '!'")))
return Field.__pow__(field, clause)

15
initdb.py
View file

@ -7,8 +7,7 @@ import os
import argparse
from datetime import datetime, timedelta, date
from peewee import (SqliteDatabase, create_model_tables, drop_model_tables, savepoint_sqlite,
savepoint)
from peewee import SqliteDatabase
from itertools import count
from uuid import UUID, uuid4
from threading import Event
@ -210,6 +209,7 @@ def finished_database_for_testing(testcase):
any changes should be discarded.
"""
testcases[testcase]['savepoint'].__exit__(True, None, None)
testcases[testcase]['atomic'].__exit__(True, None, None)
def setup_database_for_testing(testcase, with_storage=False, force_rebuild=False):
""" Called when a testcase has started using the database, indicating that
@ -242,15 +242,16 @@ def setup_database_for_testing(testcase, with_storage=False, force_rebuild=False
db_initialized_for_testing.set()
# Create a savepoint for the testcase.
test_savepoint = savepoint(db) if IS_TESTING_REAL_DATABASE else savepoint_sqlite(db)
testcases[testcase] = {}
testcases[testcase]['savepoint'] = test_savepoint
testcases[testcase]['atomic'] = db.manual_commit()
testcases[testcase]['atomic'].__enter__()
testcases[testcase]['savepoint'] = db.savepoint()
testcases[testcase]['savepoint'].__enter__()
def initialize_database():
create_model_tables(all_models)
db.create_tables(all_models)
Role.create(name='admin')
Role.create(name='write')
@ -447,7 +448,7 @@ def wipe_database():
if not IS_TESTING_REAL_DATABASE and not isinstance(db.obj, SqliteDatabase):
raise RuntimeError('Attempted to wipe production database!')
drop_model_tables(all_models, fail_silently=True)
db.drop_tables(all_models)
def populate_database(minimal=False, with_storage=False):

4
requirements-nover.txt
View file

@ -45,7 +45,7 @@ moto==0.4.25 # remove when 0.4.28+ is out
namedlist
netaddr
pathvalidate
peewee==2.8.1
peewee
psutil
psycopg2
pyasn1
@ -78,4 +78,4 @@ xhtml2pdf
recaptcha2
mockredispy
yapf
pymemcache
pymemcache

88
requirements.txt
View file

@ -1,5 +1,5 @@
aiowsgi==0.6
alembic==0.9.8
alembic==0.9.9
-e git+https://github.com/coreos/mockldap.git@59a46efbe8c7cd8146a87a7c4f2b09746b953e11#egg=mockldap
-e git+https://github.com/coreos/py-bitbucket.git@55a1ada645f2fb6369147996ec71edd7828d91c8#egg=py_bitbucket
-e git+https://github.com/coreos/resumablehashlib.git@b1b631249589b07adf40e0ee545b323a501340b4#egg=resumablehashlib
@ -9,12 +9,12 @@ alembic==0.9.8
-e git+https://github.com/DevTable/container-cloud-config.git@8e8ae177c3d5cd6608f64250fcd8770022d61562#egg=container_cloud_config
-e git+https://github.com/DevTable/python-etcd.git@f1168cb02a2a8c83bec1108c6fcd8615ef463b14#egg=python_etcd
-e git+https://github.com/jarus/flask-testing.git@17f19d7fee0e1e176703fc7cb04917a77913ba1a#egg=Flask_Testing
-e git+https://github.com/jepcastelein/marketo-rest-python.git@6134d1129f2435b313c4301503a4b74974d79a42#egg=marketorestpython
-e git+https://github.com/jepcastelein/marketo-rest-python.git@9eaad572b52a30bbb1dee17821acb197ecd2dcb8#egg=marketorestpython
-e git+https://github.com/NateFerrero/oauth2lib.git@d161b010f8a596826050a09e5e94d59443cc12d9#egg=oauth2lib
APScheduler==3.0.5
asn1crypto==0.24.0
autobahn==0.9.3.post3
azure-common==1.1.8
azure-common==1.1.10
azure-nspkg==2.0.0
azure-storage-blob==1.1.0
azure-storage-common==1.1.0
@ -27,31 +27,31 @@ bintrees==2.0.7
bitmath==1.3.1.2
blinker==1.4
boto==2.48.0
boto3==1.5.36
botocore==1.8.50
boto3==1.7.12
botocore==1.10.12
cachetools==1.1.6
certifi==2018.1.18
cffi==1.11.4
certifi==2018.4.16
cffi==1.11.5
chardet==3.0.4
click==6.7
contextlib2==0.5.5
cryptography==2.1.4
cryptography==2.2.2
debtcollector==1.19.0
decorator==4.2.1
decorator==4.3.0
enum34==1.1.6
Flask-Cors==3.0.3
Flask-Cors==3.0.4
Flask-Login==0.4.1
Flask-Mail==0.9.1
Flask-Principal==0.4.0
Flask-RESTful==0.3.6
Flask==0.12.2
Flask==1.0.2
funcparserlib==0.3.6
funcsigs==1.0.2
functools32==3.2.3.post2
furl==1.0.1
future==0.16.0
futures==3.2.0
geoip2==2.7.0
geoip2==2.8.0
gevent==1.2.2
gipc==0.6.0
greenlet==0.4.13
@ -59,19 +59,19 @@ gunicorn==18.0
hiredis==0.2.0
html5lib==0.9999999
httmock==1.2.6
httplib2==0.10.3
httplib2==0.11.3
httpretty==0.8.10
idna==2.6
ipaddress==1.0.19
ipaddress==1.0.22
iso8601==0.1.12
itsdangerous==0.24
Jinja2==2.10
jmespath==0.9.3
jsonpath-rw==1.4.0
jsonschema==2.6.0
keystoneauth1==3.4.0
keystoneauth1==3.5.0
Mako==1.0.7
marisa-trie==0.7.4
marisa-trie==0.7.5
MarkupSafe==1.0
maxminddb==1.3.0
mixpanel==4.3.2
@ -81,46 +81,47 @@ monotonic==1.4
moto==0.4.25
msgpack==0.5.6
namedlist==1.7
ndg-httpsclient==0.4.4
ndg-httpsclient==0.5.0
netaddr==0.7.19
netifaces==0.10.6
oauthlib==2.0.6
netifaces==0.10.7
oauthlib==2.0.7
orderedmultidict==0.7.11
oslo.config==5.2.0
oslo.i18n==3.19.0
oslo.serialization==2.24.0
oslo.utils==3.35.0
pathvalidate==0.16.3
pbr==3.1.1
peewee==2.8.1
Pillow==5.0.0
oslo.config==6.2.0
oslo.i18n==3.20.0
oslo.serialization==2.25.0
oslo.utils==3.36.1
pathvalidate==0.17.1
pbr==4.0.2
peewee==3.3.3
Pillow==5.1.0
ply==3.11
psutil==5.4.3
psutil==5.4.5
psycopg2==2.7.4
pyasn1==0.4.2
pyasn1-modules==0.2.1
py-bcrypt==0.4
pycparser==2.18
pycryptodome==3.4.11
pycryptodomex==3.4.12
PyGithub==1.36
pycryptodome==3.6.1
pycryptodomex==3.6.1
PyGithub==1.39
pygpgme==0.3
pyjwkest==1.4.0
PyJWT==1.5.3
PyJWT==1.6.1
PyMySQL==0.6.7
pyOpenSSL==17.5.0
pymemcache==1.4.3
pymemcache==1.4.4
pyparsing==2.2.0
PyPDF2==1.26.0
python-dateutil==2.6.1
python-dateutil==2.7.2
python-editor==1.0.3
python-gitlab==1.4.0
python-keystoneclient==3.15.0
python-ldap==2.5.2
python-keystoneclient==3.16.0
python-ldap==3.0.0
python-magic==0.4.15
python-swiftclient==3.5.0
pytz==2018.3
pytz==2018.4
PyYAML==3.12
raven==6.5.0
raven==6.7.0
recaptcha2==0.1
redis==2.10.6
redlock==1.2.0
@ -130,20 +131,19 @@ requests[security]==2.18.4
rfc3986==1.1.0
s3transfer==0.1.13
semantic-version==2.6.0
simplejson==3.13.2
six==1.11.0
SQLAlchemy==1.1.5
stevedore==1.28.0
stringscore==0.1.0
stripe==1.79.0
stripe==1.80.0
toposort==1.5
trollius==2.1
trollius==2.2
tzlocal==1.5.1
urllib3==1.22
waitress==1.1.0
WebOb==1.7.4
WebOb==1.8.1
Werkzeug==0.14.1
wrapt==1.10.11
xhtml2pdf==0.2.1
xhtml2pdf==0.2.2
xmltodict==0.11.0
yapf==0.20.2
yapf==0.21.0

2
storage/test/test_storageproxy.py
View file

@ -92,4 +92,4 @@ def test_storage_proxy_auth(storage, liveserver_app, liveserver_session, is_prox
}
resp = liveserver_session.get('_storage_proxy_auth', headers=headers)
assert resp.status_code == (404 if not is_proxying_enabled else 200)
assert resp.status_code == (500 if not is_proxying_enabled else 200)

25
test/fixtures.py
View file

@ -8,7 +8,7 @@ from flask import Flask, jsonify
from flask_login import LoginManager
from flask_principal import identity_loaded, Permission, Identity, identity_changed, Principal
from flask_mail import Mail
from peewee import SqliteDatabase, savepoint, InternalError
from peewee import SqliteDatabase, InternalError
from app import app as application
from auth.permissions import on_identity_loaded
@ -147,19 +147,20 @@ def initialized_db(appconfig):
# If under a test *real* database, setup a savepoint.
under_test_real_database = bool(os.environ.get('TEST_DATABASE_URI'))
if under_test_real_database:
test_savepoint = savepoint(db)
test_savepoint.__enter__()
with db.transaction():
test_savepoint = db.savepoint()
test_savepoint.__enter__()
yield # Run the test.
yield # Run the test.
try:
test_savepoint.rollback()
test_savepoint.__exit__(None, None, None)
except InternalError:
# If postgres fails with an exception (like IntegrityError) mid-transaction, it terminates
# it immediately, so when we go to remove the savepoint, it complains. We can safely ignore
# this case.
pass
try:
test_savepoint.rollback()
test_savepoint.__exit__(None, None, None)
except InternalError:
# If postgres fails with an exception (like IntegrityError) mid-transaction, it terminates
# it immediately, so when we go to remove the savepoint, it complains. We can safely ignore
# this case.
pass
else:
yield

2
test/test_endpoints.py
View file

@ -161,7 +161,7 @@ class BuildLogsTestCase(EndpointTestCase):
logs = ['log1', 'log2']
with patch('endpoints.web.build_logs.get_log_entries', return_value=(None, logs) ):
resp = self.getResponse('web.buildlogs', build_uuid=self.build_uuid, expected_code=200)
self.assertEquals('{"logs":["log1","log2"]}\n', resp)
self.assertEquals({"logs": logs}, py_json.loads(resp))
class ArchivedLogsTestCase(EndpointTestCase):
build_uuid = 'deadpork-dead-pork-dead-porkdeadpork'

4
tools/orphans.py
View file

@ -1,11 +1,11 @@
from data.database import Image, ImageStorage
from peewee import JOIN_LEFT_OUTER, fn
from peewee import JOIN, fn
from app import app
orphaned = (ImageStorage
.select()
.where(ImageStorage.uploading == False)
.join(Image, JOIN_LEFT_OUTER)
.join(Image, JOIN.LEFT_OUTER)
.group_by(ImageStorage)
.having(fn.Count(Image.id) == 0))