vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Upgrade Peewee to latest 3.x

Browse source 
This requires a number of small changes in the data model code, as well as additional testing.
This commit is contained in:
Brad Ison 2018-04-06 13:48:01 -04:00 • committed by Joseph Schorr
parent 70b7ee4654
commit d3d9cca182
26 changed files with 220 additions and 193 deletions
Download patch file Download diff file Expand all files Collapse all files

7
data/model/_basequery.py
View file

@ -94,9 +94,7 @@ def filter_to_repos_for_user(query, user_id=None, namespace=None, repo_kind='ima
queries = []
if include_public:
queries.append(query
.clone()
.where(Repository.visibility == get_public_repo_visibility()))
queries.append(query.where(Repository.visibility == get_public_repo_visibility()))
if user_id is not None:
AdminTeam = Team.alias()
@ -104,13 +102,11 @@ def filter_to_repos_for_user(query, user_id=None, namespace=None, repo_kind='ima
# Add repositories in which the user has permission.
queries.append(query
.clone()
.switch(RepositoryPermission)
.where(RepositoryPermission.user == user_id))
# Add repositories in which the user is a member of a team that has permission.
queries.append(query
.clone()
.switch(RepositoryPermission)
.join(Team)
.join(TeamMember)
@ -118,7 +114,6 @@ def filter_to_repos_for_user(query, user_id=None, namespace=None, repo_kind='ima
# Add repositories under namespaces in which the user is the org admin.
queries.append(query
.clone()
.switch(Repository)
.join(AdminTeam, on=(Repository.namespace_user == AdminTeam.organization))
.join(AdminTeamMember, on=(AdminTeam.id == AdminTeamMember.team))

11
data/model/build.py
View file

@ -2,7 +2,7 @@ import json
import os
from datetime import timedelta, datetime
from peewee import JOIN_LEFT_OUTER
from peewee import JOIN
import features
from data.database import (BuildTriggerService, RepositoryBuildTrigger, Repository, Namespace, User,
@ -50,7 +50,7 @@ def get_build_trigger(trigger_uuid):
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(RepositoryBuildTrigger)
.join(User)
.join(User, on=(RepositoryBuildTrigger.connected_user == User.id))
.where(RepositoryBuildTrigger.uuid == trigger_uuid)
.get())
except RepositoryBuildTrigger.DoesNotExist:
@ -94,10 +94,10 @@ def _get_build_base_query():
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(RepositoryBuild)
.join(User, JOIN_LEFT_OUTER)
.join(User, JOIN.LEFT_OUTER)
.switch(RepositoryBuild)
.join(RepositoryBuildTrigger, JOIN_LEFT_OUTER)
.join(BuildTriggerService, JOIN_LEFT_OUTER)
.join(RepositoryBuildTrigger, JOIN.LEFT_OUTER)
.join(BuildTriggerService, JOIN.LEFT_OUTER)
.order_by(RepositoryBuild.started.desc()))
@ -308,4 +308,3 @@ def update_trigger_disable_status(trigger, final_phase):
else:
# Save the trigger changes.
trigger.save()

8
data/model/image.py
View file

@ -6,7 +6,7 @@ from collections import defaultdict
from datetime import datetime
import dateutil.parser
from peewee import JOIN_LEFT_OUTER, IntegrityError, fn
from peewee import JOIN, IntegrityError, fn
from data.model import (DataModelException, db_transaction, _basequery, storage,
InvalidImageException)
@ -273,7 +273,7 @@ def find_create_or_link_image(docker_image_id, repo_obj, username, translations,
.join(ImageStorage)
.switch(Image)
.join(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER)
.join(RepositoryPermission, JOIN.LEFT_OUTER)
.switch(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(ImageStorage.uploading == False,
@ -445,8 +445,8 @@ def get_image_with_storage_and_parent_base():
.select(Image, ImageStorage, Parent, ParentImageStorage)
.join(ImageStorage)
.switch(Image)
.join(Parent, JOIN_LEFT_OUTER, on=(Image.parent == Parent.id))
.join(ParentImageStorage, JOIN_LEFT_OUTER, on=(ParentImageStorage.id == Parent.storage)))
.join(Parent, JOIN.LEFT_OUTER, on=(Image.parent == Parent.id))
.join(ParentImageStorage, JOIN.LEFT_OUTER, on=(ParentImageStorage.id == Parent.storage)))
def set_secscan_status(image, indexed, version):

6
data/model/log.py
View file

@ -2,7 +2,7 @@ import json
import logging
from calendar import timegm
from peewee import JOIN_LEFT_OUTER, fn, PeeweeException
from peewee import JOIN, fn, PeeweeException
from datetime import datetime, timedelta
from cachetools import lru_cache
@ -82,11 +82,11 @@ def get_logs_query(start_time, end_time, performer=None, repository=None, namesp
query = _logs_query(selections, start_time, end_time, performer, repository, namespace, ignore,
model=model)
query = (query.switch(model).join(Performer, JOIN_LEFT_OUTER,
query = (query.switch(model).join(Performer, JOIN.LEFT_OUTER,
on=(model.performer == Performer.id).alias('performer')))
if namespace is None and repository is None:
query = (query.switch(model).join(Account, JOIN_LEFT_OUTER,
query = (query.switch(model).join(Account, JOIN.LEFT_OUTER,
on=(model.account == Account.id).alias('account')))
return query

14
data/model/notification.py
View file

@ -37,7 +37,16 @@ def lookup_notifications_by_path_prefix(prefix):
def list_notifications(user, kind_name=None, id_filter=None, include_dismissed=False,
page=None, limit=None):
base_query = Notification.select().join(NotificationKind)
base_query = (Notification
.select(Notification.id,
Notification.uuid,
Notification.kind,
Notification.metadata_json,
Notification.dismissed,
Notification.lookup_path,
Notification.created.alias('cd'),
Notification.target)
.join(NotificationKind))
if kind_name is not None:
base_query = base_query.where(NotificationKind.name == kind_name)
@ -73,7 +82,8 @@ def list_notifications(user, kind_name=None, id_filter=None, include_dismissed=F
elif limit:
query = query.limit(limit)
return query.order_by(base_query.c.created.desc())
from peewee import SQL
return query.order_by(SQL('cd desc'))
def delete_all_notifications_by_path_prefix(prefix):

10
data/model/permission.py
View file

@ -1,4 +1,4 @@
from peewee import JOIN_LEFT_OUTER
from peewee import JOIN
from data.database import (RepositoryPermission, User, Repository, Visibility, Role, TeamMember,
PermissionPrototype, Team, TeamRole, Namespace)
@ -112,13 +112,13 @@ def get_prototype_permissions(org):
query = (PermissionPrototype
.select()
.where(PermissionPrototype.org == org)
.join(ActivatingUser, JOIN_LEFT_OUTER,
.join(ActivatingUser, JOIN.LEFT_OUTER,
on=(ActivatingUser.id == PermissionPrototype.activating_user))
.join(DelegateUser, JOIN_LEFT_OUTER,
.join(DelegateUser, JOIN.LEFT_OUTER,
on=(DelegateUser.id == PermissionPrototype.delegate_user))
.join(Team, JOIN_LEFT_OUTER,
.join(Team, JOIN.LEFT_OUTER,
on=(Team.id == PermissionPrototype.delegate_team))
.join(Role, JOIN_LEFT_OUTER, on=(Role.id == PermissionPrototype.role)))
.join(Role, JOIN.LEFT_OUTER, on=(Role.id == PermissionPrototype.role)))
return query

7
data/model/repository.py
View file

@ -3,8 +3,7 @@ import random
from enum import Enum
from datetime import timedelta, datetime
from peewee import JOIN_LEFT_OUTER, fn, SQL, IntegrityError
from playhouse.shortcuts import case
from peewee import Case, JOIN, fn, SQL, IntegrityError
from cachetools import ttl_cache
from data.model import (
@ -406,7 +405,7 @@ def get_visible_repositories(username, namespace=None, kind_filter='image', incl
user_id = None
if username:
# Note: We only need the permissions table if we will filter based on a user's permissions.
query = query.switch(Repository).distinct().join(RepositoryPermission, JOIN_LEFT_OUTER)
query = query.switch(Repository).distinct().join(RepositoryPermission, JOIN.LEFT_OUTER)
found_namespace = _get_namespace_user(username)
if not found_namespace:
return Repository.select(Repository.id.alias('rid')).where(Repository.id == -1)
@ -552,7 +551,7 @@ def _get_sorted_matching_repositories(lookup_value, repo_kind='image', include_p
if SEARCH_FIELDS.description.name in search_fields:
clause = Repository.description.match(lookup_value) | clause
cases = [(Repository.name.match(lookup_value), 100 * RepositorySearchScore.score),]
computed_score = case(None, cases, RepositorySearchScore.score).alias('score')
computed_score = Case(None, cases, RepositorySearchScore.score).alias('score')
select_fields.append(computed_score)
query = (Repository.select(*select_fields)

4
data/model/service_keys.py
View file

@ -2,7 +2,7 @@ import re
from calendar import timegm
from datetime import datetime, timedelta
from peewee import JOIN_LEFT_OUTER
from peewee import JOIN
from Crypto.PublicKey import RSA
from jwkest.jwk import RSAKey
@ -165,7 +165,7 @@ def approve_service_key(kid, approver, approval_type, notes=''):
def _list_service_keys_query(kid=None, service=None, approved_only=True, alive_only=True,
approval_type=None):
query = ServiceKey.select().join(ServiceKeyApproval, JOIN_LEFT_OUTER)
query = ServiceKey.select().join(ServiceKeyApproval, JOIN.LEFT_OUTER)
if approved_only:
query = query.where(~(ServiceKey.approval >> None))

13
data/model/sqlalchemybridge.py
View file

@ -50,8 +50,7 @@ def gen_sqlalchemy_metadata(peewee_model_list):
alchemy_type = Integer
all_indexes.add(((field.name, ), field.unique))
if not field.deferred:
target_name = '%s.%s' % (field.to_field.model_class._meta.db_table,
field.to_field.db_column)
target_name = '%s.%s' % (field.rel_model._meta.table_name, field.rel_field.column_name)
col_args.append(ForeignKey(target_name))
elif isinstance(field, BigIntegerField):
alchemy_type = BigInteger
@ -74,19 +73,19 @@ def gen_sqlalchemy_metadata(peewee_model_list):
if field.unique or field.index:
all_indexes.add(((field.name, ), field.unique))
new_col = Column(field.db_column, alchemy_type, *col_args, **col_kwargs)
new_col = Column(field.column_name, alchemy_type, *col_args, **col_kwargs)
columns.append(new_col)
new_table = Table(meta.db_table, metadata, *columns)
new_table = Table(meta.table_name, metadata, *columns)
for col_prop_names, unique in all_indexes:
col_names = [meta.fields[prop_name].db_column for prop_name in col_prop_names]
index_name = '%s_%s' % (meta.db_table, '_'.join(col_names))
col_names = [meta.fields[prop_name].column_name for prop_name in col_prop_names]
index_name = '%s_%s' % (meta.table_name, '_'.join(col_names))
col_refs = [getattr(new_table.c, col_name) for col_name in col_names]
Index(index_name, *col_refs, unique=unique)
for col_field_name in fulltext_indexes:
index_name = '%s_%s__fulltext' % (meta.db_table, col_field_name)
index_name = '%s_%s__fulltext' % (meta.table_name, col_field_name)
col_ref = getattr(new_table.c, col_field_name)
Index(index_name, col_ref, postgresql_ops={col_field_name: 'gin_trgm_ops'},
postgresql_using='gin',

15
data/model/tag.py
View file

@ -4,7 +4,7 @@ import time
from calendar import timegm
from uuid import uuid4
from peewee import IntegrityError, JOIN_LEFT_OUTER, fn
from peewee import IntegrityError, JOIN, fn
from data.model import (image, db_transaction, DataModelException, _basequery,
InvalidManifestException, TagAlreadyCreatedException, StaleTagException,
config)
@ -44,8 +44,8 @@ def get_tags_images_eligible_for_scan(clair_version):
.join(Image, on=(RepositoryTag.image == Image.id))
.join(ImageStorage, on=(Image.storage == ImageStorage.id))
.switch(Image)
.join(Parent, JOIN_LEFT_OUTER, on=(Image.parent == Parent.id))
.join(ParentImageStorage, JOIN_LEFT_OUTER, on=(ParentImageStorage.id == Parent.storage))
.join(Parent, JOIN.LEFT_OUTER, on=(Image.parent == Parent.id))
.join(ParentImageStorage, JOIN.LEFT_OUTER, on=(ParentImageStorage.id == Parent.storage))
.where(RepositoryTag.hidden == False)
.where(Image.security_indexed_engine < clair_version))
@ -71,7 +71,7 @@ def filter_tags_have_repository_event(query, event):
lifetime_start_ts.
"""
query = filter_has_repository_event(query, event)
query = query.switch(Image).join(ImageStorage)
query = query.switch(RepositoryTag).join(Image).join(ImageStorage)
query = query.switch(RepositoryTag).order_by(RepositoryTag.lifetime_start_ts.desc())
return query
@ -146,12 +146,13 @@ def get_matching_tags_for_images(image_pairs, filter_images=None, filter_tags=No
# Collect IDs of the tags found for each query.
tags = {}
for query in sharded_queries:
ImageAlias = Image.alias()
tag_query = (_tag_alive(RepositoryTag
.select(*(selections or []))
.distinct()
.join(Image)
.join(ImageAlias)
.where(RepositoryTag.hidden == False)
.where(Image.id << query)
.where(ImageAlias.id << query)
.switch(RepositoryTag)))
if filter_tags is not None:
@ -210,7 +211,7 @@ def list_active_repo_tags(repo):
.join(Image)
.where(RepositoryTag.repository == repo, RepositoryTag.hidden == False)
.switch(RepositoryTag)
.join(TagManifest, JOIN_LEFT_OUTER))
.join(TagManifest, JOIN.LEFT_OUTER))
return query

11
data/model/team.py
View file

@ -148,12 +148,11 @@ def add_or_invite_to_team(inviter, team, user_obj=None, email=None, requires_inv
def get_matching_user_teams(team_prefix, user_obj, limit=10):
team_prefix_search = prefix_search(Team.name, team_prefix)
query = (Team
.select()
.select(Team.id.distinct(), Team)
.join(User)
.switch(Team)
.join(TeamMember)
.where(TeamMember.user == user_obj, team_prefix_search)
.distinct(Team.id)
.limit(limit))
return query
@ -179,12 +178,11 @@ def get_matching_admined_teams(team_prefix, user_obj, limit=10):
.where(TeamRole.name == 'admin'))
query = (Team
.select()
.select(Team.id.distinct(), Team)
.join(User)
.switch(Team)
.join(TeamMember)
.where(team_prefix_search, Team.organization << (admined_orgs))
.distinct(Team.id)
.limit(limit))
return query
@ -260,8 +258,9 @@ def get_user_teams_within_org(username, organization):
def list_organization_members_by_teams(organization):
query = (TeamMember
.select(Team, User)
.annotate(Team)
.annotate(User)
.join(Team)
.switch(TeamMember)
.join(User)
.where(Team.organization == organization))
return query

4
data/model/test/test_basequery.py
View file

@ -1,6 +1,6 @@
import pytest
from peewee import JOIN_LEFT_OUTER
from peewee import JOIN
from playhouse.test_utils import assert_query_count
from data.database import Repository, RepositoryPermission, TeamMember, Namespace
@ -87,7 +87,7 @@ def test_filter_repositories(username, include_public, filter_to_namespace, repo
.distinct()
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER))
.join(RepositoryPermission, JOIN.LEFT_OUTER))
with assert_query_count(1):
found = list(filter_to_repos_for_user(query, user.id,

1
data/model/test/test_repository.py
View file

@ -8,6 +8,7 @@ from data.model.repository import create_repository, purge_repository, is_empty
from data.model.repository import get_filtered_matching_repositories
from test.fixtures import *
def test_duplicate_repository_different_kinds(initialized_db):
# Create an image repo.
create_repository('devtable', 'somenewrepo', None, repo_kind='image')

4
data/model/token.py
View file

@ -1,6 +1,6 @@
import logging
from peewee import JOIN_LEFT_OUTER
from peewee import JOIN
from data.database import (AccessToken, AccessTokenKind, Repository, Namespace, Role,
RepositoryBuildTrigger, LogEntryKind)
@ -38,7 +38,7 @@ def get_repository_delegate_tokens(namespace_name, repository_name):
.switch(AccessToken)
.join(Role)
.switch(AccessToken)
.join(RepositoryBuildTrigger, JOIN_LEFT_OUTER)
.join(RepositoryBuildTrigger, JOIN.LEFT_OUTER)
.where(Repository.name == repository_name, Namespace.username == namespace_name,
AccessToken.temporary == False, RepositoryBuildTrigger.uuid >> None))

37
data/model/user.py
View file

@ -4,7 +4,7 @@ import json
import uuid
from flask_login import UserMixin
from peewee import JOIN_LEFT_OUTER, IntegrityError, fn
from peewee import JOIN, IntegrityError, fn
from uuid import uuid4
from datetime import datetime, timedelta
from enum import Enum
@ -397,15 +397,18 @@ def _list_entity_robots(entity_name, include_metadata=True):
""" Return the list of robots for the specified entity. This MUST return a query, not a
materialized list so that callers can use db_for_update.
"""
query = (User
.select(User, FederatedLogin)
.join(FederatedLogin)
.where(User.robot == True, User.username ** (entity_name + '+%')))
if include_metadata:
query = (query.switch(User)
.join(RobotAccountMetadata, JOIN_LEFT_OUTER)
.select(User, FederatedLogin, RobotAccountMetadata))
query = (User
.select(User, FederatedLogin, RobotAccountMetadata)
.join(FederatedLogin)
.switch(User)
.join(RobotAccountMetadata, JOIN.LEFT_OUTER)
.where(User.robot == True, User.username ** (entity_name + '+%')))
else:
query = (User
.select(User, FederatedLogin)
.join(FederatedLogin)
.where(User.robot == True, User.username ** (entity_name + '+%')))
return query
@ -417,12 +420,12 @@ def list_entity_robot_permission_teams(entity_name, limit=None, include_permissi
RobotAccountMetadata.description, RobotAccountMetadata.unstructured_json]
if include_permissions:
query = (query
.join(RepositoryPermission, JOIN_LEFT_OUTER,
.join(RepositoryPermission, JOIN.LEFT_OUTER,
on=(RepositoryPermission.user == FederatedLogin.user))
.join(Repository, JOIN_LEFT_OUTER)
.join(Repository, JOIN.LEFT_OUTER)
.switch(User)
.join(TeamMember, JOIN_LEFT_OUTER)
.join(Team, JOIN_LEFT_OUTER))
.join(TeamMember, JOIN.LEFT_OUTER)
.join(Team, JOIN.LEFT_OUTER))
fields.append(Repository.name)
fields.append(Team.name)
@ -684,7 +687,7 @@ def get_matching_user_namespaces(namespace_prefix, username, limit=10):
.select()
.distinct()
.join(Repository, on=(Repository.namespace_user == Namespace.id))
.join(RepositoryPermission, JOIN_LEFT_OUTER)
.join(RepositoryPermission, JOIN.LEFT_OUTER)
.where(namespace_search))
return _basequery.filter_to_repos_for_user(base_query, namespace_user_id).limit(limit)
@ -710,8 +713,8 @@ def get_matching_users(username_prefix, robot_namespace=None, organization=None,
if organization:
query = (query
.select(User.id, User.username, User.email, User.robot, fn.Sum(Team.id))
.join(TeamMember, JOIN_LEFT_OUTER)
.join(Team, JOIN_LEFT_OUTER, on=((Team.id == TeamMember.team) &
.join(TeamMember, JOIN.LEFT_OUTER)
.join(Team, JOIN.LEFT_OUTER, on=((Team.id == TeamMember.team) &
(Team.organization == organization)))
.order_by(User.robot.desc()))
@ -790,7 +793,7 @@ def verify_user(username_or_email, password):
def get_all_repo_users(namespace_name, repository_name):
return (RepositoryPermission
.select(User.username, User.email, User.robot, Role.name, RepositoryPermission)
.select(User, Role, RepositoryPermission)
.join(User)
.switch(RepositoryPermission)
.join(Role)