vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Require some data from all models in initdb

Browse source
This commit is contained in:
Jake Moshenko 2016-01-12 15:57:03 -05:00
parent 1b392dcb9a
commit fe2bdeb6cb
5 changed files with 69 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
data/database.py
View file

@ -4,6 +4,8 @@ import uuid
import time
import toposort
import resumablehashlib
import sys
import inspect
from random import SystemRandom
from datetime import datetime
@ -861,15 +863,5 @@ class TorrentInfo(BaseModel):
(('storage', 'piece_length'), True),
)
all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, Visibility,
RepositoryTag, EmailConfirmation, FederatedLogin, LoginService, QueueItem,
RepositoryBuild, Team, TeamMember, TeamRole, LogEntryKind, LogEntry,
PermissionPrototype, ImageStorage, BuildTriggerService, RepositoryBuildTrigger,
OAuthApplication, OAuthAuthorizationCode, OAuthAccessToken, NotificationKind,
Notification, ImageStorageLocation, ImageStoragePlacement,
ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification,
RepositoryAuthorizedEmail, ImageStorageTransformation,
TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind,
AccessTokenKind, Star, RepositoryActionCount, TagManifest, UserRegion,
QuayService, QuayRegion, QuayRelease, BlobUpload, DerivedStorageForImage, TorrentInfo]
is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel
all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)]

2
data/model/__init__.py
View file

@ -95,4 +95,4 @@ config = Config()
# moving the minimal number of things to _basequery
# TODO document the methods and modules for each one of the submodules below.
from data.model import (blob, build, image, log, notification, oauth, organization, permission,
repository, storage, tag, team, token, user)
repository, storage, tag, team, token, user, release)

57
initdb.py
View file

@ -10,16 +10,18 @@ from datetime import datetime, timedelta
from peewee import (SqliteDatabase, create_model_tables, drop_model_tables, savepoint_sqlite,
savepoint)
from itertools import count
from uuid import UUID
from uuid import UUID, uuid4
from threading import Event
from email.utils import formatdate
from data.database import (db, all_models, Role, TeamRole, Visibility, LoginService,
BuildTriggerService, AccessTokenKind, LogEntryKind, ImageStorageLocation,
ImageStorageTransformation, ImageStorageSignatureKind,
ExternalNotificationEvent, ExternalNotificationMethod, NotificationKind)
ExternalNotificationEvent, ExternalNotificationMethod, NotificationKind,
QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode)
from data import model
from app import app, storage as store
from data.queue import WorkQueue
from app import app, storage as store, tf
from storage.basestorage import StoragePaths
from endpoints.v2.manifest import _generate_and_store_manifest
@ -85,6 +87,9 @@ def __create_subtree(repo, structure, creator_username, parent, tag_map):
new_image.storage.uploading = False
new_image.storage.save()
# Write out a fake torrentinfo
model.storage.save_torrent_info(new_image.storage, 1, 'deadbeef')
# Write some data for the storage.
if os.environ.get('WRITE_STORAGE_FILES'):
storage_paths = StoragePaths()
@ -127,6 +132,8 @@ def __create_subtree(repo, structure, creator_username, parent, tag_map):
for tag_name in last_node_tags:
new_tag = model.tag.create_or_update_tag(repo.namespace_user.username, repo.name, tag_name,
new_image.docker_image_id)
derived = model.image.find_or_create_derived_storage(new_tag, 'squash', 'local_us')
model.storage.find_or_create_storage_signature(derived, 'gpg2')
_generate_and_store_manifest(repo.namespace_user.username, repo.name, tag_name)
tag_map[tag_name] = new_tag
@ -191,6 +198,11 @@ def setup_database_for_testing(testcase):
initialize_database()
populate_database()
models_missing_data = find_models_missing_data()
if models_missing_data:
raise RuntimeError('%s models are missing data: %s', len(models_missing_data),
models_missing_data)
# Enable foreign key constraints.
if not IS_TESTING_REAL_DATABASE:
db.obj.execute_sql('PRAGMA foreign_keys = ON;')
@ -333,6 +345,9 @@ def initialize_database():
NotificationKind.create(name='test_notification')
QuayRegion.create(name='us')
QuayService.create(name='quay')
def wipe_database():
logger.debug('Wiping all data from the DB.')
@ -356,6 +371,11 @@ def populate_database(minimal=False):
logger.debug('Skipping most db population because user requested mininal db')
return
UserRegion.create(user=new_user_1, location=1)
model.release.set_region_release('quay', 'us', 'v0.1.2')
model.user.create_confirm_email_code(new_user_1, new_email='typo@devtable.com')
disabled_user = model.user.create_user('disabled', 'password', 'jschorr+disabled@devtable.com')
disabled_user.verified = True
disabled_user.enabled = False
@ -413,6 +433,8 @@ def populate_database(minimal=False):
simple_repo = __generate_repository(new_user_1, 'simple', 'Simple repository.', False,
[], (4, [], ['latest', 'prod']))
model.blob.initiate_upload(new_user_1.username, simple_repo.name, str(uuid4()), 'local_us', {})
model.notification.create_repo_notification(simple_repo, 'repo_push', 'quay_notification', {}, {})
__generate_repository(new_user_1, 'sharedtags',
'Shared tags repository',
@ -517,8 +539,9 @@ def populate_database(minimal=False):
model.user.create_robot('coolrobot', org)
model.oauth.create_application(org, 'Some Test App', 'http://localhost:8000',
'http://localhost:8000/o2c.html', client_id='deadbeef')
oauth_app_1 = model.oauth.create_application(org, 'Some Test App', 'http://localhost:8000',
'http://localhost:8000/o2c.html',
client_id='deadbeef')
model.oauth.create_application(org, 'Some Other Test App', 'http://quay.io',
'http://localhost:8000/o2c.html', client_id='deadpork',
@ -526,6 +549,9 @@ def populate_database(minimal=False):
model.oauth.create_access_token_for_testing(new_user_1, 'deadbeef', 'repo:admin')
OAuthAuthorizationCode.create(application=oauth_app_1, code='Z932odswfhasdf1', scope='repo:admin',
data='{"somejson": "goeshere"}')
model.user.create_robot('neworgrobot', org)
ownerbot = model.user.create_robot('ownerbot', org)[0]
@ -544,6 +570,7 @@ def populate_database(minimal=False):
creators = model.team.create_team('creators', org, 'creator', 'Creators of orgrepo.')
reader_team = model.team.create_team('readers', org, 'member', 'Readers of orgrepo.')
model.team.add_or_invite_to_team(new_user_1, reader_team, outside_org)
model.permission.set_team_repo_permission(reader_team.name, org_repo.namespace_user.username,
org_repo.name, 'read')
@ -640,10 +667,25 @@ def populate_database(minimal=False):
'trigger_id': trigger.uuid, 'config': json.loads(trigger.config),
'service': trigger.service.name})
fake_queue = WorkQueue('fakequeue', tf)
fake_queue.put(['canonical', 'job', 'name'], '{}')
while repositoryactioncounter.count_repository_actions():
pass
def find_models_missing_data():
# As a sanity check we are going to make sure that all db tables have some data
models_missing_data = set()
for one_model in all_models:
try:
one_model.select().get()
except one_model.DoesNotExist:
models_missing_data.add(one_model.__name__)
return models_missing_data
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Initialize the test database.')
parser.add_argument('--simple', action='store_true')
@ -658,3 +700,8 @@ if __name__ == '__main__':
initialize_database()
populate_database(args.simple)
if not args.simple:
models_missing_data = find_models_missing_data()
if models_missing_data:
logger.warning('The following models do not have any data: %s', models_missing_data)

BIN
test/data/test.db
View file

Binary file not shown.

17
test/test_api_usage.py
View file

@ -24,8 +24,7 @@ from endpoints.api.team import TeamMember, TeamMemberList, TeamMemberInvite, Org
from endpoints.api.tag import RepositoryTagImages, RepositoryTag, RevertTag, ListRepositoryTags
from endpoints.api.search import EntitySearch, ConductSearch
from endpoints.api.image import RepositoryImage, RepositoryImageList
from endpoints.api.build import (RepositoryBuildStatus, RepositoryBuildLogs, RepositoryBuildList,
RepositoryBuildResource)
from endpoints.api.build import RepositoryBuildStatus, RepositoryBuildList, RepositoryBuildResource
from endpoints.api.robot import (UserRobotList, OrgRobot, OrgRobotList, UserRobot,
RegenerateUserRobot, RegenerateOrgRobot)
from endpoints.api.trigger import (BuildTriggerActivate, BuildTriggerSources, BuildTriggerSubdirs,
@ -1226,6 +1225,14 @@ class TestDeleteOrganizationTeamMember(ApiTestCase):
def test_deletememberinvite(self):
self.login(ADMIN_ACCESS_USER)
# Verify the initial member count
json = self.getJsonResponse(TeamMemberList,
params=dict(orgname=ORGANIZATION,
teamname='readers',
includePending=True))
self.assertEquals(len(json['members']), 3)
membername = NO_ACCESS_USER
response = self.putJsonResponse(TeamMember,
params=dict(orgname=ORGANIZATION, teamname='readers',
@ -1240,7 +1247,7 @@ class TestDeleteOrganizationTeamMember(ApiTestCase):
teamname='readers',
includePending=True))
assert len(json['members']) == 3
self.assertEquals(len(json['members']), 4)
# Delete the invite.
self.deleteResponse(TeamMember,
@ -1254,7 +1261,7 @@ class TestDeleteOrganizationTeamMember(ApiTestCase):
teamname='readers',
includePending=True))
assert len(json['members']) == 2
self.assertEquals(len(json['members']), 3)
def test_deletemember(self):
@ -1270,7 +1277,7 @@ class TestDeleteOrganizationTeamMember(ApiTestCase):
params=dict(orgname=ORGANIZATION,
teamname='readers'))
assert len(json['members']) == 1
self.assertEquals(len(json['members']), 1)
class TestCreateRepo(ApiTestCase):