diff --git a/data/database.py b/data/database.py index 0573eadea..e4a0ad957 100644 --- a/data/database.py +++ b/data/database.py @@ -4,6 +4,8 @@ import uuid import time import toposort import resumablehashlib +import sys +import inspect from random import SystemRandom from datetime import datetime @@ -861,15 +863,5 @@ class TorrentInfo(BaseModel): (('storage', 'piece_length'), True), ) - -all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, Visibility, - RepositoryTag, EmailConfirmation, FederatedLogin, LoginService, QueueItem, - RepositoryBuild, Team, TeamMember, TeamRole, LogEntryKind, LogEntry, - PermissionPrototype, ImageStorage, BuildTriggerService, RepositoryBuildTrigger, - OAuthApplication, OAuthAuthorizationCode, OAuthAccessToken, NotificationKind, - Notification, ImageStorageLocation, ImageStoragePlacement, - ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification, - RepositoryAuthorizedEmail, ImageStorageTransformation, - TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind, - AccessTokenKind, Star, RepositoryActionCount, TagManifest, UserRegion, - QuayService, QuayRegion, QuayRelease, BlobUpload, DerivedStorageForImage, TorrentInfo] +is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel +all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)] diff --git a/data/model/__init__.py b/data/model/__init__.py index 7115aa9a9..cfc68a0c3 100644 --- a/data/model/__init__.py +++ b/data/model/__init__.py @@ -95,4 +95,4 @@ config = Config() # moving the minimal number of things to _basequery # TODO document the methods and modules for each one of the submodules below. from data.model import (blob, build, image, log, notification, oauth, organization, permission, - repository, storage, tag, team, token, user) + repository, storage, tag, team, token, user, release) diff --git a/data/model/release.py b/data/model/release.py index 883ae146e..9427f1e71 100644 --- a/data/model/release.py +++ b/data/model/release.py @@ -10,14 +10,12 @@ def set_region_release(service_name, region_name, version): def get_recent_releases(service_name, region_name): return (QuayRelease - .select(QuayRelease) - .join(QuayService) - .switch(QuayRelease) - .join(QuayRegion) - .where( - QuayService.name == service_name, - QuayRegion.name == region_name, - QuayRelease.reverted == False, - ) - .order_by(QuayRelease.created.desc()) - ) + .select(QuayRelease) + .join(QuayService) + .switch(QuayRelease) + .join(QuayRegion) + .where(QuayService.name == service_name, + QuayRegion.name == region_name, + QuayRelease.reverted == False, + ) + .order_by(QuayRelease.created.desc())) diff --git a/data/model/storage.py b/data/model/storage.py index 41eca2fd8..0f2220f91 100644 --- a/data/model/storage.py +++ b/data/model/storage.py @@ -75,6 +75,12 @@ def garbage_collect_storage(storage_id_whitelist): .execute()) logger.debug('Removed %s torrent info records', torrents_removed) + signatures_removed = (ImageStorageSignature + .delete() + .where(ImageStorageSignature.storage << orphaned_storages) + .execute()) + logger.debug('Removed %s image storage signatures', signatures_removed) + storages_removed = (ImageStorage .delete() .where(ImageStorage.id << orphaned_storages) @@ -97,17 +103,17 @@ def create_v1_storage(location_name): return storage -def find_or_create_storage_signature(storage, signature_kind): - found = lookup_storage_signature(storage, signature_kind) +def find_or_create_storage_signature(storage, signature_kind_name): + found = lookup_storage_signature(storage, signature_kind_name) if found is None: - kind = ImageStorageSignatureKind.get(name=signature_kind) + kind = ImageStorageSignatureKind.get(name=signature_kind_name) found = ImageStorageSignature.create(storage=storage, kind=kind) return found -def lookup_storage_signature(storage, signature_kind): - kind = ImageStorageSignatureKind.get(name=signature_kind) +def lookup_storage_signature(storage, signature_kind_name): + kind = ImageStorageSignatureKind.get(name=signature_kind_name) try: return (ImageStorageSignature .select() diff --git a/data/model/user.py b/data/model/user.py index 4db9e132b..38e240c9d 100644 --- a/data/model/user.py +++ b/data/model/user.py @@ -368,6 +368,7 @@ def lookup_federated_login(user, service_name): except FederatedLogin.DoesNotExist: return None + def create_confirm_email_code(user, new_email=None): if new_email: if not validate_email(new_email): diff --git a/initdb.py b/initdb.py index 73f5902dc..776ca165a 100644 --- a/initdb.py +++ b/initdb.py @@ -10,16 +10,18 @@ from datetime import datetime, timedelta from peewee import (SqliteDatabase, create_model_tables, drop_model_tables, savepoint_sqlite, savepoint) from itertools import count -from uuid import UUID +from uuid import UUID, uuid4 from threading import Event from email.utils import formatdate from data.database import (db, all_models, Role, TeamRole, Visibility, LoginService, BuildTriggerService, AccessTokenKind, LogEntryKind, ImageStorageLocation, ImageStorageTransformation, ImageStorageSignatureKind, - ExternalNotificationEvent, ExternalNotificationMethod, NotificationKind) + ExternalNotificationEvent, ExternalNotificationMethod, NotificationKind, + QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode) from data import model -from app import app, storage as store +from data.queue import WorkQueue +from app import app, storage as store, tf from storage.basestorage import StoragePaths from endpoints.v2.manifest import _generate_and_store_manifest @@ -85,6 +87,9 @@ def __create_subtree(repo, structure, creator_username, parent, tag_map): new_image.storage.uploading = False new_image.storage.save() + # Write out a fake torrentinfo + model.storage.save_torrent_info(new_image.storage, 1, 'deadbeef') + # Write some data for the storage. if os.environ.get('WRITE_STORAGE_FILES'): storage_paths = StoragePaths() @@ -127,6 +132,8 @@ def __create_subtree(repo, structure, creator_username, parent, tag_map): for tag_name in last_node_tags: new_tag = model.tag.create_or_update_tag(repo.namespace_user.username, repo.name, tag_name, new_image.docker_image_id) + derived = model.image.find_or_create_derived_storage(new_tag, 'squash', 'local_us') + model.storage.find_or_create_storage_signature(derived, 'gpg2') _generate_and_store_manifest(repo.namespace_user.username, repo.name, tag_name) tag_map[tag_name] = new_tag @@ -191,6 +198,11 @@ def setup_database_for_testing(testcase): initialize_database() populate_database() + models_missing_data = find_models_missing_data() + if models_missing_data: + raise RuntimeError('%s models are missing data: %s', len(models_missing_data), + models_missing_data) + # Enable foreign key constraints. if not IS_TESTING_REAL_DATABASE: db.obj.execute_sql('PRAGMA foreign_keys = ON;') @@ -333,6 +345,9 @@ def initialize_database(): NotificationKind.create(name='test_notification') + QuayRegion.create(name='us') + QuayService.create(name='quay') + def wipe_database(): logger.debug('Wiping all data from the DB.') @@ -356,6 +371,11 @@ def populate_database(minimal=False): logger.debug('Skipping most db population because user requested mininal db') return + UserRegion.create(user=new_user_1, location=1) + model.release.set_region_release('quay', 'us', 'v0.1.2') + + model.user.create_confirm_email_code(new_user_1, new_email='typo@devtable.com') + disabled_user = model.user.create_user('disabled', 'password', 'jschorr+disabled@devtable.com') disabled_user.verified = True disabled_user.enabled = False @@ -413,6 +433,8 @@ def populate_database(minimal=False): simple_repo = __generate_repository(new_user_1, 'simple', 'Simple repository.', False, [], (4, [], ['latest', 'prod'])) + model.blob.initiate_upload(new_user_1.username, simple_repo.name, str(uuid4()), 'local_us', {}) + model.notification.create_repo_notification(simple_repo, 'repo_push', 'quay_notification', {}, {}) __generate_repository(new_user_1, 'sharedtags', 'Shared tags repository', @@ -517,8 +539,9 @@ def populate_database(minimal=False): model.user.create_robot('coolrobot', org) - model.oauth.create_application(org, 'Some Test App', 'http://localhost:8000', - 'http://localhost:8000/o2c.html', client_id='deadbeef') + oauth_app_1 = model.oauth.create_application(org, 'Some Test App', 'http://localhost:8000', + 'http://localhost:8000/o2c.html', + client_id='deadbeef') model.oauth.create_application(org, 'Some Other Test App', 'http://quay.io', 'http://localhost:8000/o2c.html', client_id='deadpork', @@ -526,6 +549,9 @@ def populate_database(minimal=False): model.oauth.create_access_token_for_testing(new_user_1, 'deadbeef', 'repo:admin') + OAuthAuthorizationCode.create(application=oauth_app_1, code='Z932odswfhasdf1', scope='repo:admin', + data='{"somejson": "goeshere"}') + model.user.create_robot('neworgrobot', org) ownerbot = model.user.create_robot('ownerbot', org)[0] @@ -544,6 +570,7 @@ def populate_database(minimal=False): creators = model.team.create_team('creators', org, 'creator', 'Creators of orgrepo.') reader_team = model.team.create_team('readers', org, 'member', 'Readers of orgrepo.') + model.team.add_or_invite_to_team(new_user_1, reader_team, outside_org) model.permission.set_team_repo_permission(reader_team.name, org_repo.namespace_user.username, org_repo.name, 'read') @@ -640,10 +667,25 @@ def populate_database(minimal=False): 'trigger_id': trigger.uuid, 'config': json.loads(trigger.config), 'service': trigger.service.name}) + fake_queue = WorkQueue('fakequeue', tf) + fake_queue.put(['canonical', 'job', 'name'], '{}') + while repositoryactioncounter.count_repository_actions(): pass +def find_models_missing_data(): + # As a sanity check we are going to make sure that all db tables have some data + models_missing_data = set() + for one_model in all_models: + try: + one_model.select().get() + except one_model.DoesNotExist: + models_missing_data.add(one_model.__name__) + + return models_missing_data + + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Initialize the test database.') parser.add_argument('--simple', action='store_true') @@ -658,3 +700,8 @@ if __name__ == '__main__': initialize_database() populate_database(args.simple) + + if not args.simple: + models_missing_data = find_models_missing_data() + if models_missing_data: + logger.warning('The following models do not have any data: %s', models_missing_data) diff --git a/test/data/test.db b/test/data/test.db index 1dfcb09e4..4a72f1686 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/test/test_api_usage.py b/test/test_api_usage.py index 27a8926e2..f815784ad 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -24,8 +24,7 @@ from endpoints.api.team import TeamMember, TeamMemberList, TeamMemberInvite, Org from endpoints.api.tag import RepositoryTagImages, RepositoryTag, RevertTag, ListRepositoryTags from endpoints.api.search import EntitySearch, ConductSearch from endpoints.api.image import RepositoryImage, RepositoryImageList -from endpoints.api.build import (RepositoryBuildStatus, RepositoryBuildLogs, RepositoryBuildList, - RepositoryBuildResource) +from endpoints.api.build import RepositoryBuildStatus, RepositoryBuildList, RepositoryBuildResource from endpoints.api.robot import (UserRobotList, OrgRobot, OrgRobotList, UserRobot, RegenerateUserRobot, RegenerateOrgRobot) from endpoints.api.trigger import (BuildTriggerActivate, BuildTriggerSources, BuildTriggerSubdirs, @@ -1226,6 +1225,14 @@ class TestDeleteOrganizationTeamMember(ApiTestCase): def test_deletememberinvite(self): self.login(ADMIN_ACCESS_USER) + # Verify the initial member count + json = self.getJsonResponse(TeamMemberList, + params=dict(orgname=ORGANIZATION, + teamname='readers', + includePending=True)) + + self.assertEquals(len(json['members']), 3) + membername = NO_ACCESS_USER response = self.putJsonResponse(TeamMember, params=dict(orgname=ORGANIZATION, teamname='readers', @@ -1240,7 +1247,7 @@ class TestDeleteOrganizationTeamMember(ApiTestCase): teamname='readers', includePending=True)) - assert len(json['members']) == 3 + self.assertEquals(len(json['members']), 4) # Delete the invite. self.deleteResponse(TeamMember, @@ -1254,7 +1261,7 @@ class TestDeleteOrganizationTeamMember(ApiTestCase): teamname='readers', includePending=True)) - assert len(json['members']) == 2 + self.assertEquals(len(json['members']), 3) def test_deletemember(self): @@ -1270,7 +1277,7 @@ class TestDeleteOrganizationTeamMember(ApiTestCase): params=dict(orgname=ORGANIZATION, teamname='readers')) - assert len(json['members']) == 1 + self.assertEquals(len(json['members']), 1) class TestCreateRepo(ApiTestCase):