From 6070c251ae74815384922638a94c9d94956fd951 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 24 Sep 2014 10:42:42 -0400 Subject: [PATCH 1/4] Fix the backfill script to handle images without any json data at all. --- tools/uncompressedsize.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/uncompressedsize.py b/tools/uncompressedsize.py index 32ba0957c..4ed9a772a 100644 --- a/tools/uncompressedsize.py +++ b/tools/uncompressedsize.py @@ -34,9 +34,13 @@ def backfill_sizes(): uuid = image_storage.uuid with_locations = model.get_storage_by_uuid(uuid) - json_string = store.get_content(with_locations.locations, store.image_json_path(uuid)) - json_data = json.loads(json_string) - size = json_data.get('Size', json_data.get('size', -1)) + try: + json_string = store.get_content(with_locations.locations, store.image_json_path(uuid)) + json_data = json.loads(json_string) + size = json_data.get('Size', json_data.get('size', -1)) + except IOError: + logger.debug('Image storage with no json %s', uuid) + size = -1 if size == -1: missing += 1 From 03190efde376ddc93e169cbe6db27c3ef5fdfff1 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 24 Sep 2014 18:01:35 -0400 Subject: [PATCH 2/4] Phase 2 of migrating repo namespaces to referencing user objects, backfilling the rows without a value for namespace_user, and changing all accesses to go through the namespace_user object. All tests are passing, manual testing still required. --- auth/permissions.py | 4 +- data/database.py | 4 +- ...4671_backfill_the_namespace_user_fields.py | 27 + data/model/legacy.py | 490 ++++++++++-------- endpoints/api/build.py | 2 +- endpoints/api/repoemail.py | 2 +- endpoints/api/repository.py | 8 +- endpoints/api/search.py | 4 +- endpoints/api/trigger.py | 2 +- endpoints/common.py | 13 +- endpoints/index.py | 4 +- endpoints/notificationevent.py | 4 - endpoints/notificationhelper.py | 11 +- endpoints/notificationmethod.py | 16 +- endpoints/web.py | 4 +- initdb.py | 45 +- test/data/test.db | Bin 638976 -> 630784 bytes tools/auditancestry.py | 36 +- tools/relationships.py | 2 +- 19 files changed, 373 insertions(+), 305 deletions(-) create mode 100644 data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py diff --git a/auth/permissions.py b/auth/permissions.py index 59b190b3c..eb9059c22 100644 --- a/auth/permissions.py +++ b/auth/permissions.py @@ -112,7 +112,7 @@ class QuayDeferredPermissionUser(Identity): # Add repository permissions for perm in model.get_all_user_permissions(user_object): - repo_grant = _RepositoryNeed(perm.repository.namespace, perm.repository.name, + repo_grant = _RepositoryNeed(perm.repository.namespace_user.username, perm.repository.name, self._repo_role_for_scopes(perm.role.name)) logger.debug('User added permission: {0}'.format(repo_grant)) self.provides.add(repo_grant) @@ -239,7 +239,7 @@ def on_identity_loaded(sender, identity): logger.debug('Loading permissions for token: %s', identity.id) token_data = model.load_token_data(identity.id) - repo_grant = _RepositoryNeed(token_data.repository.namespace, + repo_grant = _RepositoryNeed(token_data.repository.namespace_user.username, token_data.repository.name, token_data.role.name) logger.debug('Delegate token added permission: {0}'.format(repo_grant)) diff --git a/data/database.py b/data/database.py index f5d780d80..b9716bf0f 100644 --- a/data/database.py +++ b/data/database.py @@ -169,7 +169,7 @@ class Visibility(BaseModel): class Repository(BaseModel): namespace = CharField() - namespace_user = ForeignKeyField(User, null=True) + namespace_user = ForeignKeyField(User) name = CharField() visibility = ForeignKeyField(Visibility) description = TextField(null=True) @@ -181,7 +181,7 @@ class Repository(BaseModel): indexes = ( # create a unique index on namespace and name (('namespace', 'name'), True), - (('namespace_user', 'name'), False), + (('namespace_user', 'name'), True), ) diff --git a/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py b/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py new file mode 100644 index 000000000..7c3b7924b --- /dev/null +++ b/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py @@ -0,0 +1,27 @@ +"""Backfill the namespace_user fields. + +Revision ID: 3f4fe1194671 +Revises: 6f2ecf5afcf +Create Date: 2014-09-24 14:29:45.192179 + +""" + +# revision identifiers, used by Alembic. +revision = '3f4fe1194671' +down_revision = '6f2ecf5afcf' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + conn = op.get_bind() + conn.execute('update repository set namespace_user_id = (select id from user where user.username == repository.namespace) where namespace_user_id is NULL') + + op.alter_column('repository', 'namespace_user_id', nullable=False) + op.create_index('repository_namespace_user_id_name', 'repository', ['namespace_user_id', 'name'], unique=True) + + +def downgrade(tables): + op.drop_index('repository_namespace_user_id_name', table_name='repository') + op.alter_column('repository', 'namespace_user_id', nullable=True) diff --git a/data/model/legacy.py b/data/model/legacy.py index 2cd8be94b..07c8b9e2a 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -25,6 +25,9 @@ EXPONENTIAL_BACKOFF_SCALE = timedelta(seconds=1) PRESUMED_DEAD_BUILD_AGE = timedelta(days=15) +Namespace = User.alias() + + logger = logging.getLogger(__name__) @@ -100,13 +103,24 @@ class TooManyLoginAttemptsException(Exception): super(TooManyLoginAttemptsException, self).__init__(message) self.retry_after = retry_after + +def _get_repository(namespace_name, repository_name): + return (Repository + .select(Repository, Namespace) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name) + .get()) + + def hash_password(password, salt=None): salt = salt or bcrypt.gensalt() return bcrypt.hashpw(password.encode('utf-8'), salt) + def is_create_user_allowed(): return True + def create_user(username, password, email, auto_verify=False): """ Creates a regular user, if allowed. """ if not validate_password(password): @@ -122,6 +136,7 @@ def create_user(username, password, email, auto_verify=False): return created + def _create_user(username, email): if not validate_email(email): raise InvalidEmailAddressException('Invalid email address: %s' % email) @@ -733,7 +748,7 @@ def get_visible_repositories(username=None, include_public=True, page=None, limit=None, sort=False, namespace=None): query = _visible_repository_query(username=username, include_public=include_public, page=page, limit=limit, namespace=namespace, - select_models=[Repository, Visibility]) + select_models=[Repository, Namespace, Visibility]) if sort: query = query.order_by(Repository.description.desc()) @@ -747,11 +762,13 @@ def get_visible_repositories(username=None, include_public=True, page=None, def _visible_repository_query(username=None, include_public=True, limit=None, page=None, namespace=None, select_models=[]): query = (Repository - .select(*select_models) # Note: We need to leave this blank for the get_count case. Otherwise, MySQL/RDS complains. - .distinct() - .join(Visibility) - .switch(Repository) - .join(RepositoryPermission, JOIN_LEFT_OUTER)) + .select(*select_models) # MySQL/RDS complains is there are selected models for counts. + .distinct() + .join(Visibility) + .switch(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(Repository) + .join(RepositoryPermission, JOIN_LEFT_OUTER)) query = _filter_to_repos_for_user(query, username, namespace, include_public) @@ -782,26 +799,20 @@ def _filter_to_repos_for_user(query, username=None, namespace=None, .switch(RepositoryPermission) .join(Team, JOIN_LEFT_OUTER) .join(TeamMember, JOIN_LEFT_OUTER) - .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == - TeamMember.user)) + .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == TeamMember.user)) .switch(Repository) - .join(Org, JOIN_LEFT_OUTER, on=(Org.username == Repository.namespace)) - .join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id == - AdminTeam.organization)) + .join(Org, JOIN_LEFT_OUTER, on=(Repository.namespace_user == Org.id)) + .join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id == AdminTeam.organization)) .join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id)) .switch(AdminTeam) - .join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id == - AdminTeamMember.team)) - .join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user == - AdminUser.id))) + .join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id == AdminTeamMember.team)) + .join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user == AdminUser.id))) - where_clause = ((User.username == username) | - (UserThroughTeam.username == username) | - ((AdminUser.username == username) & - (TeamRole.name == 'admin'))) + where_clause = ((User.username == username) | (UserThroughTeam.username == username) | + ((AdminUser.username == username) & (TeamRole.name == 'admin'))) if namespace: - where_clause = where_clause & (Repository.namespace == namespace) + where_clause = where_clause & (Namespace.username == namespace) if include_public: new_clause = (Visibility.name == 'public') @@ -820,7 +831,7 @@ def get_matching_repositories(repo_term, username=None): visible = get_visible_repositories(username) search_clauses = (Repository.name ** ('%' + name_term + '%') | - Repository.namespace ** ('%' + namespace_term + '%')) + Namespace.username ** ('%' + namespace_term + '%')) # Handle the case where the user has already entered a namespace path. if repo_term.find('/') > 0: @@ -829,7 +840,7 @@ def get_matching_repositories(repo_term, username=None): name_term = parts[-1] search_clauses = (Repository.name ** ('%' + name_term + '%') & - Repository.namespace ** ('%' + namespace_term + '%')) + Namespace.username ** ('%' + namespace_term + '%')) final = visible.where(search_clauses).limit(10) return list(final) @@ -859,22 +870,20 @@ def update_email(user, new_email, auto_verify=False): def get_all_user_permissions(user): - select = RepositoryPermission.select(RepositoryPermission, Role, Repository) - with_role = select.join(Role) - with_repo = with_role.switch(RepositoryPermission).join(Repository) - through_user = with_repo.switch(RepositoryPermission).join(User, - JOIN_LEFT_OUTER) - as_perm = through_user.switch(RepositoryPermission) - through_team = as_perm.join(Team, JOIN_LEFT_OUTER).join(TeamMember, - JOIN_LEFT_OUTER) - UserThroughTeam = User.alias() - with_team_member = through_team.join(UserThroughTeam, JOIN_LEFT_OUTER, - on=(UserThroughTeam.id == - TeamMember.user)) - return with_team_member.where((User.id == user) | - (UserThroughTeam.id == user)) + return (RepositoryPermission + .select(RepositoryPermission, Role, Repository, Namespace) + .join(Role) + .switch(RepositoryPermission) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(RepositoryPermission) + .join(User, JOIN_LEFT_OUTER) + .switch(RepositoryPermission) + .join(Team, JOIN_LEFT_OUTER).join(TeamMember, JOIN_LEFT_OUTER) + .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == TeamMember.user)) + .where((User.id == user) | (UserThroughTeam.id == user))) def delete_prototype_permission(org, uid): @@ -939,33 +948,37 @@ def get_org_wide_permissions(user): def get_all_repo_teams(namespace_name, repository_name): - select = RepositoryPermission.select(Team.name.alias('team_name'), - Role.name, RepositoryPermission) - with_team = select.join(Team) - with_role = with_team.switch(RepositoryPermission).join(Role) - with_repo = with_role.switch(RepositoryPermission).join(Repository) - return with_repo.where(Repository.namespace == namespace_name, - Repository.name == repository_name) + return (RepositoryPermission.select(Team.name.alias('team_name'), Role.name, RepositoryPermission) + .join(Team) + .switch(RepositoryPermission) + .join(Role) + .switch(RepositoryPermission) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name)) def get_all_repo_users(namespace_name, repository_name): - select = RepositoryPermission.select(User.username, User.robot, Role.name, - RepositoryPermission) - with_user = select.join(User) - with_role = with_user.switch(RepositoryPermission).join(Role) - with_repo = with_role.switch(RepositoryPermission).join(Repository) - return with_repo.where(Repository.namespace == namespace_name, - Repository.name == repository_name) + return (RepositoryPermission.select(User.username, User.robot, Role.name, RepositoryPermission) + .join(User) + .switch(RepositoryPermission) + .join(Role) + .switch(RepositoryPermission) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name)) def get_all_repo_users_transitive_via_teams(namespace_name, repository_name): - select = User.select().distinct() - with_team_member = select.join(TeamMember) - with_team = with_team_member.join(Team) - with_perm = with_team.join(RepositoryPermission) - with_repo = with_perm.join(Repository) - return with_repo.where(Repository.namespace == namespace_name, - Repository.name == repository_name) + return (User + .select() + .distinct() + .join(TeamMember) + .join(Team) + .join(RepositoryPermission) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name)) def get_all_repo_users_transitive(namespace_name, repository_name): @@ -989,10 +1002,12 @@ def get_all_repo_users_transitive(namespace_name, repository_name): def get_repository_for_resource(resource_key): try: return (Repository - .select() - .join(RepositoryBuild) - .where(RepositoryBuild.resource_key == resource_key) - .get()) + .select(Repository, Namespace) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(Repository) + .join(RepositoryBuild) + .where(RepositoryBuild.resource_key == resource_key) + .get()) except Repository.DoesNotExist: return None @@ -1006,8 +1021,7 @@ def lookup_repository(repo_id): def get_repository(namespace_name, repository_name): try: - return Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + return _get_repository(namespace_name, repository_name) except Repository.DoesNotExist: return None @@ -1024,11 +1038,18 @@ def get_repo_image(namespace_name, repository_name, image_id): def repository_is_public(namespace_name, repository_name): - joined = Repository.select().join(Visibility) - query = joined.where(Repository.namespace == namespace_name, - Repository.name == repository_name, - Visibility.name == 'public') - return len(list(query)) > 0 + try: + (Repository + .select() + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(Repository) + .join(Visibility) + .where(Namespace.username == namespace_name, Repository.name == repository_name, + Visibility.name == 'public') + .get()) + return True + except Repository.DoesNotExist: + return False def set_repository_visibility(repo, visibility): @@ -1128,7 +1149,7 @@ def __translate_ancestry(old_ancestry, translations, repository, username, prefe def find_create_or_link_image(docker_image_id, repository, username, translations, preferred_location): with config.app_config['DB_TRANSACTION_FACTORY'](db): - repo_image = get_repo_image(repository.namespace, repository.name, + repo_image = get_repo_image(repository.namespace_user.username, repository.name, docker_image_id) if repo_image: return repo_image @@ -1142,6 +1163,8 @@ def find_create_or_link_image(docker_image_id, repository, username, translation .join(Visibility) .switch(Repository) .join(RepositoryPermission, JOIN_LEFT_OUTER) + .switch(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) .where(ImageStorage.uploading == False)) query = (_filter_to_repos_for_user(query, username) @@ -1186,11 +1209,11 @@ def find_create_or_link_image(docker_image_id, repository, username, translation def get_storage_by_uuid(storage_uuid): placements = list(ImageStoragePlacement - .select(ImageStoragePlacement, ImageStorage, ImageStorageLocation) - .join(ImageStorageLocation) - .switch(ImageStoragePlacement) - .join(ImageStorage) - .where(ImageStorage.uuid == storage_uuid)) + .select(ImageStoragePlacement, ImageStorage, ImageStorageLocation) + .join(ImageStorageLocation) + .switch(ImageStoragePlacement) + .join(ImageStorage) + .where(ImageStorage.uuid == storage_uuid)) if not placements: raise InvalidImageException('No storage found with uuid: %s', storage_uuid) @@ -1205,14 +1228,14 @@ def set_image_size(docker_image_id, namespace_name, repository_name, image_size): try: image = (Image - .select(Image, ImageStorage) - .join(Repository) - .switch(Image) - .join(ImageStorage, JOIN_LEFT_OUTER) - .where(Repository.name == repository_name, - Repository.namespace == namespace_name, - Image.docker_image_id == docker_image_id) - .get()) + .select(Image, ImageStorage) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(Image) + .join(ImageStorage, JOIN_LEFT_OUTER) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + Image.docker_image_id == docker_image_id) + .get()) except Image.DoesNotExist: raise DataModelException('No image with specified id and repository') @@ -1231,13 +1254,13 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created command, uncompressed_size, parent=None): with config.app_config['DB_TRANSACTION_FACTORY'](db): query = (Image - .select(Image, ImageStorage) - .join(Repository) - .switch(Image) - .join(ImageStorage) - .where(Repository.name == repository_name, - Repository.namespace == namespace_name, - Image.docker_image_id == docker_image_id)) + .select(Image, ImageStorage) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(Image) + .join(ImageStorage) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + Image.docker_image_id == docker_image_id)) try: fetched = query.get() @@ -1248,7 +1271,7 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created fetched.storage.checksum = None fetched.storage.created = dateutil.parser.parse(created_date_str).replace(tzinfo=None) fetched.storage.comment = comment - fetched.storage.command = command + fetched.storage.command = command fetched.storage.uncompressed_size = uncompressed_size if parent: @@ -1261,14 +1284,14 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created def _get_repository_images_base(namespace_name, repository_name, query_modifier): query = (ImageStoragePlacement - .select(ImageStoragePlacement, Image, ImageStorage, ImageStorageLocation) - .join(ImageStorageLocation) - .switch(ImageStoragePlacement) - .join(ImageStorage, JOIN_LEFT_OUTER) - .join(Image) - .join(Repository) - .where(Repository.name == repository_name, - Repository.namespace == namespace_name)) + .select(ImageStoragePlacement, Image, ImageStorage, ImageStorageLocation) + .join(ImageStorageLocation) + .switch(ImageStoragePlacement) + .join(ImageStorage, JOIN_LEFT_OUTER) + .join(Image) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.name == repository_name, Namespace.username == namespace_name)) query = query_modifier(query) @@ -1299,24 +1322,26 @@ def get_repository_images(namespace_name, repository_name): def list_repository_tags(namespace_name, repository_name): - select = RepositoryTag.select(RepositoryTag, Image) - with_repo = select.join(Repository) - with_image = with_repo.switch(RepositoryTag).join(Image) - return with_image.where(Repository.name == repository_name, - Repository.namespace == namespace_name) + return (RepositoryTag + .select(RepositoryTag, Image) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(RepositoryTag) + .join(Image) + .where(Repository.name == repository_name, Namespace.username == namespace_name)) def garbage_collect_repository(namespace_name, repository_name): with config.app_config['DB_TRANSACTION_FACTORY'](db): # Get a list of all images used by tags in the repository tag_query = (RepositoryTag - .select(RepositoryTag, Image, ImageStorage) - .join(Image) - .join(ImageStorage, JOIN_LEFT_OUTER) - .switch(RepositoryTag) - .join(Repository) - .where(Repository.name == repository_name, - Repository.namespace == namespace_name)) + .select(RepositoryTag, Image, ImageStorage) + .join(Image) + .join(ImageStorage, JOIN_LEFT_OUTER) + .switch(RepositoryTag) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.name == repository_name, Namespace.username == namespace_name)) referenced_anscestors = set() for tag in tag_query: @@ -1344,11 +1369,11 @@ def garbage_collect_repository(namespace_name, repository_name): if uuids_to_check_for_gc: storage_to_remove = (ImageStorage - .select() - .join(Image, JOIN_LEFT_OUTER) - .group_by(ImageStorage) - .where(ImageStorage.uuid << list(uuids_to_check_for_gc)) - .having(fn.Count(Image.id) == 0)) + .select() + .join(Image, JOIN_LEFT_OUTER) + .group_by(ImageStorage) + .where(ImageStorage.uuid << list(uuids_to_check_for_gc)) + .having(fn.Count(Image.id) == 0)) for storage in storage_to_remove: logger.debug('Garbage collecting image storage: %s', storage.uuid) @@ -1367,9 +1392,9 @@ def garbage_collect_repository(namespace_name, repository_name): def get_tag_image(namespace_name, repository_name, tag_name): def limit_to_tag(query): return (query - .switch(Image) - .join(RepositoryTag) - .where(RepositoryTag.name == tag_name)) + .switch(Image) + .join(RepositoryTag) + .where(RepositoryTag.name == tag_name)) images = _get_repository_images_base(namespace_name, repository_name, limit_to_tag) if not images: @@ -1407,22 +1432,17 @@ def get_parent_images(namespace_name, repository_name, image_obj): def create_or_update_tag(namespace_name, repository_name, tag_name, tag_docker_image_id): try: - repo = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + repo = _get_repository(namespace_name, repository_name) except Repository.DoesNotExist: - raise DataModelException('Invalid repository %s/%s' % - (namespace_name, repository_name)) + raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name)) try: - image = Image.get(Image.docker_image_id == tag_docker_image_id, - Image.repository == repo) + image = Image.get(Image.docker_image_id == tag_docker_image_id, Image.repository == repo) except Image.DoesNotExist: - raise DataModelException('Invalid image with id: %s' % - tag_docker_image_id) + raise DataModelException('Invalid image with id: %s' % tag_docker_image_id) try: - tag = RepositoryTag.get(RepositoryTag.repository == repo, - RepositoryTag.name == tag_name) + tag = RepositoryTag.get(RepositoryTag.repository == repo, RepositoryTag.name == tag_name) tag.image = image tag.save() except RepositoryTag.DoesNotExist: @@ -1432,41 +1452,46 @@ def create_or_update_tag(namespace_name, repository_name, tag_name, def delete_tag(namespace_name, repository_name, tag_name): - joined = RepositoryTag.select().join(Repository) - found = list(joined.where(Repository.name == repository_name, - Repository.namespace == namespace_name, - RepositoryTag.name == tag_name)) + try: + found = (RepositoryTag + .select() + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + RepositoryTag.name == tag_name) + .get()) - if not found: + except RepositoryTag.DoesNotExist: msg = ('Invalid repository tag \'%s\' on repository \'%s/%s\'' % (tag_name, namespace_name, repository_name)) raise DataModelException(msg) - found[0].delete_instance() + found.delete_instance() def delete_all_repository_tags(namespace_name, repository_name): try: - repo = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + repo = _get_repository(namespace_name, repository_name) except Repository.DoesNotExist: raise DataModelException('Invalid repository \'%s/%s\'' % (namespace_name, repository_name)) RepositoryTag.delete().where(RepositoryTag.repository == repo.id).execute() -def __entity_permission_repo_query(entity_id, entity_table, - entity_id_property, namespace_name, +def __entity_permission_repo_query(entity_id, entity_table, entity_id_property, namespace_name, repository_name): """ This method works for both users and teams. """ - selected = RepositoryPermission.select(entity_table, Repository, Role, - RepositoryPermission) - with_user = selected.join(entity_table) - with_role = with_user.switch(RepositoryPermission).join(Role) - with_repo = with_role.switch(RepositoryPermission).join(Repository) - return with_repo.where(Repository.name == repository_name, - Repository.namespace == namespace_name, - entity_id_property == entity_id) + + return (RepositoryPermission + .select(entity_table, Repository, Namespace, Role, RepositoryPermission) + .join(entity_table) + .switch(RepositoryPermission) + .join(Role) + .switch(RepositoryPermission) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + entity_id_property == entity_id)) def get_user_reponame_permission(username, namespace_name, repository_name): @@ -1514,8 +1539,7 @@ def delete_team_permission(team_name, namespace_name, repository_name): def __set_entity_repo_permission(entity, permission_entity_property, namespace_name, repository_name, role_name): - repo = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + repo = _get_repository(namespace_name, repository_name) new_role = Role.get(Role.name == role_name) # Fetch any existing permission for this entity on the repo @@ -1566,15 +1590,18 @@ def purge_repository(namespace_name, repository_name): garbage_collect_repository(namespace_name, repository_name) # Delete the rest of the repository metadata - fetched = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + fetched = _get_repository(namespace_name, repository_name) fetched.delete_instance(recursive=True) def get_private_repo_count(username): - joined = Repository.select().join(Visibility) - return joined.where(Repository.namespace == username, - Visibility.name == 'private').count() + return (Repository + .select() + .join(Visibility) + .switch(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == username, Visibility.name == 'private') + .count()) def create_access_token(repository, role): @@ -1587,22 +1614,23 @@ def create_access_token(repository, role): def create_delegate_token(namespace_name, repository_name, friendly_name, role='read'): read_only = Role.get(name=role) - repo = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + repo = _get_repository(namespace_name, repository_name) new_token = AccessToken.create(repository=repo, role=read_only, friendly_name=friendly_name, temporary=False) return new_token def get_repository_delegate_tokens(namespace_name, repository_name): - return (AccessToken.select(AccessToken, Role) - .join(Repository) - .switch(AccessToken) - .join(Role) - .switch(AccessToken) - .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) - .where(Repository.name == repository_name, Repository.namespace == namespace_name, - AccessToken.temporary == False, RepositoryBuildTrigger.uuid >> None)) + return (AccessToken + .select(AccessToken, Role) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(AccessToken) + .join(Role) + .switch(AccessToken) + .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + AccessToken.temporary == False, RepositoryBuildTrigger.uuid >> None)) def get_repo_delegate_token(namespace_name, repository_name, code): @@ -1636,14 +1664,17 @@ def delete_delegate_token(namespace_name, repository_name, code): def load_token_data(code): """ Load the permissions for any token by code. """ - selected = AccessToken.select(AccessToken, Repository, Role) - with_role = selected.join(Role) - with_repo = with_role.switch(AccessToken).join(Repository) - fetched = list(with_repo.where(AccessToken.code == code)) + try: + return (AccessToken + .select(AccessToken, Repository, Namespace, Role) + .join(Role) + .switch(AccessToken) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(AccessToken.code == code) + .get()) - if fetched: - return fetched[0] - else: + except AccessToken.DoesNotExist: raise InvalidTokenException('Invalid delegate token code: %s' % code) @@ -1660,15 +1691,15 @@ def get_repository_build(namespace_name, repository_name, build_uuid): def list_repository_builds(namespace_name, repository_name, limit, include_inactive=True): query = (RepositoryBuild - .select(RepositoryBuild, RepositoryBuildTrigger, BuildTriggerService) - .join(Repository) - .switch(RepositoryBuild) - .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) - .join(BuildTriggerService, JOIN_LEFT_OUTER) - .where(Repository.name == repository_name, - Repository.namespace == namespace_name) - .order_by(RepositoryBuild.started.desc()) - .limit(limit)) + .select(RepositoryBuild, RepositoryBuildTrigger, BuildTriggerService) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(RepositoryBuild) + .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) + .join(BuildTriggerService, JOIN_LEFT_OUTER) + .where(Repository.name == repository_name, Namespace.username == namespace_name) + .order_by(RepositoryBuild.started.desc()) + .limit(limit)) if not include_inactive: query = query.where(RepositoryBuild.phase != 'error', @@ -1732,16 +1763,17 @@ def create_repo_notification(repo, event_name, method_name, config): def get_repo_notification(namespace_name, repository_name, uuid): - joined = RepositoryNotification.select().join(Repository) - found = list(joined.where(Repository.namespace == namespace_name, - Repository.name == repository_name, - RepositoryNotification.uuid == uuid)) - - if not found: + try: + return (RepositoryNotification + .select(RepositoryNotification, Repository, Namespace) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name, + RepositoryNotification.uuid == uuid) + .get()) + except RepositoryNotification.DoesNotExist: raise InvalidNotificationException('No repository notification found with id: %s' % uuid) - return found[0] - def delete_repo_notification(namespace_name, repository_name, uuid): found = get_repo_notification(namespace_name, repository_name, uuid) @@ -1750,15 +1782,19 @@ def delete_repo_notification(namespace_name, repository_name, uuid): def list_repo_notifications(namespace_name, repository_name, event_name=None): - joined = RepositoryNotification.select().join(Repository) - where = joined.where(Repository.namespace == namespace_name, - Repository.name == repository_name) + query = (RepositoryNotification + .select(RepositoryNotification, Repository, Namespace) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name)) if event_name: - event = ExternalNotificationEvent.get(ExternalNotificationEvent.name == event_name) - where = where.where(RepositoryNotification.event == event) + query = (query + .switch(RepositoryNotification) + .join(ExternalNotificationEvent) + .where(ExternalNotificationEvent.name == event_name)) - return where + return query def list_logs(start_time, end_time, performer=None, repository=None, namespace=None): @@ -1802,16 +1838,17 @@ def create_build_trigger(repo, service_name, auth_token, user, pull_robot=None): def get_build_trigger(namespace_name, repository_name, trigger_uuid): try: return (RepositoryBuildTrigger - .select(RepositoryBuildTrigger, BuildTriggerService, Repository) - .join(BuildTriggerService) - .switch(RepositoryBuildTrigger) - .join(Repository) - .switch(RepositoryBuildTrigger) - .join(User) - .where(RepositoryBuildTrigger.uuid == trigger_uuid, - Repository.namespace == namespace_name, - Repository.name == repository_name) - .get()) + .select(RepositoryBuildTrigger, BuildTriggerService, Repository, Namespace) + .join(BuildTriggerService) + .switch(RepositoryBuildTrigger) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(RepositoryBuildTrigger) + .join(User) + .where(RepositoryBuildTrigger.uuid == trigger_uuid, + Namespace.username == namespace_name, + Repository.name == repository_name) + .get()) except RepositoryBuildTrigger.DoesNotExist: msg = 'No build trigger with uuid: %s' % trigger_uuid raise InvalidBuildTriggerException(msg) @@ -1819,12 +1856,12 @@ def get_build_trigger(namespace_name, repository_name, trigger_uuid): def list_build_triggers(namespace_name, repository_name): return (RepositoryBuildTrigger - .select(RepositoryBuildTrigger, BuildTriggerService, Repository) - .join(BuildTriggerService) - .switch(RepositoryBuildTrigger) - .join(Repository) - .where(Repository.namespace == namespace_name, - Repository.name == repository_name)) + .select(RepositoryBuildTrigger, BuildTriggerService, Repository) + .join(BuildTriggerService) + .switch(RepositoryBuildTrigger) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name)) def list_trigger_builds(namespace_name, repository_name, trigger_uuid, @@ -1913,6 +1950,7 @@ def delete_notifications_by_kind(target, kind_name): Notification.delete().where(Notification.target == target, Notification.kind == kind_ref).execute() + def delete_matching_notifications(target, kind_name, **kwargs): kind_ref = NotificationKind.get(name=kind_name) @@ -1943,6 +1981,7 @@ def delete_matching_notifications(target, kind_name, **kwargs): def get_active_users(): return User.select().where(User.organization == False, User.robot == False) + def get_active_user_count(): return get_active_users().count() @@ -1956,11 +1995,13 @@ def detach_external_login(user, service_name): FederatedLogin.delete().where(FederatedLogin.user == user, FederatedLogin.service == service).execute() + def delete_user(user): user.delete_instance(recursive=True, delete_nullable=True) # TODO: also delete any repository data associated + def check_health(): # We will connect to the db, check that it contains some log entry kinds try: @@ -1969,24 +2010,23 @@ def check_health(): except: return False -def get_email_authorized_for_repo(namespace, repository, email): - found = list(RepositoryAuthorizedEmail.select() - .join(Repository) - .where(Repository.namespace == namespace, - Repository.name == repository, - RepositoryAuthorizedEmail.email == email) - .switch(RepositoryAuthorizedEmail) - .limit(1)) - if not found or len(found) < 1: - return None - return found[0] +def get_email_authorized_for_repo(namespace, repository, email): + try: + return (RepositoryAuthorizedEmail + .select(RepositoryAuthorizedEmail, Repository, Namespace) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace, Repository.name == repository, + RepositoryAuthorizedEmail.email == email) + .get()) + except RepositoryAuthorizedEmail.DoesNotExist: + return None def create_email_authorization_for_repo(namespace_name, repository_name, email): try: - repo = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + repo = _get_repository(namespace_name, repository_name) except Repository.DoesNotExist: raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name)) @@ -1996,7 +2036,11 @@ def create_email_authorization_for_repo(namespace_name, repository_name, email): def confirm_email_authorization_for_repo(code): try: - found = RepositoryAuthorizedEmail.get(RepositoryAuthorizedEmail.code == code) + found = (RepositoryAuthorizedEmail + .select(RepositoryAuthorizedEmail, Repository, Namespace) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .get(RepositoryAuthorizedEmail.code == code)) except RepositoryAuthorizedEmail.DoesNotExist: raise DataModelException('Invalid confirmation code.') diff --git a/endpoints/api/build.py b/endpoints/api/build.py index d792234dd..adf6f43ec 100644 --- a/endpoints/api/build.py +++ b/endpoints/api/build.py @@ -169,7 +169,7 @@ class RepositoryBuildList(RepositoryParamResource): # was used. associated_repository = model.get_repository_for_resource(dockerfile_id) if associated_repository: - if not ModifyRepositoryPermission(associated_repository.namespace, + if not ModifyRepositoryPermission(associated_repository.namespace_user.username, associated_repository.name): raise Unauthorized() diff --git a/endpoints/api/repoemail.py b/endpoints/api/repoemail.py index 4dbc845a7..db4c9b571 100644 --- a/endpoints/api/repoemail.py +++ b/endpoints/api/repoemail.py @@ -20,7 +20,7 @@ def record_view(record): return { 'email': record.email, 'repository': record.repository.name, - 'namespace': record.repository.namespace, + 'namespace': record.repository.namespace_user.username, 'confirmed': record.confirmed } diff --git a/endpoints/api/repository.py b/endpoints/api/repository.py index 4e85a35b5..be8c9e8f9 100644 --- a/endpoints/api/repository.py +++ b/endpoints/api/repository.py @@ -80,8 +80,7 @@ class RepositoryList(ApiResource): visibility = req['visibility'] - repo = model.create_repository(namespace_name, repository_name, owner, - visibility) + repo = model.create_repository(namespace_name, repository_name, owner, visibility) repo.description = req['description'] repo.save() @@ -110,7 +109,7 @@ class RepositoryList(ApiResource): """Fetch the list of repositories under a variety of situations.""" def repo_view(repo_obj): return { - 'namespace': repo_obj.namespace, + 'namespace': repo_obj.namespace_user.username, 'name': repo_obj.name, 'description': repo_obj.description, 'is_public': repo_obj.visibility.name == 'public', @@ -134,7 +133,8 @@ class RepositoryList(ApiResource): response['repositories'] = [repo_view(repo) for repo in repo_query if (repo.visibility.name == 'public' or - ReadRepositoryPermission(repo.namespace, repo.name).can())] + ReadRepositoryPermission(repo.namespace_user.username, + repo.name).can())] return response diff --git a/endpoints/api/search.py b/endpoints/api/search.py index 7cb1a1fda..1cce618d9 100644 --- a/endpoints/api/search.py +++ b/endpoints/api/search.py @@ -111,7 +111,7 @@ class FindRepositories(ApiResource): def repo_view(repo): return { - 'namespace': repo.namespace, + 'namespace': repo.namespace_user.username, 'name': repo.name, 'description': repo.description } @@ -125,5 +125,5 @@ class FindRepositories(ApiResource): return { 'repositories': [repo_view(repo) for repo in matching if (repo.visibility.name == 'public' or - ReadRepositoryPermission(repo.namespace, repo.name).can())] + ReadRepositoryPermission(repo.namespace_user.username, repo.name).can())] } diff --git a/endpoints/api/trigger.py b/endpoints/api/trigger.py index 4ec20bfdc..ccccf8010 100644 --- a/endpoints/api/trigger.py +++ b/endpoints/api/trigger.py @@ -205,7 +205,7 @@ class BuildTriggerActivate(RepositoryParamResource): 'write') try: - repository_path = '%s/%s' % (trigger.repository.namespace, + repository_path = '%s/%s' % (trigger.repository.namespace_user.username, trigger.repository.name) path = url_for('webhooks.build_trigger_webhook', repository=repository_path, trigger_uuid=trigger.uuid) diff --git a/endpoints/common.py b/endpoints/common.py index 1c6439371..37ae80ee8 100644 --- a/endpoints/common.py +++ b/endpoints/common.py @@ -205,7 +205,7 @@ def check_repository_usage(user_or_org, plan_found): def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, trigger=None, pull_robot_name=None): host = urlparse.urlparse(request.url).netloc - repo_path = '%s/%s/%s' % (host, repository.namespace, repository.name) + repo_path = '%s/%s/%s' % (host, repository.namespace_user.username, repository.name) token = model.create_access_token(repository, 'write') logger.debug('Creating build %s with repo %s tags %s and dockerfile_id %s', @@ -221,9 +221,9 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, dockerfile_id, build_name, trigger, pull_robot_name=pull_robot_name) - dockerfile_build_queue.put([repository.namespace, repository.name], json.dumps({ + dockerfile_build_queue.put([repository.namespace_user.username, repository.name], json.dumps({ 'build_uuid': build_request.uuid, - 'namespace': repository.namespace, + 'namespace': repository.namespace_user.username, 'repository': repository.name, 'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None }), retries_remaining=1) @@ -231,7 +231,7 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, # Add the build to the repo's log. metadata = { 'repo': repository.name, - 'namespace': repository.namespace, + 'namespace': repository.namespace_user.username, 'fileid': dockerfile_id, 'manual': manual, } @@ -241,9 +241,8 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, metadata['config'] = json.loads(trigger.config) metadata['service'] = trigger.service.name - model.log_action('build_dockerfile', repository.namespace, - ip=request.remote_addr, metadata=metadata, - repository=repository) + model.log_action('build_dockerfile', repository.namespace_user.username, ip=request.remote_addr, + metadata=metadata, repository=repository) # Add notifications for the build queue. profile.debug('Adding notifications for repository') diff --git a/endpoints/index.py b/endpoints/index.py index 5c8d7058a..46c5b9771 100644 --- a/endpoints/index.py +++ b/endpoints/index.py @@ -420,7 +420,7 @@ def put_repository_auth(namespace, repository): def get_search(): def result_view(repo): return { - "name": repo.namespace + '/' + repo.name, + "name": repo.namespace_user.username + '/' + repo.name, "description": repo.description } @@ -438,7 +438,7 @@ def get_search(): results = [result_view(repo) for repo in matching if (repo.visibility.name == 'public' or - ReadRepositoryPermission(repo.namespace, repo.name).can())] + ReadRepositoryPermission(repo.namespace_user.username, repo.name).can())] data = { "query": query, diff --git a/endpoints/notificationevent.py b/endpoints/notificationevent.py index f3f4d6a77..4a195fbd7 100644 --- a/endpoints/notificationevent.py +++ b/endpoints/notificationevent.py @@ -1,8 +1,4 @@ import logging -import io -import os.path -import tarfile -import base64 from notificationhelper import build_event_data diff --git a/endpoints/notificationhelper.py b/endpoints/notificationhelper.py index 773779fb7..6f80f83d0 100644 --- a/endpoints/notificationhelper.py +++ b/endpoints/notificationhelper.py @@ -4,7 +4,7 @@ from data import model import json def build_event_data(repo, extra_data={}, subpage=None): - repo_string = '%s/%s' % (repo.namespace, repo.name) + repo_string = '%s/%s' % (repo.namespace_user.username, repo.name) homepage = '%s://%s/repository/%s' % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'], repo_string) @@ -17,7 +17,7 @@ def build_event_data(repo, extra_data={}, subpage=None): event_data = { 'repository': repo_string, - 'namespace': repo.namespace, + 'namespace': repo.namespace_user.username, 'name': repo.name, 'docker_url': '%s/%s' % (app.config['SERVER_HOSTNAME'], repo_string), 'homepage': homepage, @@ -30,7 +30,7 @@ def build_event_data(repo, extra_data={}, subpage=None): def build_notification_data(notification, event_data): return { 'notification_uuid': notification.uuid, - 'repository_namespace': notification.repository.namespace, + 'repository_namespace': notification.repository.namespace_user.username, 'repository_name': notification.repository.name, 'event_data': event_data } @@ -39,8 +39,9 @@ def build_notification_data(notification, event_data): def spawn_notification(repo, event_name, extra_data={}, subpage=None, pathargs=[]): event_data = build_event_data(repo, extra_data=extra_data, subpage=subpage) - notifications = model.list_repo_notifications(repo.namespace, repo.name, event_name=event_name) + notifications = model.list_repo_notifications(repo.namespace_user.username, repo.name, + event_name=event_name) for notification in notifications: notification_data = build_notification_data(notification, event_data) - path = [repo.namespace, repo.name, event_name] + pathargs + path = [repo.namespace_user.username, repo.name, event_name] + pathargs notification_queue.put(path, json.dumps(notification_data)) diff --git a/endpoints/notificationmethod.py b/endpoints/notificationmethod.py index e6c259cb1..589ebd06d 100644 --- a/endpoints/notificationmethod.py +++ b/endpoints/notificationmethod.py @@ -88,7 +88,7 @@ class QuayNotificationMethod(NotificationMethod): return (True, 'Unknown organization %s' % target_info['name'], None) # Only repositories under the organization can cause notifications to that org. - if target_info['name'] != repository.namespace: + if target_info['name'] != repository.namespace_user.username: return (False, 'Organization name must match repository namespace') return (True, None, [target]) @@ -96,7 +96,7 @@ class QuayNotificationMethod(NotificationMethod): # Lookup the team. team = None try: - team = model.get_organization_team(repository.namespace, target_info['name']) + team = model.get_organization_team(repository.namespace_user.username, target_info['name']) except model.InvalidTeamException: # Probably deleted. return (True, 'Unknown team %s' % target_info['name'], None) @@ -133,7 +133,8 @@ class EmailMethod(NotificationMethod): if not email: raise CannotValidateNotificationMethodException('Missing e-mail address') - record = model.get_email_authorized_for_repo(repository.namespace, repository.name, email) + record = model.get_email_authorized_for_repo(repository.namespace_user.username, + repository.name, email) if not record or not record.confirmed: raise CannotValidateNotificationMethodException('The specified e-mail address ' 'is not authorized to receive ' @@ -210,7 +211,7 @@ class FlowdockMethod(NotificationMethod): if not token: return - owner = model.get_user(notification.repository.namespace) + owner = model.get_user(notification.repository.namespace_user.username) if not owner: # Something went wrong. return @@ -223,7 +224,8 @@ class FlowdockMethod(NotificationMethod): 'subject': event_handler.get_summary(notification_data['event_data'], notification_data), 'content': event_handler.get_message(notification_data['event_data'], notification_data), 'from_name': owner.username, - 'project': notification.repository.namespace + ' ' + notification.repository.name, + 'project': (notification.repository.namespace_user.username + ' ' + + notification.repository.name), 'tags': ['#' + event_handler.event_name()], 'link': notification_data['event_data']['homepage'] } @@ -265,7 +267,7 @@ class HipchatMethod(NotificationMethod): if not token or not room_id: return - owner = model.get_user(notification.repository.namespace) + owner = model.get_user(notification.repository.namespace_user.username) if not owner: # Something went wrong. return @@ -332,7 +334,7 @@ class SlackMethod(NotificationMethod): if not token or not subdomain: return - owner = model.get_user(notification.repository.namespace) + owner = model.get_user(notification.repository.namespace_user.username) if not owner: # Something went wrong. return diff --git a/endpoints/web.py b/endpoints/web.py index 94c5583cd..63e463666 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -237,8 +237,8 @@ def confirm_repo_email(): Your E-mail address has been authorized to receive notifications for repository %s/%s. """ % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'], - record.repository.namespace, record.repository.name, - record.repository.namespace, record.repository.name) + record.repository.namespace_user.username, record.repository.name, + record.repository.namespace_user.username, record.repository.name) return render_page_template('message.html', message=message) diff --git a/initdb.py b/initdb.py index 5ae3c1ff2..cfb67a39a 100644 --- a/initdb.py +++ b/initdb.py @@ -51,7 +51,7 @@ def __gen_checksum(image_id): def __gen_image_id(repo, image_num): - str_to_hash = "%s/%s/%s" % (repo.namespace, repo.name, image_num) + str_to_hash = "%s/%s/%s" % (repo.namespace_user.username, repo.name, image_num) h = hashlib.md5(str_to_hash) return h.hexdigest() + h.hexdigest() @@ -79,11 +79,10 @@ def __create_subtree(repo, structure, creator_username, parent): creation_time = REFERENCE_DATE + timedelta(days=image_num) command_list = SAMPLE_CMDS[image_num % len(SAMPLE_CMDS)] command = json.dumps(command_list) if command_list else None - new_image = model.set_image_metadata(docker_image_id, repo.namespace, - repo.name, str(creation_time), - 'no comment', command, 0, parent) + new_image = model.set_image_metadata(docker_image_id, repo.namespace_user.username, repo.name, + str(creation_time), 'no comment', command, 0, parent) - model.set_image_size(docker_image_id, repo.namespace, repo.name, + model.set_image_size(docker_image_id, repo.namespace_user.username, repo.name, random.randrange(1, 1024 * 1024 * 1024)) # Populate the diff file @@ -100,7 +99,7 @@ def __create_subtree(repo, structure, creator_username, parent): last_node_tags = [last_node_tags] for tag_name in last_node_tags: - model.create_or_update_tag(repo.namespace, repo.name, tag_name, + model.create_or_update_tag(repo.namespace_user.username, repo.name, tag_name, new_image.docker_image_id) for subtree in subtrees: @@ -326,7 +325,8 @@ def populate_database(): outside_org.verified = True outside_org.save() - model.create_notification('test_notification', new_user_1, metadata={'some': 'value', 'arr': [1,2,3], 'obj': {'a': 1, 'b': 2}}) + model.create_notification('test_notification', new_user_1, + metadata={'some':'value', 'arr':[1, 2, 3], 'obj':{'a':1, 'b':2}}) from_date = datetime.utcnow() to_date = from_date + timedelta(hours=1) @@ -390,18 +390,20 @@ def populate_database(): }) trigger.save() - repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name) + repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace_user.username, building.name) job_config = { 'repository': repo, 'docker_tags': ['latest'], 'build_subdir': '', } - record = model.create_email_authorization_for_repo(new_user_1.username, 'simple', 'jschorr@devtable.com') + record = model.create_email_authorization_for_repo(new_user_1.username, 'simple', + 'jschorr@devtable.com') record.confirmed = True record.save() - model.create_email_authorization_for_repo(new_user_1.username, 'simple', 'jschorr+other@devtable.com') + model.create_email_authorization_for_repo(new_user_1.username, 'simple', + 'jschorr+other@devtable.com') build2 = model.create_repository_build(building, token, job_config, '68daeebd-a5b9-457f-80a0-4363b882f8ea', @@ -428,12 +430,12 @@ def populate_database(): model.create_robot('coolrobot', org) - oauth.create_application(org, 'Some Test App', 'http://localhost:8000', 'http://localhost:8000/o2c.html', - client_id='deadbeef') + oauth.create_application(org, 'Some Test App', 'http://localhost:8000', + 'http://localhost:8000/o2c.html', client_id='deadbeef') - oauth.create_application(org, 'Some Other Test App', 'http://quay.io', 'http://localhost:8000/o2c.html', - client_id='deadpork', - description = 'This is another test application') + oauth.create_application(org, 'Some Other Test App', 'http://quay.io', + 'http://localhost:8000/o2c.html', client_id='deadpork', + description='This is another test application') model.oauth.create_access_token_for_testing(new_user_1, 'deadbeef', 'repo:admin') @@ -455,8 +457,8 @@ def populate_database(): reader_team = model.create_team('readers', org, 'member', 'Readers of orgrepo.') - model.set_team_repo_permission(reader_team.name, org_repo.namespace, - org_repo.name, 'read') + model.set_team_repo_permission(reader_team.name, org_repo.namespace_user.username, org_repo.name, + 'read') model.add_user_to_team(new_user_2, reader_team) model.add_user_to_team(reader, reader_team) @@ -478,12 +480,9 @@ def populate_database(): (2, [], 'latest17'), (2, [], 'latest18'),]) - model.add_prototype_permission(org, 'read', activating_user=new_user_1, - delegate_user=new_user_2) - model.add_prototype_permission(org, 'read', activating_user=new_user_1, - delegate_team=reader_team) - model.add_prototype_permission(org, 'write', activating_user=new_user_2, - delegate_user=new_user_1) + model.add_prototype_permission(org, 'read', activating_user=new_user_1, delegate_user=new_user_2) + model.add_prototype_permission(org, 'read', activating_user=new_user_1, delegate_team=reader_team) + model.add_prototype_permission(org, 'write', activating_user=new_user_2, delegate_user=new_user_1) today = datetime.today() week_ago = today - timedelta(6) diff --git a/test/data/test.db b/test/data/test.db index afd5fe2386371bbfabab5992cbaa59e5c7aa18eb..af85d23e2016a9254ab3ad3b01876942f8d50c4f 100644 GIT binary patch delta 9922 zcmeI2dvqK1mB%%DSQ@_+#UXK=*iszl;Ur_t`(g5uZ22YGdf19>MUd#31ixa(c4P-| zhIXL~WeFHE5SBoIrO*;sC~E^rd6YeDyDbo)d+3(eE`>u&fdzKUqk*t@Mz&_01P&!< z|JZZTA38_+{pQa1e(wF<`RRAHfB&lePpmv(M-DyUcynN@(fY0qUWcqFtsh%Iw7y$- zQ^6+%9r?e@--N!89}@jRYr^C%0pL}Nt~cCoRe&ipyiYE_T^d2%btG2Uy+@q zgIU1}bOkyy3nQf`uR|Bus^gVA#nSQu#gteMv#vFxEAn%*2Jwt{UVZ;19{jaHdaF8=>mhE_{YSkVo| zPq;pe7OA>rTDe)fZ=A`J6*vCVHh*Exi8c z&;BzWHLGqYDBm4lV_nnp=9E7#r#y@D(7tlpijIxVVyQ`$E2ErTe{srMc;tp3WsIsc z8Qjd0U|~rF3(*C#Bh|M2D^_?!he0D%rZ@S)C4Dy9W@!=adR3|h85p6Z2X>%wYblTX zeH2kuYk;xx5-nK(EBVGTcssZi-mcvMZwKs`?Vs9@w2eo)eJyQ`E{~@{sCTuw$*!hO z&P6jM;qz|kaMf|$L5lC{Ztm!A^0hWmWGfHv92I2UR1+O=hdTnWerXIBK#hYWY8kec zBx@Lk#~DJp`!7gkgbfGCKp@f;BpQNlk^@U!9c)8~hhVvSPlLaSZV_l-kZdAam|$C{ zpKa(2dct@;O|~;FZlZ}ht9_c_1yZUyiPYJ&w87?2Juouf!F4D39Sv;Q zbYRs7$9T?3-+~Y&x{;{EvZq}Qu=M$~n{GLopfBRq>Y%+??91udN5qOHEY3Wa& z1CXQ;Pw>548us!%-Vq<&v$H+YH#{=JH#8;F=~&}!|()#Yws{o$}L9CDu#Sy0UKI7{Nv+h2gS6`&VbYhW3XF(w;lBv<*SiCp2HaR>r5FZ%!cDMRl+FJ$QMdNH;XCS-< z_qO=pG6@uYNYrxg%uWwG_(8b1ejg9-~-}kGV>SAno^#DDQ@LRiBIm+l z>HALoa?iPjL%+ztpfbG$Q@H-6bBDF>60gCmLlCjZR$%z+nN7^Ckzh?fW6fkZS9D+u znK7m}d>u9Wl*vLN)U0G-CC?5h|0 z)B*j;w$)9HFW~V9+bF^#kX?}g?GG|tktPq*MiAjpT_DodQ3uc5ko5TLdf21BuJ^Cz z{e+JV;akW^Q>Y`*8EWaK$ySC6Q|@ju%r#N%Att<~+toY`mwez1a!ia$CkYP26P$q2 zafZWqBE?{7mZ1fB%HkZE9!l*VsEsB2cJ&XZ_6!dX^rZT8b%=4`cr8uU;_~C2Bv^cE zp5Ya1)i*JF&|$#Q;+w5E>b?(K59x@*@P)Ly+F+D!tu!Rnk5|CQ>02v}<rqNn!CA9>)kGkxZo-fx@8-{>P<8e4;azT<3tjM|*0wL)JRG3Bes04zJ2} zj>_SIZOPbhsn#agusYK1l5HRR5ZCP42!2?bezJ3 zGA2P`v=Ha8I7_854n)HfmrW*8ap|53V*}J$jU9?(=&Ak2JO}@@4tZMtlwqx5(pX{o zab8*80kh7$1HCifm;aZ7+Y8qfK2kJM^hI&PdbxGG?vU;va8q7V+ik|EwEZYjBptZZ zxK*EaOV8eJTqL#LVI-vIju^+M;q7oCS{5=dTDA=Nn+a+3KI1`Y|F4lkssA~oVj8>O zc*i_^jw;zZeYwFU?@nj8a;J-a!uV5b>opp zrZ74c6EJ~^C9p)CCt*j#kt7i}=$(3t4hf@BFY{A0!KQHnk|IeaFq$PpcNk$-Wf@I?Hm=x=jHlBC z&e4nzqZ87vdre#wc*ipdI*G-Y1k6luEaY^Iz$i8im!)x4QE~&2oaK6L<>8V8y_ihs`V|BxoEE3qp#5AMvTAAi4U%B$G&{Sw2n4uHtZWFfPOh zP{1;2jv?4oib+U!_nXS$<~*MuQ!y+7(ZOgk9>-!d!(cHYNzz1&OebQr^v8Zva}~iR z2?5pumWE#g7)_HjCJ1yI!%2z|ID$=p$I_Mo)7s@UL&w2&z?r0ACMiqAc_>wYoJnyB zIzdZ!4w(2Vj%Cvs9m0b^oZl7+IwvjQ=-+hp2ap!-s%`w|(`>-0a6 z=AAGtL#~^&IU1$06Q;|j^PV?diJ;|M%!~ZX5V2FTy&2~c(|T)F7E^m*L0A%&Ld4!5Sx_o6N7wnG985 z;HN*{WFG5Lzo%+AoE30F+W!;tb^dQ6qwa$FM>&hhlsM&p%F}S!|dg!Qm@!7{V&nNL&6j~i+Qp~O)?59fdVFnap%ME0RLYu$4C_}pRhoZ&N|=4pqCGq{@0GCTq23$r&j&KW<0 zvEl27=REO||Mw7 z_JkRZ^OD|t<%TcmtS?#L{i)6ID{J)v1BRS#L|P)wpRvND|)Nqp~=mowA9**ZZ3#!n0xjtUD5~5mF2@*KlZOzkIa=O$J%~qt_YLI z=ZU4~t7qoQ*}U{jKQve9_gi(DTJ_XiKAN9tB=FMXC|Z562VH1ex0JeGbO@RwbEOmX z>O5$#*gqui745uwW}fW?J!gaVO3mQb{wSAIt8{`MpATZ4uU@##&#GdTPEcP7h^@Z& z(_*od(TG)wgLMIjt(xe2Cqw7N%7^pPrwgD{wem`-S#(eueVQ|QUOKW5tybDA8}FWU zh;~w=Ri-r4iQ(bbZuWrRc=GuED3Oz`?VG_QTspE7by&~ud+U8auDT$To9Q?37%n}D zLJMxi7svf#sZ-;O%x#wMY2juo-zP0 z99aFBXkVvsMxl7~611;)zNI9(R&_u@c-2w>Z@={ozkf|mwSrJ62kyG#_0Nc<)f%Zv z2kqr@;J%W5^kinWDo=)%bHX;3&5^S8`A>~IJ!cWswCF42uQ7U+;-4km6Mu<{YE!QA-}hZC+8wHTO;-qvX1hYh)$H|mPB~EH=B@wN zMZmj9E&^LsW=f>?4Ffae=tmp<5tW%HQi62k7%b%_ca8p5EbUg=WyAEV5rC)r?Xllw zx>R_YVjxHlP0ZMzOSABUZ`e2gX5c!7tJuWI{QyN+uuL2TR7V@7VLA zBdF0Q^O=dPATRPutp}AiUeOix=aehNW|q2;UFu7eu6&6! zc!9r7bwtY#7W+cmCVRs_i>0j^e`ItyY0wE)TJlu>fA?m5IoA}1GwV%|(z@TzVqK%a|26!;~3-xlpH8n+arPsSk%YgboTqRpx=S)d6B!?KA5xB8oM>J^~d zk`M)J=fOU)v{9p4NjM}0QK&usgXNh9RjSNSdM5>8$SeQzL!!f@>YH0eDO?SM{n0y) ziuQVq-Z{k-{?CA-tK`1-dTX)oRa>-9l|QGR9@(e#s?AC;3Lo5%@Vj&FC=hd%o>Y47 zCAKx&uiGk?x>VO>h`B1Rc8zD+-{mhzpJXLCg0m818OUdVJ4~nIOv!2ZU=y>j0NaC{Us>PYzIalQHJr(WCd7~*QFiF`xWJ}8Y z(N#b9_o@zQDJvyqVQLymWRJ!#wWP?VW=hKZ;}2eyxhUsiwk#i(AahN+djg_km+G5R zM|Rx^0YVS2`&ZGvQ{#)mbL&k|M@af}YjlU|fRgUsLy%4OsZgFjos+C2`)7wB1gq!& zWs_K%(nwWO@aCH#1SJQj)@72aJ{21L)gfCe*4yxF9(-1GBvh%{XP5sL(CQ3Kz>_Dg z(JG^oT!%p`x@J{48dJ3@DM;K3T9JP_=g%8_9)}0nfm+@Ftzc2(2D;B{t?{% delta 20540 zcmeHvX>=Udb!IoZfkqP>85i5(zzL=siYD{ukOT){!e3GNflRmL=!R#EunqGjNre|RlQU7 ztEzvj`d?K)uX?le6Nzt@{Erf~_@Ve?vCkL%RrHt9-pEst!`W1A^Db%F|NBTJNfgyK zgrucqW{R{bF_Jb_LeskCsM?&&P~EyUWszZbJK3}?9?8l_ZXTUAjvhNcU3YB1p=flM zVaQw|%odI+QA)T(Emd`NuG*^5RnEfXiX@Y?#L}{Nq%3)(C&iN`sqJ5U`mf!Z>VPpIIO)EM zWhYsttvi~mn6j+m{|m(+g5gdYRV7O{%`ka3daA9OKXmt7Zl)@@R2qu>#IgK4nGUB4 z6GXREO_mJHU=Gn8OVcD%b}Y)nQrmNFuzyG*9QyH98hm-!{Z2n0W$48sKmhjM2 zTQiA?3rdbE+m5X|N|-!bdwME$4p`3GqlWtO39)F7}$!*NWD!oqbX zQ?3e5!sOW(Pw!Y={j*BjttnqLLhruANp-hs=*&^5BFnM?qt`V@rG~>erLcd=vclxq znll@#YcBF1yOU*swqR`83y1SB!QfU@b`-^y2-9I}hAwfcF&rcuuIXX&?2$7fl}xpk zxb3TQY^}d!nXQ?48m6qw;ugHwIG zL#W8}T4EcrLKLoQDyKG;eC6qqV@uGc&MiGmp56YyRO(LeQ*XF+i6y%$6dw#r5K4tC znF880RKhvJtIj3WV3trxm^}NP2X>^?k->j#C_5hN%fd zRTUg*XxuRwQN!f(nxtt(&u%)grkegZz1y8E4)o|-K4 z2qUI}kQgS6yJTuWHU9#b)und^K!xRVc2x-L+6->AEF!{pe zyes!4rEJ6Ts_Ly3H#lx47F^Fa&gD_V=CY-ks!UA65z;x8bW^u9=BTFS=#CjC&pv!^ zI>mqX$=lqSqTr;u_*~fNFjZBh3PqeS1;uA;luCF`Lb6R+5;_f&FFvkp^!T=H|LHqc zuX)mT>!M5M=tq$A*V+hd-BN5DpDmlVqA`Sb%~W*cX5d-nE`vCi(% zfxgzxk&cdmw$8qO-Z3yT&^Fr4hB|r%)uYKV@w^>lxD4qH?lt7#XHqzeQ1=l@;=hs z)VOzKkTmJ7dxlx_@Oax`lhoTa(p&KJ@xY1dZG9)&oXR~`QxuhZr{0OyYNv*pI)<8z z_5*#L%3*VV!=A1a9Y^XrB{tGOyn84v?6kYTf99PDaC({M}=XFqgxeYtE_B2J$|a?gmYN8`-doZ4z%>Q@P-3@ z4VfM>JTfufI65Y^DP1QI9G#W=t)rc)-7<5sNz7zq@A5m*>sIF+rBD9Oyp4jK0Cf{^Om)T@wfD z`SfrXZJHYBviPB{dheyH(M{(UpA@N=sH$^zvZx}Gj<^-2k-l(3>+Gr1V|J6&&nJ$v184f1 zCmK&5v0M1iQ0w@fwg$&Ie5$2wpmChlQ}4z1qYX>iQRp5)74>c52JgX!qAjf-_Hh)gi)l$;d)Hl*ieCHh%DVq- zS1b{UM%{{3S?ph~v-8)+NPJq+!lxC-KCJ5SVbO&TD~f%9-nReRc6{mTzM@}zczwm& zi=nINzd}`WnmWIjPGSom^kKDB)aHMoc}vaZnlQndtAI z(7FZ&`g(e#_7OTfHq;~ajg5Cx(lpS$XH56Ld8O!PZ_kyYzFS$}Xp1~C+TA!lPCI)C zM_LJ+7$5E)ni$^G+t^Cww#Md`QL46G!asStVSrdDC;%L!sbX1a6-`RobWD3{ zlN6KLN5$dHF2kNaFf%KT&CX^HiJ6=@q)59|x=TWOM>R{x#nrJlsh4st~m{E#be-J8L)3X7<^} ztk}G(cJuPT*5_)L;IGf-+*c;^qI%>pMUmWiG#MF=Cf(x3Qj6=7qRM)juXiZEpW@(~ z;umQLKNxcyT?p!Z@qE0^54KC60}z5=ybv#`-SbK$`byEuF*WwN__pG&mTW2cOd^>$ zoP2iGa9OPEOXXVmw<`WkWl7~!)kBq6swN|UfWmF@zqmb+SXT!HCJ)SbcRm%ryU1zu z(w~p7^WJzm{^+Id-;IB)BAMEoShs0Y^tYzH>F45~_Fj4_T61Z~^YK4eEnV}C{F@bR zj5Q_;UwrNhzZm~&RnM>UM2fb>hLcOW`C!#tyc|DOb>8Htcqz5ggh)0(Elp0F=#|n6 zk{p^drl1Xr6*Ux1j|)Ax*6TkSz1?_lU8k=r*#J=4b(u9Z3}pXN{eC*(Yb#R zjYLaEP?3OH!H> z!$>POvo(XthU$36RB`)OCa6p`Jba;P_(R7gX#@RE+R%s)l5WA9xcBl@G22RXp`ZXW z(n5t?l`Wb!4F#>7V*)U;sAkRnc|IGEsiguS$3K* zOHZpfJ#A3UNK+Fan5@dGNi^^8W{NwuqOEl}x+eH8gOY3k-xZn#{1P3Nj3=uI*&E3e z*KJ`orD!1;uAw3@Ng6asZ8dE%t}>Yz4z<1C%M|NdIUYSAM1(Br;O08QfvLh=0Elo! z(wSwO?Be0#;`_>?C6Q=J^g>ah=wWa5<>F1zrxADBz0;SAAM&32L2=^J`nms;P6m;^WM(x*5Ri9em+Kq8i%P$SYu=7P_NWF(ArwM+_?}% z6`hewS066%_b8rNpCs zzs}d#e0Hd86r@-tK+v*~ATnWxK0ozNCt!@3G$Nd@iv)r7x4seVJ#TOVkyR{q1v!p%h#8 zPw@4w_sr)L4FzORKcCpO{MVP>dOq>&jNgg`4{83JmDj`?qlHXtUR<`ac-6|{WWM

dJ)T5^qenO=(vh|kYK4D(luUn5_p|Yln;TBc>%1^tW z@h3KwXDb^ju2fuFS6ljKRmp!C!|$_~-ft)^m!hA&RDNISzN+Y_y{SxTSM;$%K;Eqr9PF z*gz;uzMw`-ZL%+&s8teIe@kF06j>9wH&Rtqxv^rf{6N{aQ9x`@veN0qZ`of1a!$ikZ2BJ=sXQhbSg~D%){jK zUz25}2gK^Shu(MVZeALr%}sUl&cks&%B3c5 zdjIGqw`S|g!(AQR3rznNM{+QH!pNUcN09^tEyt2|s)flH4h)97H~#t7*WJl2fl?QS zyLp!c)S(N9nX*6F&`icHYNIdL3}SPTKuD;3p2w4Q&)%EbxZ!J8-1g1a&Uo{N^0uZh zKpVtTiA2E4z~maUN?eblqqJlZCijk1CflmZul{h0o7ogxt?*!1w}H4&uv!Sv3Bm;6 z3<>8Ms)gaXY=il*vfO*CGC7^vu%q^EccwNtYGJawTeBSuy@ILw}r`=beCkm@Fx_P-Vb65-35GQQ0(9#()mOAVcJf zRx#(Dshj?C_Lpu=O^zk{)Q~)OmtrwRfn!;Y;D!X#GZo+KOr{bIWGk?qLgWkVE+t!g z;=1bVpZf>Po!l6x&^N^8gfd4~48{z9?62BXVA{_qTmg(5xIDw$Iz+yBO`lL!pMCOV zeM)cX-sZM%$g#)eBoPhR8SEW6o32{Aje{hIs4x~oH<=7FOo%-D%Oh>ow+(#bbvJWk za6y=3A(n{d#|{nDH4qFa#qCV6He~43;sQJeBTSw>bac9!Jo%41-I?{lQDD9lqLIOm z6#(0gfEF#`K$$Qa4r?`aVkwRYlV>ZBO{G?S_ZL5P>u$&~&gI+?2BZ+TfrY3<*dSY} zz~uyBJz~Q)DG!rppF6fAwXUo2lWxt&mQ1n$i43eb5U@BW0>GXrz_`)`7q+Ylivg(= zVe)MG?8fT4*Z%2IcXC~zJl_zP^Fb`fkR@)(5>pX&9TkR!7^Mo;1{1+nK=@l(p4~Sq zR#$xG73Q|DT{OKpI!HTlV9<(T!KPG)a^fS(RFMn;t{MgA875z_aqq}ba&1;Rx4nA% zm9dPQxjwj%9|xBQ5X*ENN3%_&Ow(aR_5&Z1ifpR7p;@{WCJ#JICg2R+GRMEt|L))u z?#y+;xxN+pQKaWuB8mlsSfK=_ZD_V6Q^#}&(UI4s zJ5p6y*-|l4{-4XsP)yvKyti~V@q@(8C?=ZXr(&NfdaLLz6ccx&n5eIGGd%cmAK;B3 zJ2J(zeP0B*3EmDI9z0grk_o_aB=-=x_no5TbhWm=`WNnu9-I}pW1b?QX84Irg@Ujx z6ba}i(P2eU0~fNRg~+`_F(e|s?zTR+j;-j94)Aq>RE|t1OSZ0=RHC+Q65F;l#5gHT z?v=-rJ5p;)l;3b`=!*WRsfM2tY*PU`tziI&$-jhA1EyDCIw1;#$i0K{en=Cz?<#aB=;7sHmV3`kdvJ<=!{qNwHe}Lie3+yLzohb}CfD zh{;A+R&-2oWZwo16hy){2*xzvZiUF_5}lr2@Qs^}uB}%8=^FGx?gb#jhX7D(-?rs*KN5l2#y@Dy$k zoXiyq_eT5y({5#X&aw9gT=CU+z4g2wP({3gm+|jq(v~ zpSv?VgQH-t1#-SXNd!(2&a(|2nO8ztrppY=2CyW|D#PU7xny#B_2$ntxOH_o2D&`u zBkV(62Ihf|t)UbY*l9pT;Skgois8s%a<67pa>wf1|Dx-hTeD-)BC^11pMj_!?=%mnq`-u|{2d1zQ zCinK2!QgJ|`jbvKvn{xlZ*a?_hvw*nIaH%yj@uy3+6e!)O;FpSiKE~^hsfuBilvF) z7;N*)m-wA`?SK8^>STQP&NYz-BUQAryW&gb@0FLMtmscVrT?vTUE&s$6$j%NVpn2v z(QcF#4@QPiSZrHbq5;#|z(#|Lsi=*?v@B!hOl3Qb8p}X&3f?c*-S+MrH5OR+K=p4@ z!F(fEe408q0uEP%=9tJu$UR|lZ{HZyxWUc-*v)k1su6yhSA#ghksT)6h=l|@D>6sv z!%R4_g1CgAA#(5avE)p(@_RJl&U6MR6?R)N6R_xEg1yZ&u7LTBy$cfzd0QB|VcXy! ztStA=jiWW*u#0L^g8x^&NxM$gFnB1$GfZuL>{Zqg1 z*6g{)Z?(J7k%2g%8i;c;0p=L`r{Y6-1fE{dn?e_BfcpDkqA%{C>khp zHK7@(sz9Oe+g&+K9^|JvU%I1u=i@K@p_^$9uD9U7d07}B8ZaNI549CaDquevmWyf7M*)UOBt2FMi1N16@ffa@UiU=xH90UijT zpfLG-U6;N5z(((pSaMJ5X7;Iax2}1~91Ep?2Cb?n=55tL8-WyJ%N$K35H+kkf$AJ4 z57dk83;?OA8#`X!<<>MU8s>cJ&r>frIs%Jfpkr`ZP}TNJaI8@<4gwDWDq6-@ubYqQr(0LcPG_lniimP`tv3RRs`Oh}=7JFQOxP%8a;~yMjykM!LKL z0JmKhh>DKH@$8WFQ87uv)MZ21KDZGk_uje}En3}3q{^MSGdOAiD|{@Q-0eNG4>3)7 zcf(_D-5pCtRG?c&Lj@!e#6+U$3Q%Cnqzo%QHn1WfV%YKtkuTbiGG{}nXD;-(HMi&3 zP=RcYwL%7j2*{LX3eGuJ02P135lbp?TvT)+a?gk&*xgn0HghL$3vTaQ&+=^U=Kurk z4hlKwlw*rX*0Au%WE~q*=(ob;i`Vmx+`j0xrOrzqKWzGmk0r{=*3J5sf_@_5Ccmj7kh-=J!ck`I^uB=L61 zRa6a~@h`^SiLEO77^((4qGAx@|F^h&>uVDfDy+Gn6#(mvVEGp4L)x)*f|^R8?IiBx zz8o!0mvM4?Uj|cf18;|%PA~BQ=hU!m6uqYNL_kMU5&KSMpy}xkV z@6FNR@-0q)_DsS&gj!gjBdP^N8kWg?R2Ee->a!5JcjO$LvEuZ(KXxvswYC%(leN`YCVRG-Sb8yBRUaft@ow++WYQY)3Y`^b~=i!Ydjp{|W z?l-RW#=MSM22zWv*$$mh)!3r#s|&bLp!HtyC;tn89D z#S{p2wF3?(IM&cQR81vCFcXwG4G{Uta?comOWyI|PyWoE91nC**d;mkSUKDR#6437 z4Hj(**A(zTvBixk5HiWi^5Bkg_Q=V*Q#JJDq}x80V@u1u(IhgK2UT#gB~%5d*pNNI zZS{4E^a=KOh}`Sh3tK9FyZkF|W;D1QY^f0H4ulBKVK_csq$2`IIv_qn!^jYo54MoQ z$lney+rGb6!SKE6_*oVO7y9bMN2@z>HBDpDj@z$&06Baxwp4Uv0OgRsk@ z?O*wxTQ{6zm&=nd1-K4~!C!l~2-a@UEfC_MkwB(G(znCpUilD!zEwrNjc(1*k|h>M z!WL#gX!kkX4s$e41^Cp4&m)%UT(uY`_YMxh5VyTu2KZz!(4TLJ%agDvuxu$2*;X9T zb8#oUr2zJ!YYHq7Gnp{C_l+R{eYLZnebsH>yJ&#(&^x*U(?(CM`6wcM$KueGqZ)jXJnf~Bh-w+Gb`N*SK3ii3sCdKN54Z^nUGjml8UXf+6g0QkY z=ivL%Gy6c_>d!xS(5>ssu}UQO{8|*d>Y54Ulxa3XB@-yWG2w79AF_2D%fn&vxmuKG zGbcCBzY(JN-oHw?HN80oieQq5xv78!sJ}5RQN%D1^6`2IQ=o1`f(Jx|_3M@8!CmJt zw>8yw-2Fe%vh@TChNaF$mAoele+KIwoAZW+`Vgk70scnW1Or1Mj&~Y_$QKXweo~RV zJ09ySja|Gi8D9-R;@L>mzRE9GyjcFd@;FM1)5-6azMlBoL=8%dZ2YIO{~aqT(otGG z8#xO=qJIgHz#0qod%2aiL6HFaUgc>F$fUHt`J!T%1u%o*)*Q{z$L!7+7Ot0q_xm7;p{?eHr$6fFq*E z2A;ICeBm~WKEI)=_)#~L2`;qY^LaQ6&cJ&+5La+5RGH|C{P$469c=#%3owwC<=*#? z17}&MY~Se4%mgQWfR|%5j#NYa#S^^ef(ZDxjZy(SuPTrz5E4S<3-|Nk{J_goNBIV~ zZhA#8r-0lr^a0Dku_S{K;0gX4S>_@OP#7fimF0n27rk7s>WH~DQ#qEo+{=ME6H6d` zfnq@o16?j!HUIT9XitlIQcr8Qxa|iQt+ZebArPre2?#0?K0yX0C$*8*K`FtnIb~sT zZ{I1@MmzuA$4obKAh@4zs>_QARbXud7&~SY2%M<)xMQHzCO%$)i6};uE6cqfokD4} z>&?Oc}gc=>u-7V zX}4}N$2gbQ5IX7-e1O4@sZ0LU z7_AjDo4+Fn@Xmm%X*x#tA#(4PGwAEm9WTG^w%a)d=hq5(NgHJ_10le~cwU9;LSLAV z5zKK=U;2%In0(I2B@SBM_N-qSUALzG1vg^__w!9}d8J?>-s^Zji=QgNCqn;-juP)l zK|6)uiN(wid2r|gK36`+N=V&*_Az(H3{Lhfad{x|T^6w)6ANszp|1o`j|$Dgpd22j z0aOZ+FPuD2vNKbu!7pFu))`ASIae*@saOTC2uo@z;wA7K4Gc8ALB)0~RArdfqdW_d z2e)<4F4RSl=brqyTXWx{eHKIp!D?dI-vbd1lV*WV3ylI^`Gkp$ivI-&8zNsg)n~`9 zjix&H25RfMDswjlqF!`KjWa*l&ewX>}@4Gdpavn*6x*W%Viv!k=*JZv2abi#jL=KmoP-d#LfZYW}x%H%X4& z=cu2;u3rAOn>ij_4)#>YwE#9FHu_^gOtVlTpy0!38pXNKqy?TCCSSOncX@Bpsiv!6 z{gFE}8yxAET?M*)sL;fyPt)*vB24%t9pgxUixeYXyhjL{4Uv0O$6%-Bop-Kv>yG8v s>GA{(S{tx#FhYP;pj`!#0E0rjs?7klX9GtJlY8Z}$sO^?tMbGD4@NyE*8l(j diff --git a/tools/auditancestry.py b/tools/auditancestry.py index 95e082642..59d636836 100644 --- a/tools/auditancestry.py +++ b/tools/auditancestry.py @@ -1,7 +1,7 @@ import logging import json -from data.database import Image, ImageStorage, Repository, configure +from data.database import Image, ImageStorage, Repository, User, configure from data import model from app import app, storage as store @@ -16,17 +16,18 @@ logging.getLogger('boto').setLevel(logging.CRITICAL) query = (Image - .select(Image, ImageStorage, Repository) - .join(ImageStorage) - .switch(Image) - .join(Repository) - .where(ImageStorage.uploading == False)) + .select(Image, ImageStorage, Repository, User) + .join(ImageStorage) + .switch(Image) + .join(Repository) + .join(User) + .where(ImageStorage.uploading == False)) bad_count = 0 good_count = 0 def resolve_or_create(repo, docker_image_id, new_ancestry): - existing = model.get_repo_image(repo.namespace, repo.name, docker_image_id) + existing = model.get_repo_image(repo.namespace_user.username, repo.name, docker_image_id) if existing: logger.debug('Found existing image: %s, %s', existing.id, docker_image_id) return existing @@ -45,7 +46,7 @@ def resolve_or_create(repo, docker_image_id, new_ancestry): return created except ImageStorage.DoesNotExist: msg = 'No image available anywhere for storage: %s in namespace: %s' - logger.error(msg, docker_image_id, repo.namespace) + logger.error(msg, docker_image_id, repo.namespace_user.username) raise RuntimeError() @@ -62,20 +63,19 @@ def all_ancestors_exist(ancestors): cant_fix = [] for img in query: try: - with_locations = model.get_repo_image(img.repository.namespace, img.repository.name, - img.docker_image_id) + with_locations = model.get_repo_image(img.repository.namespace_user.username, + img.repository.name, img.docker_image_id) ancestry_storage = store.image_ancestry_path(img.storage.uuid) if store.exists(with_locations.storage.locations, ancestry_storage): - full_ancestry = json.loads(store.get_content(with_locations.storage.locations, ancestry_storage))[1:] + full_ancestry = json.loads(store.get_content(with_locations.storage.locations, + ancestry_storage))[1:] full_ancestry.reverse() - ancestor_dbids = [int(anc_id) - for anc_id in img.ancestors.split('/')[1:-1]] + ancestor_dbids = [int(anc_id) for anc_id in img.ancestors.split('/')[1:-1]] if len(full_ancestry) != len(ancestor_dbids) or not all_ancestors_exist(ancestor_dbids): - logger.error('Image has incomplete ancestry: %s, %s, %s, %s' % - (img.id, img.docker_image_id, full_ancestry, - ancestor_dbids)) + logger.error('Image has incomplete ancestry: %s, %s, %s, %s', img.id, img.docker_image_id, + full_ancestry, ancestor_dbids) fixed_ancestry = '/' for ancestor in full_ancestry: @@ -99,5 +99,5 @@ for img in query: len(cant_fix)) for cant in cant_fix: - logger.error('Unable to fix %s in repo %s/%s', cant.id, - cant.repository.namespace, cant.repository.name) \ No newline at end of file + logger.error('Unable to fix %s in repo %s/%s', cant.id, cant.repository.namespace_user.username, + cant.repository.name) diff --git a/tools/relationships.py b/tools/relationships.py index d701a18ce..c2cad5de5 100644 --- a/tools/relationships.py +++ b/tools/relationships.py @@ -9,7 +9,7 @@ with open('outfile.dot', 'w') as outfile: outfile.write('digraph relationships {\n') for repo in Repository.select(): - ns = fix_ident(repo.namespace) + ns = fix_ident(repo.namespace_user.username) outfile.write('%s_%s -> %s\n' % (ns, fix_ident(repo.name), ns)) teams_in_orgs = set() From e9745dbc96a67c54b964b9a7ad55ed2108c489e9 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Fri, 26 Sep 2014 11:17:42 -0400 Subject: [PATCH 3/4] Fix the SQL migration. --- .../versions/3f4fe1194671_backfill_the_namespace_user_fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py b/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py index 7c3b7924b..5156e5b01 100644 --- a/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py +++ b/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py @@ -16,7 +16,7 @@ import sqlalchemy as sa def upgrade(tables): conn = op.get_bind() - conn.execute('update repository set namespace_user_id = (select id from user where user.username == repository.namespace) where namespace_user_id is NULL') + conn.execute('update repository set namespace_user_id = (select id from user where user.username = repository.namespace) where namespace_user_id is NULL') op.alter_column('repository', 'namespace_user_id', nullable=False) op.create_index('repository_namespace_user_id_name', 'repository', ['namespace_user_id', 'name'], unique=True) From 3002371f5aa2c94aa207731088a45ad4a1674130 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 26 Sep 2014 11:20:26 -0400 Subject: [PATCH 4/4] Add an orphans lookup script --- tools/orphans.py | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/tools/orphans.py b/tools/orphans.py index 17babf31a..a8990f79b 100644 --- a/tools/orphans.py +++ b/tools/orphans.py @@ -1,22 +1,15 @@ -from data.database import Image -from app import app, storage as store +from data.database import Image, ImageStorage +from peewee import JOIN_LEFT_OUTER, fn +from app import app -live_image_id_set = set() +orphaned = (ImageStorage + .select() + .where(ImageStorage.uploading == False) + .join(Image, JOIN_LEFT_OUTER) + .group_by(ImageStorage) + .having(fn.Count(Image.id) == 0)) -for image in Image.select(): - live_image_id_set.add(image.docker_image_id) - -storage_image_id_set = set() -for customer in store.list_directory('images/'): - for repo in store.list_directory(customer): - for image in store.list_directory(repo): - storage_image_id_set.add(image.split('/')[-1]) - -orphans = storage_image_id_set.difference(live_image_id_set) -missing_image_data = live_image_id_set.difference(storage_image_id_set) - -for orphan in orphans: - print "Orphan: %s" % orphan - -for missing in missing_image_data: - print "Missing: %s" % missing +counter = 0 +for orphan in orphaned: + counter += 1 + print orphan.uuid