diff --git a/auth/permissions.py b/auth/permissions.py index 59b190b3c..eb9059c22 100644 --- a/auth/permissions.py +++ b/auth/permissions.py @@ -112,7 +112,7 @@ class QuayDeferredPermissionUser(Identity): # Add repository permissions for perm in model.get_all_user_permissions(user_object): - repo_grant = _RepositoryNeed(perm.repository.namespace, perm.repository.name, + repo_grant = _RepositoryNeed(perm.repository.namespace_user.username, perm.repository.name, self._repo_role_for_scopes(perm.role.name)) logger.debug('User added permission: {0}'.format(repo_grant)) self.provides.add(repo_grant) @@ -239,7 +239,7 @@ def on_identity_loaded(sender, identity): logger.debug('Loading permissions for token: %s', identity.id) token_data = model.load_token_data(identity.id) - repo_grant = _RepositoryNeed(token_data.repository.namespace, + repo_grant = _RepositoryNeed(token_data.repository.namespace_user.username, token_data.repository.name, token_data.role.name) logger.debug('Delegate token added permission: {0}'.format(repo_grant)) diff --git a/data/database.py b/data/database.py index f5d780d80..b9716bf0f 100644 --- a/data/database.py +++ b/data/database.py @@ -169,7 +169,7 @@ class Visibility(BaseModel): class Repository(BaseModel): namespace = CharField() - namespace_user = ForeignKeyField(User, null=True) + namespace_user = ForeignKeyField(User) name = CharField() visibility = ForeignKeyField(Visibility) description = TextField(null=True) @@ -181,7 +181,7 @@ class Repository(BaseModel): indexes = ( # create a unique index on namespace and name (('namespace', 'name'), True), - (('namespace_user', 'name'), False), + (('namespace_user', 'name'), True), ) diff --git a/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py b/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py new file mode 100644 index 000000000..5156e5b01 --- /dev/null +++ b/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py @@ -0,0 +1,27 @@ +"""Backfill the namespace_user fields. + +Revision ID: 3f4fe1194671 +Revises: 6f2ecf5afcf +Create Date: 2014-09-24 14:29:45.192179 + +""" + +# revision identifiers, used by Alembic. +revision = '3f4fe1194671' +down_revision = '6f2ecf5afcf' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + conn = op.get_bind() + conn.execute('update repository set namespace_user_id = (select id from user where user.username = repository.namespace) where namespace_user_id is NULL') + + op.alter_column('repository', 'namespace_user_id', nullable=False) + op.create_index('repository_namespace_user_id_name', 'repository', ['namespace_user_id', 'name'], unique=True) + + +def downgrade(tables): + op.drop_index('repository_namespace_user_id_name', table_name='repository') + op.alter_column('repository', 'namespace_user_id', nullable=True) diff --git a/data/model/legacy.py b/data/model/legacy.py index 2cd8be94b..07c8b9e2a 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -25,6 +25,9 @@ EXPONENTIAL_BACKOFF_SCALE = timedelta(seconds=1) PRESUMED_DEAD_BUILD_AGE = timedelta(days=15) +Namespace = User.alias() + + logger = logging.getLogger(__name__) @@ -100,13 +103,24 @@ class TooManyLoginAttemptsException(Exception): super(TooManyLoginAttemptsException, self).__init__(message) self.retry_after = retry_after + +def _get_repository(namespace_name, repository_name): + return (Repository + .select(Repository, Namespace) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name) + .get()) + + def hash_password(password, salt=None): salt = salt or bcrypt.gensalt() return bcrypt.hashpw(password.encode('utf-8'), salt) + def is_create_user_allowed(): return True + def create_user(username, password, email, auto_verify=False): """ Creates a regular user, if allowed. """ if not validate_password(password): @@ -122,6 +136,7 @@ def create_user(username, password, email, auto_verify=False): return created + def _create_user(username, email): if not validate_email(email): raise InvalidEmailAddressException('Invalid email address: %s' % email) @@ -733,7 +748,7 @@ def get_visible_repositories(username=None, include_public=True, page=None, limit=None, sort=False, namespace=None): query = _visible_repository_query(username=username, include_public=include_public, page=page, limit=limit, namespace=namespace, - select_models=[Repository, Visibility]) + select_models=[Repository, Namespace, Visibility]) if sort: query = query.order_by(Repository.description.desc()) @@ -747,11 +762,13 @@ def get_visible_repositories(username=None, include_public=True, page=None, def _visible_repository_query(username=None, include_public=True, limit=None, page=None, namespace=None, select_models=[]): query = (Repository - .select(*select_models) # Note: We need to leave this blank for the get_count case. Otherwise, MySQL/RDS complains. - .distinct() - .join(Visibility) - .switch(Repository) - .join(RepositoryPermission, JOIN_LEFT_OUTER)) + .select(*select_models) # MySQL/RDS complains is there are selected models for counts. + .distinct() + .join(Visibility) + .switch(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(Repository) + .join(RepositoryPermission, JOIN_LEFT_OUTER)) query = _filter_to_repos_for_user(query, username, namespace, include_public) @@ -782,26 +799,20 @@ def _filter_to_repos_for_user(query, username=None, namespace=None, .switch(RepositoryPermission) .join(Team, JOIN_LEFT_OUTER) .join(TeamMember, JOIN_LEFT_OUTER) - .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == - TeamMember.user)) + .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == TeamMember.user)) .switch(Repository) - .join(Org, JOIN_LEFT_OUTER, on=(Org.username == Repository.namespace)) - .join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id == - AdminTeam.organization)) + .join(Org, JOIN_LEFT_OUTER, on=(Repository.namespace_user == Org.id)) + .join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id == AdminTeam.organization)) .join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id)) .switch(AdminTeam) - .join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id == - AdminTeamMember.team)) - .join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user == - AdminUser.id))) + .join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id == AdminTeamMember.team)) + .join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user == AdminUser.id))) - where_clause = ((User.username == username) | - (UserThroughTeam.username == username) | - ((AdminUser.username == username) & - (TeamRole.name == 'admin'))) + where_clause = ((User.username == username) | (UserThroughTeam.username == username) | + ((AdminUser.username == username) & (TeamRole.name == 'admin'))) if namespace: - where_clause = where_clause & (Repository.namespace == namespace) + where_clause = where_clause & (Namespace.username == namespace) if include_public: new_clause = (Visibility.name == 'public') @@ -820,7 +831,7 @@ def get_matching_repositories(repo_term, username=None): visible = get_visible_repositories(username) search_clauses = (Repository.name ** ('%' + name_term + '%') | - Repository.namespace ** ('%' + namespace_term + '%')) + Namespace.username ** ('%' + namespace_term + '%')) # Handle the case where the user has already entered a namespace path. if repo_term.find('/') > 0: @@ -829,7 +840,7 @@ def get_matching_repositories(repo_term, username=None): name_term = parts[-1] search_clauses = (Repository.name ** ('%' + name_term + '%') & - Repository.namespace ** ('%' + namespace_term + '%')) + Namespace.username ** ('%' + namespace_term + '%')) final = visible.where(search_clauses).limit(10) return list(final) @@ -859,22 +870,20 @@ def update_email(user, new_email, auto_verify=False): def get_all_user_permissions(user): - select = RepositoryPermission.select(RepositoryPermission, Role, Repository) - with_role = select.join(Role) - with_repo = with_role.switch(RepositoryPermission).join(Repository) - through_user = with_repo.switch(RepositoryPermission).join(User, - JOIN_LEFT_OUTER) - as_perm = through_user.switch(RepositoryPermission) - through_team = as_perm.join(Team, JOIN_LEFT_OUTER).join(TeamMember, - JOIN_LEFT_OUTER) - UserThroughTeam = User.alias() - with_team_member = through_team.join(UserThroughTeam, JOIN_LEFT_OUTER, - on=(UserThroughTeam.id == - TeamMember.user)) - return with_team_member.where((User.id == user) | - (UserThroughTeam.id == user)) + return (RepositoryPermission + .select(RepositoryPermission, Role, Repository, Namespace) + .join(Role) + .switch(RepositoryPermission) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(RepositoryPermission) + .join(User, JOIN_LEFT_OUTER) + .switch(RepositoryPermission) + .join(Team, JOIN_LEFT_OUTER).join(TeamMember, JOIN_LEFT_OUTER) + .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == TeamMember.user)) + .where((User.id == user) | (UserThroughTeam.id == user))) def delete_prototype_permission(org, uid): @@ -939,33 +948,37 @@ def get_org_wide_permissions(user): def get_all_repo_teams(namespace_name, repository_name): - select = RepositoryPermission.select(Team.name.alias('team_name'), - Role.name, RepositoryPermission) - with_team = select.join(Team) - with_role = with_team.switch(RepositoryPermission).join(Role) - with_repo = with_role.switch(RepositoryPermission).join(Repository) - return with_repo.where(Repository.namespace == namespace_name, - Repository.name == repository_name) + return (RepositoryPermission.select(Team.name.alias('team_name'), Role.name, RepositoryPermission) + .join(Team) + .switch(RepositoryPermission) + .join(Role) + .switch(RepositoryPermission) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name)) def get_all_repo_users(namespace_name, repository_name): - select = RepositoryPermission.select(User.username, User.robot, Role.name, - RepositoryPermission) - with_user = select.join(User) - with_role = with_user.switch(RepositoryPermission).join(Role) - with_repo = with_role.switch(RepositoryPermission).join(Repository) - return with_repo.where(Repository.namespace == namespace_name, - Repository.name == repository_name) + return (RepositoryPermission.select(User.username, User.robot, Role.name, RepositoryPermission) + .join(User) + .switch(RepositoryPermission) + .join(Role) + .switch(RepositoryPermission) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name)) def get_all_repo_users_transitive_via_teams(namespace_name, repository_name): - select = User.select().distinct() - with_team_member = select.join(TeamMember) - with_team = with_team_member.join(Team) - with_perm = with_team.join(RepositoryPermission) - with_repo = with_perm.join(Repository) - return with_repo.where(Repository.namespace == namespace_name, - Repository.name == repository_name) + return (User + .select() + .distinct() + .join(TeamMember) + .join(Team) + .join(RepositoryPermission) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name)) def get_all_repo_users_transitive(namespace_name, repository_name): @@ -989,10 +1002,12 @@ def get_all_repo_users_transitive(namespace_name, repository_name): def get_repository_for_resource(resource_key): try: return (Repository - .select() - .join(RepositoryBuild) - .where(RepositoryBuild.resource_key == resource_key) - .get()) + .select(Repository, Namespace) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(Repository) + .join(RepositoryBuild) + .where(RepositoryBuild.resource_key == resource_key) + .get()) except Repository.DoesNotExist: return None @@ -1006,8 +1021,7 @@ def lookup_repository(repo_id): def get_repository(namespace_name, repository_name): try: - return Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + return _get_repository(namespace_name, repository_name) except Repository.DoesNotExist: return None @@ -1024,11 +1038,18 @@ def get_repo_image(namespace_name, repository_name, image_id): def repository_is_public(namespace_name, repository_name): - joined = Repository.select().join(Visibility) - query = joined.where(Repository.namespace == namespace_name, - Repository.name == repository_name, - Visibility.name == 'public') - return len(list(query)) > 0 + try: + (Repository + .select() + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(Repository) + .join(Visibility) + .where(Namespace.username == namespace_name, Repository.name == repository_name, + Visibility.name == 'public') + .get()) + return True + except Repository.DoesNotExist: + return False def set_repository_visibility(repo, visibility): @@ -1128,7 +1149,7 @@ def __translate_ancestry(old_ancestry, translations, repository, username, prefe def find_create_or_link_image(docker_image_id, repository, username, translations, preferred_location): with config.app_config['DB_TRANSACTION_FACTORY'](db): - repo_image = get_repo_image(repository.namespace, repository.name, + repo_image = get_repo_image(repository.namespace_user.username, repository.name, docker_image_id) if repo_image: return repo_image @@ -1142,6 +1163,8 @@ def find_create_or_link_image(docker_image_id, repository, username, translation .join(Visibility) .switch(Repository) .join(RepositoryPermission, JOIN_LEFT_OUTER) + .switch(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) .where(ImageStorage.uploading == False)) query = (_filter_to_repos_for_user(query, username) @@ -1186,11 +1209,11 @@ def find_create_or_link_image(docker_image_id, repository, username, translation def get_storage_by_uuid(storage_uuid): placements = list(ImageStoragePlacement - .select(ImageStoragePlacement, ImageStorage, ImageStorageLocation) - .join(ImageStorageLocation) - .switch(ImageStoragePlacement) - .join(ImageStorage) - .where(ImageStorage.uuid == storage_uuid)) + .select(ImageStoragePlacement, ImageStorage, ImageStorageLocation) + .join(ImageStorageLocation) + .switch(ImageStoragePlacement) + .join(ImageStorage) + .where(ImageStorage.uuid == storage_uuid)) if not placements: raise InvalidImageException('No storage found with uuid: %s', storage_uuid) @@ -1205,14 +1228,14 @@ def set_image_size(docker_image_id, namespace_name, repository_name, image_size): try: image = (Image - .select(Image, ImageStorage) - .join(Repository) - .switch(Image) - .join(ImageStorage, JOIN_LEFT_OUTER) - .where(Repository.name == repository_name, - Repository.namespace == namespace_name, - Image.docker_image_id == docker_image_id) - .get()) + .select(Image, ImageStorage) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(Image) + .join(ImageStorage, JOIN_LEFT_OUTER) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + Image.docker_image_id == docker_image_id) + .get()) except Image.DoesNotExist: raise DataModelException('No image with specified id and repository') @@ -1231,13 +1254,13 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created command, uncompressed_size, parent=None): with config.app_config['DB_TRANSACTION_FACTORY'](db): query = (Image - .select(Image, ImageStorage) - .join(Repository) - .switch(Image) - .join(ImageStorage) - .where(Repository.name == repository_name, - Repository.namespace == namespace_name, - Image.docker_image_id == docker_image_id)) + .select(Image, ImageStorage) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(Image) + .join(ImageStorage) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + Image.docker_image_id == docker_image_id)) try: fetched = query.get() @@ -1248,7 +1271,7 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created fetched.storage.checksum = None fetched.storage.created = dateutil.parser.parse(created_date_str).replace(tzinfo=None) fetched.storage.comment = comment - fetched.storage.command = command + fetched.storage.command = command fetched.storage.uncompressed_size = uncompressed_size if parent: @@ -1261,14 +1284,14 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created def _get_repository_images_base(namespace_name, repository_name, query_modifier): query = (ImageStoragePlacement - .select(ImageStoragePlacement, Image, ImageStorage, ImageStorageLocation) - .join(ImageStorageLocation) - .switch(ImageStoragePlacement) - .join(ImageStorage, JOIN_LEFT_OUTER) - .join(Image) - .join(Repository) - .where(Repository.name == repository_name, - Repository.namespace == namespace_name)) + .select(ImageStoragePlacement, Image, ImageStorage, ImageStorageLocation) + .join(ImageStorageLocation) + .switch(ImageStoragePlacement) + .join(ImageStorage, JOIN_LEFT_OUTER) + .join(Image) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.name == repository_name, Namespace.username == namespace_name)) query = query_modifier(query) @@ -1299,24 +1322,26 @@ def get_repository_images(namespace_name, repository_name): def list_repository_tags(namespace_name, repository_name): - select = RepositoryTag.select(RepositoryTag, Image) - with_repo = select.join(Repository) - with_image = with_repo.switch(RepositoryTag).join(Image) - return with_image.where(Repository.name == repository_name, - Repository.namespace == namespace_name) + return (RepositoryTag + .select(RepositoryTag, Image) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(RepositoryTag) + .join(Image) + .where(Repository.name == repository_name, Namespace.username == namespace_name)) def garbage_collect_repository(namespace_name, repository_name): with config.app_config['DB_TRANSACTION_FACTORY'](db): # Get a list of all images used by tags in the repository tag_query = (RepositoryTag - .select(RepositoryTag, Image, ImageStorage) - .join(Image) - .join(ImageStorage, JOIN_LEFT_OUTER) - .switch(RepositoryTag) - .join(Repository) - .where(Repository.name == repository_name, - Repository.namespace == namespace_name)) + .select(RepositoryTag, Image, ImageStorage) + .join(Image) + .join(ImageStorage, JOIN_LEFT_OUTER) + .switch(RepositoryTag) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.name == repository_name, Namespace.username == namespace_name)) referenced_anscestors = set() for tag in tag_query: @@ -1344,11 +1369,11 @@ def garbage_collect_repository(namespace_name, repository_name): if uuids_to_check_for_gc: storage_to_remove = (ImageStorage - .select() - .join(Image, JOIN_LEFT_OUTER) - .group_by(ImageStorage) - .where(ImageStorage.uuid << list(uuids_to_check_for_gc)) - .having(fn.Count(Image.id) == 0)) + .select() + .join(Image, JOIN_LEFT_OUTER) + .group_by(ImageStorage) + .where(ImageStorage.uuid << list(uuids_to_check_for_gc)) + .having(fn.Count(Image.id) == 0)) for storage in storage_to_remove: logger.debug('Garbage collecting image storage: %s', storage.uuid) @@ -1367,9 +1392,9 @@ def garbage_collect_repository(namespace_name, repository_name): def get_tag_image(namespace_name, repository_name, tag_name): def limit_to_tag(query): return (query - .switch(Image) - .join(RepositoryTag) - .where(RepositoryTag.name == tag_name)) + .switch(Image) + .join(RepositoryTag) + .where(RepositoryTag.name == tag_name)) images = _get_repository_images_base(namespace_name, repository_name, limit_to_tag) if not images: @@ -1407,22 +1432,17 @@ def get_parent_images(namespace_name, repository_name, image_obj): def create_or_update_tag(namespace_name, repository_name, tag_name, tag_docker_image_id): try: - repo = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + repo = _get_repository(namespace_name, repository_name) except Repository.DoesNotExist: - raise DataModelException('Invalid repository %s/%s' % - (namespace_name, repository_name)) + raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name)) try: - image = Image.get(Image.docker_image_id == tag_docker_image_id, - Image.repository == repo) + image = Image.get(Image.docker_image_id == tag_docker_image_id, Image.repository == repo) except Image.DoesNotExist: - raise DataModelException('Invalid image with id: %s' % - tag_docker_image_id) + raise DataModelException('Invalid image with id: %s' % tag_docker_image_id) try: - tag = RepositoryTag.get(RepositoryTag.repository == repo, - RepositoryTag.name == tag_name) + tag = RepositoryTag.get(RepositoryTag.repository == repo, RepositoryTag.name == tag_name) tag.image = image tag.save() except RepositoryTag.DoesNotExist: @@ -1432,41 +1452,46 @@ def create_or_update_tag(namespace_name, repository_name, tag_name, def delete_tag(namespace_name, repository_name, tag_name): - joined = RepositoryTag.select().join(Repository) - found = list(joined.where(Repository.name == repository_name, - Repository.namespace == namespace_name, - RepositoryTag.name == tag_name)) + try: + found = (RepositoryTag + .select() + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + RepositoryTag.name == tag_name) + .get()) - if not found: + except RepositoryTag.DoesNotExist: msg = ('Invalid repository tag \'%s\' on repository \'%s/%s\'' % (tag_name, namespace_name, repository_name)) raise DataModelException(msg) - found[0].delete_instance() + found.delete_instance() def delete_all_repository_tags(namespace_name, repository_name): try: - repo = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + repo = _get_repository(namespace_name, repository_name) except Repository.DoesNotExist: raise DataModelException('Invalid repository \'%s/%s\'' % (namespace_name, repository_name)) RepositoryTag.delete().where(RepositoryTag.repository == repo.id).execute() -def __entity_permission_repo_query(entity_id, entity_table, - entity_id_property, namespace_name, +def __entity_permission_repo_query(entity_id, entity_table, entity_id_property, namespace_name, repository_name): """ This method works for both users and teams. """ - selected = RepositoryPermission.select(entity_table, Repository, Role, - RepositoryPermission) - with_user = selected.join(entity_table) - with_role = with_user.switch(RepositoryPermission).join(Role) - with_repo = with_role.switch(RepositoryPermission).join(Repository) - return with_repo.where(Repository.name == repository_name, - Repository.namespace == namespace_name, - entity_id_property == entity_id) + + return (RepositoryPermission + .select(entity_table, Repository, Namespace, Role, RepositoryPermission) + .join(entity_table) + .switch(RepositoryPermission) + .join(Role) + .switch(RepositoryPermission) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + entity_id_property == entity_id)) def get_user_reponame_permission(username, namespace_name, repository_name): @@ -1514,8 +1539,7 @@ def delete_team_permission(team_name, namespace_name, repository_name): def __set_entity_repo_permission(entity, permission_entity_property, namespace_name, repository_name, role_name): - repo = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + repo = _get_repository(namespace_name, repository_name) new_role = Role.get(Role.name == role_name) # Fetch any existing permission for this entity on the repo @@ -1566,15 +1590,18 @@ def purge_repository(namespace_name, repository_name): garbage_collect_repository(namespace_name, repository_name) # Delete the rest of the repository metadata - fetched = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + fetched = _get_repository(namespace_name, repository_name) fetched.delete_instance(recursive=True) def get_private_repo_count(username): - joined = Repository.select().join(Visibility) - return joined.where(Repository.namespace == username, - Visibility.name == 'private').count() + return (Repository + .select() + .join(Visibility) + .switch(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == username, Visibility.name == 'private') + .count()) def create_access_token(repository, role): @@ -1587,22 +1614,23 @@ def create_access_token(repository, role): def create_delegate_token(namespace_name, repository_name, friendly_name, role='read'): read_only = Role.get(name=role) - repo = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + repo = _get_repository(namespace_name, repository_name) new_token = AccessToken.create(repository=repo, role=read_only, friendly_name=friendly_name, temporary=False) return new_token def get_repository_delegate_tokens(namespace_name, repository_name): - return (AccessToken.select(AccessToken, Role) - .join(Repository) - .switch(AccessToken) - .join(Role) - .switch(AccessToken) - .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) - .where(Repository.name == repository_name, Repository.namespace == namespace_name, - AccessToken.temporary == False, RepositoryBuildTrigger.uuid >> None)) + return (AccessToken + .select(AccessToken, Role) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(AccessToken) + .join(Role) + .switch(AccessToken) + .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + AccessToken.temporary == False, RepositoryBuildTrigger.uuid >> None)) def get_repo_delegate_token(namespace_name, repository_name, code): @@ -1636,14 +1664,17 @@ def delete_delegate_token(namespace_name, repository_name, code): def load_token_data(code): """ Load the permissions for any token by code. """ - selected = AccessToken.select(AccessToken, Repository, Role) - with_role = selected.join(Role) - with_repo = with_role.switch(AccessToken).join(Repository) - fetched = list(with_repo.where(AccessToken.code == code)) + try: + return (AccessToken + .select(AccessToken, Repository, Namespace, Role) + .join(Role) + .switch(AccessToken) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(AccessToken.code == code) + .get()) - if fetched: - return fetched[0] - else: + except AccessToken.DoesNotExist: raise InvalidTokenException('Invalid delegate token code: %s' % code) @@ -1660,15 +1691,15 @@ def get_repository_build(namespace_name, repository_name, build_uuid): def list_repository_builds(namespace_name, repository_name, limit, include_inactive=True): query = (RepositoryBuild - .select(RepositoryBuild, RepositoryBuildTrigger, BuildTriggerService) - .join(Repository) - .switch(RepositoryBuild) - .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) - .join(BuildTriggerService, JOIN_LEFT_OUTER) - .where(Repository.name == repository_name, - Repository.namespace == namespace_name) - .order_by(RepositoryBuild.started.desc()) - .limit(limit)) + .select(RepositoryBuild, RepositoryBuildTrigger, BuildTriggerService) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(RepositoryBuild) + .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) + .join(BuildTriggerService, JOIN_LEFT_OUTER) + .where(Repository.name == repository_name, Namespace.username == namespace_name) + .order_by(RepositoryBuild.started.desc()) + .limit(limit)) if not include_inactive: query = query.where(RepositoryBuild.phase != 'error', @@ -1732,16 +1763,17 @@ def create_repo_notification(repo, event_name, method_name, config): def get_repo_notification(namespace_name, repository_name, uuid): - joined = RepositoryNotification.select().join(Repository) - found = list(joined.where(Repository.namespace == namespace_name, - Repository.name == repository_name, - RepositoryNotification.uuid == uuid)) - - if not found: + try: + return (RepositoryNotification + .select(RepositoryNotification, Repository, Namespace) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name, + RepositoryNotification.uuid == uuid) + .get()) + except RepositoryNotification.DoesNotExist: raise InvalidNotificationException('No repository notification found with id: %s' % uuid) - return found[0] - def delete_repo_notification(namespace_name, repository_name, uuid): found = get_repo_notification(namespace_name, repository_name, uuid) @@ -1750,15 +1782,19 @@ def delete_repo_notification(namespace_name, repository_name, uuid): def list_repo_notifications(namespace_name, repository_name, event_name=None): - joined = RepositoryNotification.select().join(Repository) - where = joined.where(Repository.namespace == namespace_name, - Repository.name == repository_name) + query = (RepositoryNotification + .select(RepositoryNotification, Repository, Namespace) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name)) if event_name: - event = ExternalNotificationEvent.get(ExternalNotificationEvent.name == event_name) - where = where.where(RepositoryNotification.event == event) + query = (query + .switch(RepositoryNotification) + .join(ExternalNotificationEvent) + .where(ExternalNotificationEvent.name == event_name)) - return where + return query def list_logs(start_time, end_time, performer=None, repository=None, namespace=None): @@ -1802,16 +1838,17 @@ def create_build_trigger(repo, service_name, auth_token, user, pull_robot=None): def get_build_trigger(namespace_name, repository_name, trigger_uuid): try: return (RepositoryBuildTrigger - .select(RepositoryBuildTrigger, BuildTriggerService, Repository) - .join(BuildTriggerService) - .switch(RepositoryBuildTrigger) - .join(Repository) - .switch(RepositoryBuildTrigger) - .join(User) - .where(RepositoryBuildTrigger.uuid == trigger_uuid, - Repository.namespace == namespace_name, - Repository.name == repository_name) - .get()) + .select(RepositoryBuildTrigger, BuildTriggerService, Repository, Namespace) + .join(BuildTriggerService) + .switch(RepositoryBuildTrigger) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(RepositoryBuildTrigger) + .join(User) + .where(RepositoryBuildTrigger.uuid == trigger_uuid, + Namespace.username == namespace_name, + Repository.name == repository_name) + .get()) except RepositoryBuildTrigger.DoesNotExist: msg = 'No build trigger with uuid: %s' % trigger_uuid raise InvalidBuildTriggerException(msg) @@ -1819,12 +1856,12 @@ def get_build_trigger(namespace_name, repository_name, trigger_uuid): def list_build_triggers(namespace_name, repository_name): return (RepositoryBuildTrigger - .select(RepositoryBuildTrigger, BuildTriggerService, Repository) - .join(BuildTriggerService) - .switch(RepositoryBuildTrigger) - .join(Repository) - .where(Repository.namespace == namespace_name, - Repository.name == repository_name)) + .select(RepositoryBuildTrigger, BuildTriggerService, Repository) + .join(BuildTriggerService) + .switch(RepositoryBuildTrigger) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name)) def list_trigger_builds(namespace_name, repository_name, trigger_uuid, @@ -1913,6 +1950,7 @@ def delete_notifications_by_kind(target, kind_name): Notification.delete().where(Notification.target == target, Notification.kind == kind_ref).execute() + def delete_matching_notifications(target, kind_name, **kwargs): kind_ref = NotificationKind.get(name=kind_name) @@ -1943,6 +1981,7 @@ def delete_matching_notifications(target, kind_name, **kwargs): def get_active_users(): return User.select().where(User.organization == False, User.robot == False) + def get_active_user_count(): return get_active_users().count() @@ -1956,11 +1995,13 @@ def detach_external_login(user, service_name): FederatedLogin.delete().where(FederatedLogin.user == user, FederatedLogin.service == service).execute() + def delete_user(user): user.delete_instance(recursive=True, delete_nullable=True) # TODO: also delete any repository data associated + def check_health(): # We will connect to the db, check that it contains some log entry kinds try: @@ -1969,24 +2010,23 @@ def check_health(): except: return False -def get_email_authorized_for_repo(namespace, repository, email): - found = list(RepositoryAuthorizedEmail.select() - .join(Repository) - .where(Repository.namespace == namespace, - Repository.name == repository, - RepositoryAuthorizedEmail.email == email) - .switch(RepositoryAuthorizedEmail) - .limit(1)) - if not found or len(found) < 1: - return None - return found[0] +def get_email_authorized_for_repo(namespace, repository, email): + try: + return (RepositoryAuthorizedEmail + .select(RepositoryAuthorizedEmail, Repository, Namespace) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace, Repository.name == repository, + RepositoryAuthorizedEmail.email == email) + .get()) + except RepositoryAuthorizedEmail.DoesNotExist: + return None def create_email_authorization_for_repo(namespace_name, repository_name, email): try: - repo = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + repo = _get_repository(namespace_name, repository_name) except Repository.DoesNotExist: raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name)) @@ -1996,7 +2036,11 @@ def create_email_authorization_for_repo(namespace_name, repository_name, email): def confirm_email_authorization_for_repo(code): try: - found = RepositoryAuthorizedEmail.get(RepositoryAuthorizedEmail.code == code) + found = (RepositoryAuthorizedEmail + .select(RepositoryAuthorizedEmail, Repository, Namespace) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .get(RepositoryAuthorizedEmail.code == code)) except RepositoryAuthorizedEmail.DoesNotExist: raise DataModelException('Invalid confirmation code.') diff --git a/endpoints/api/build.py b/endpoints/api/build.py index d792234dd..adf6f43ec 100644 --- a/endpoints/api/build.py +++ b/endpoints/api/build.py @@ -169,7 +169,7 @@ class RepositoryBuildList(RepositoryParamResource): # was used. associated_repository = model.get_repository_for_resource(dockerfile_id) if associated_repository: - if not ModifyRepositoryPermission(associated_repository.namespace, + if not ModifyRepositoryPermission(associated_repository.namespace_user.username, associated_repository.name): raise Unauthorized() diff --git a/endpoints/api/repoemail.py b/endpoints/api/repoemail.py index 4dbc845a7..db4c9b571 100644 --- a/endpoints/api/repoemail.py +++ b/endpoints/api/repoemail.py @@ -20,7 +20,7 @@ def record_view(record): return { 'email': record.email, 'repository': record.repository.name, - 'namespace': record.repository.namespace, + 'namespace': record.repository.namespace_user.username, 'confirmed': record.confirmed } diff --git a/endpoints/api/repository.py b/endpoints/api/repository.py index 4e85a35b5..be8c9e8f9 100644 --- a/endpoints/api/repository.py +++ b/endpoints/api/repository.py @@ -80,8 +80,7 @@ class RepositoryList(ApiResource): visibility = req['visibility'] - repo = model.create_repository(namespace_name, repository_name, owner, - visibility) + repo = model.create_repository(namespace_name, repository_name, owner, visibility) repo.description = req['description'] repo.save() @@ -110,7 +109,7 @@ class RepositoryList(ApiResource): """Fetch the list of repositories under a variety of situations.""" def repo_view(repo_obj): return { - 'namespace': repo_obj.namespace, + 'namespace': repo_obj.namespace_user.username, 'name': repo_obj.name, 'description': repo_obj.description, 'is_public': repo_obj.visibility.name == 'public', @@ -134,7 +133,8 @@ class RepositoryList(ApiResource): response['repositories'] = [repo_view(repo) for repo in repo_query if (repo.visibility.name == 'public' or - ReadRepositoryPermission(repo.namespace, repo.name).can())] + ReadRepositoryPermission(repo.namespace_user.username, + repo.name).can())] return response diff --git a/endpoints/api/search.py b/endpoints/api/search.py index 7cb1a1fda..1cce618d9 100644 --- a/endpoints/api/search.py +++ b/endpoints/api/search.py @@ -111,7 +111,7 @@ class FindRepositories(ApiResource): def repo_view(repo): return { - 'namespace': repo.namespace, + 'namespace': repo.namespace_user.username, 'name': repo.name, 'description': repo.description } @@ -125,5 +125,5 @@ class FindRepositories(ApiResource): return { 'repositories': [repo_view(repo) for repo in matching if (repo.visibility.name == 'public' or - ReadRepositoryPermission(repo.namespace, repo.name).can())] + ReadRepositoryPermission(repo.namespace_user.username, repo.name).can())] } diff --git a/endpoints/api/trigger.py b/endpoints/api/trigger.py index 4ec20bfdc..ccccf8010 100644 --- a/endpoints/api/trigger.py +++ b/endpoints/api/trigger.py @@ -205,7 +205,7 @@ class BuildTriggerActivate(RepositoryParamResource): 'write') try: - repository_path = '%s/%s' % (trigger.repository.namespace, + repository_path = '%s/%s' % (trigger.repository.namespace_user.username, trigger.repository.name) path = url_for('webhooks.build_trigger_webhook', repository=repository_path, trigger_uuid=trigger.uuid) diff --git a/endpoints/common.py b/endpoints/common.py index 1c6439371..37ae80ee8 100644 --- a/endpoints/common.py +++ b/endpoints/common.py @@ -205,7 +205,7 @@ def check_repository_usage(user_or_org, plan_found): def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, trigger=None, pull_robot_name=None): host = urlparse.urlparse(request.url).netloc - repo_path = '%s/%s/%s' % (host, repository.namespace, repository.name) + repo_path = '%s/%s/%s' % (host, repository.namespace_user.username, repository.name) token = model.create_access_token(repository, 'write') logger.debug('Creating build %s with repo %s tags %s and dockerfile_id %s', @@ -221,9 +221,9 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, dockerfile_id, build_name, trigger, pull_robot_name=pull_robot_name) - dockerfile_build_queue.put([repository.namespace, repository.name], json.dumps({ + dockerfile_build_queue.put([repository.namespace_user.username, repository.name], json.dumps({ 'build_uuid': build_request.uuid, - 'namespace': repository.namespace, + 'namespace': repository.namespace_user.username, 'repository': repository.name, 'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None }), retries_remaining=1) @@ -231,7 +231,7 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, # Add the build to the repo's log. metadata = { 'repo': repository.name, - 'namespace': repository.namespace, + 'namespace': repository.namespace_user.username, 'fileid': dockerfile_id, 'manual': manual, } @@ -241,9 +241,8 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, metadata['config'] = json.loads(trigger.config) metadata['service'] = trigger.service.name - model.log_action('build_dockerfile', repository.namespace, - ip=request.remote_addr, metadata=metadata, - repository=repository) + model.log_action('build_dockerfile', repository.namespace_user.username, ip=request.remote_addr, + metadata=metadata, repository=repository) # Add notifications for the build queue. profile.debug('Adding notifications for repository') diff --git a/endpoints/index.py b/endpoints/index.py index 5c8d7058a..46c5b9771 100644 --- a/endpoints/index.py +++ b/endpoints/index.py @@ -420,7 +420,7 @@ def put_repository_auth(namespace, repository): def get_search(): def result_view(repo): return { - "name": repo.namespace + '/' + repo.name, + "name": repo.namespace_user.username + '/' + repo.name, "description": repo.description } @@ -438,7 +438,7 @@ def get_search(): results = [result_view(repo) for repo in matching if (repo.visibility.name == 'public' or - ReadRepositoryPermission(repo.namespace, repo.name).can())] + ReadRepositoryPermission(repo.namespace_user.username, repo.name).can())] data = { "query": query, diff --git a/endpoints/notificationevent.py b/endpoints/notificationevent.py index f3f4d6a77..4a195fbd7 100644 --- a/endpoints/notificationevent.py +++ b/endpoints/notificationevent.py @@ -1,8 +1,4 @@ import logging -import io -import os.path -import tarfile -import base64 from notificationhelper import build_event_data diff --git a/endpoints/notificationhelper.py b/endpoints/notificationhelper.py index 773779fb7..6f80f83d0 100644 --- a/endpoints/notificationhelper.py +++ b/endpoints/notificationhelper.py @@ -4,7 +4,7 @@ from data import model import json def build_event_data(repo, extra_data={}, subpage=None): - repo_string = '%s/%s' % (repo.namespace, repo.name) + repo_string = '%s/%s' % (repo.namespace_user.username, repo.name) homepage = '%s://%s/repository/%s' % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'], repo_string) @@ -17,7 +17,7 @@ def build_event_data(repo, extra_data={}, subpage=None): event_data = { 'repository': repo_string, - 'namespace': repo.namespace, + 'namespace': repo.namespace_user.username, 'name': repo.name, 'docker_url': '%s/%s' % (app.config['SERVER_HOSTNAME'], repo_string), 'homepage': homepage, @@ -30,7 +30,7 @@ def build_event_data(repo, extra_data={}, subpage=None): def build_notification_data(notification, event_data): return { 'notification_uuid': notification.uuid, - 'repository_namespace': notification.repository.namespace, + 'repository_namespace': notification.repository.namespace_user.username, 'repository_name': notification.repository.name, 'event_data': event_data } @@ -39,8 +39,9 @@ def build_notification_data(notification, event_data): def spawn_notification(repo, event_name, extra_data={}, subpage=None, pathargs=[]): event_data = build_event_data(repo, extra_data=extra_data, subpage=subpage) - notifications = model.list_repo_notifications(repo.namespace, repo.name, event_name=event_name) + notifications = model.list_repo_notifications(repo.namespace_user.username, repo.name, + event_name=event_name) for notification in notifications: notification_data = build_notification_data(notification, event_data) - path = [repo.namespace, repo.name, event_name] + pathargs + path = [repo.namespace_user.username, repo.name, event_name] + pathargs notification_queue.put(path, json.dumps(notification_data)) diff --git a/endpoints/notificationmethod.py b/endpoints/notificationmethod.py index e6c259cb1..589ebd06d 100644 --- a/endpoints/notificationmethod.py +++ b/endpoints/notificationmethod.py @@ -88,7 +88,7 @@ class QuayNotificationMethod(NotificationMethod): return (True, 'Unknown organization %s' % target_info['name'], None) # Only repositories under the organization can cause notifications to that org. - if target_info['name'] != repository.namespace: + if target_info['name'] != repository.namespace_user.username: return (False, 'Organization name must match repository namespace') return (True, None, [target]) @@ -96,7 +96,7 @@ class QuayNotificationMethod(NotificationMethod): # Lookup the team. team = None try: - team = model.get_organization_team(repository.namespace, target_info['name']) + team = model.get_organization_team(repository.namespace_user.username, target_info['name']) except model.InvalidTeamException: # Probably deleted. return (True, 'Unknown team %s' % target_info['name'], None) @@ -133,7 +133,8 @@ class EmailMethod(NotificationMethod): if not email: raise CannotValidateNotificationMethodException('Missing e-mail address') - record = model.get_email_authorized_for_repo(repository.namespace, repository.name, email) + record = model.get_email_authorized_for_repo(repository.namespace_user.username, + repository.name, email) if not record or not record.confirmed: raise CannotValidateNotificationMethodException('The specified e-mail address ' 'is not authorized to receive ' @@ -210,7 +211,7 @@ class FlowdockMethod(NotificationMethod): if not token: return - owner = model.get_user(notification.repository.namespace) + owner = model.get_user(notification.repository.namespace_user.username) if not owner: # Something went wrong. return @@ -223,7 +224,8 @@ class FlowdockMethod(NotificationMethod): 'subject': event_handler.get_summary(notification_data['event_data'], notification_data), 'content': event_handler.get_message(notification_data['event_data'], notification_data), 'from_name': owner.username, - 'project': notification.repository.namespace + ' ' + notification.repository.name, + 'project': (notification.repository.namespace_user.username + ' ' + + notification.repository.name), 'tags': ['#' + event_handler.event_name()], 'link': notification_data['event_data']['homepage'] } @@ -265,7 +267,7 @@ class HipchatMethod(NotificationMethod): if not token or not room_id: return - owner = model.get_user(notification.repository.namespace) + owner = model.get_user(notification.repository.namespace_user.username) if not owner: # Something went wrong. return @@ -332,7 +334,7 @@ class SlackMethod(NotificationMethod): if not token or not subdomain: return - owner = model.get_user(notification.repository.namespace) + owner = model.get_user(notification.repository.namespace_user.username) if not owner: # Something went wrong. return diff --git a/endpoints/web.py b/endpoints/web.py index 94c5583cd..63e463666 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -237,8 +237,8 @@ def confirm_repo_email(): Your E-mail address has been authorized to receive notifications for repository %s/%s. """ % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'], - record.repository.namespace, record.repository.name, - record.repository.namespace, record.repository.name) + record.repository.namespace_user.username, record.repository.name, + record.repository.namespace_user.username, record.repository.name) return render_page_template('message.html', message=message) diff --git a/initdb.py b/initdb.py index 5ae3c1ff2..cfb67a39a 100644 --- a/initdb.py +++ b/initdb.py @@ -51,7 +51,7 @@ def __gen_checksum(image_id): def __gen_image_id(repo, image_num): - str_to_hash = "%s/%s/%s" % (repo.namespace, repo.name, image_num) + str_to_hash = "%s/%s/%s" % (repo.namespace_user.username, repo.name, image_num) h = hashlib.md5(str_to_hash) return h.hexdigest() + h.hexdigest() @@ -79,11 +79,10 @@ def __create_subtree(repo, structure, creator_username, parent): creation_time = REFERENCE_DATE + timedelta(days=image_num) command_list = SAMPLE_CMDS[image_num % len(SAMPLE_CMDS)] command = json.dumps(command_list) if command_list else None - new_image = model.set_image_metadata(docker_image_id, repo.namespace, - repo.name, str(creation_time), - 'no comment', command, 0, parent) + new_image = model.set_image_metadata(docker_image_id, repo.namespace_user.username, repo.name, + str(creation_time), 'no comment', command, 0, parent) - model.set_image_size(docker_image_id, repo.namespace, repo.name, + model.set_image_size(docker_image_id, repo.namespace_user.username, repo.name, random.randrange(1, 1024 * 1024 * 1024)) # Populate the diff file @@ -100,7 +99,7 @@ def __create_subtree(repo, structure, creator_username, parent): last_node_tags = [last_node_tags] for tag_name in last_node_tags: - model.create_or_update_tag(repo.namespace, repo.name, tag_name, + model.create_or_update_tag(repo.namespace_user.username, repo.name, tag_name, new_image.docker_image_id) for subtree in subtrees: @@ -326,7 +325,8 @@ def populate_database(): outside_org.verified = True outside_org.save() - model.create_notification('test_notification', new_user_1, metadata={'some': 'value', 'arr': [1,2,3], 'obj': {'a': 1, 'b': 2}}) + model.create_notification('test_notification', new_user_1, + metadata={'some':'value', 'arr':[1, 2, 3], 'obj':{'a':1, 'b':2}}) from_date = datetime.utcnow() to_date = from_date + timedelta(hours=1) @@ -390,18 +390,20 @@ def populate_database(): }) trigger.save() - repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name) + repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace_user.username, building.name) job_config = { 'repository': repo, 'docker_tags': ['latest'], 'build_subdir': '', } - record = model.create_email_authorization_for_repo(new_user_1.username, 'simple', 'jschorr@devtable.com') + record = model.create_email_authorization_for_repo(new_user_1.username, 'simple', + 'jschorr@devtable.com') record.confirmed = True record.save() - model.create_email_authorization_for_repo(new_user_1.username, 'simple', 'jschorr+other@devtable.com') + model.create_email_authorization_for_repo(new_user_1.username, 'simple', + 'jschorr+other@devtable.com') build2 = model.create_repository_build(building, token, job_config, '68daeebd-a5b9-457f-80a0-4363b882f8ea', @@ -428,12 +430,12 @@ def populate_database(): model.create_robot('coolrobot', org) - oauth.create_application(org, 'Some Test App', 'http://localhost:8000', 'http://localhost:8000/o2c.html', - client_id='deadbeef') + oauth.create_application(org, 'Some Test App', 'http://localhost:8000', + 'http://localhost:8000/o2c.html', client_id='deadbeef') - oauth.create_application(org, 'Some Other Test App', 'http://quay.io', 'http://localhost:8000/o2c.html', - client_id='deadpork', - description = 'This is another test application') + oauth.create_application(org, 'Some Other Test App', 'http://quay.io', + 'http://localhost:8000/o2c.html', client_id='deadpork', + description='This is another test application') model.oauth.create_access_token_for_testing(new_user_1, 'deadbeef', 'repo:admin') @@ -455,8 +457,8 @@ def populate_database(): reader_team = model.create_team('readers', org, 'member', 'Readers of orgrepo.') - model.set_team_repo_permission(reader_team.name, org_repo.namespace, - org_repo.name, 'read') + model.set_team_repo_permission(reader_team.name, org_repo.namespace_user.username, org_repo.name, + 'read') model.add_user_to_team(new_user_2, reader_team) model.add_user_to_team(reader, reader_team) @@ -478,12 +480,9 @@ def populate_database(): (2, [], 'latest17'), (2, [], 'latest18'),]) - model.add_prototype_permission(org, 'read', activating_user=new_user_1, - delegate_user=new_user_2) - model.add_prototype_permission(org, 'read', activating_user=new_user_1, - delegate_team=reader_team) - model.add_prototype_permission(org, 'write', activating_user=new_user_2, - delegate_user=new_user_1) + model.add_prototype_permission(org, 'read', activating_user=new_user_1, delegate_user=new_user_2) + model.add_prototype_permission(org, 'read', activating_user=new_user_1, delegate_team=reader_team) + model.add_prototype_permission(org, 'write', activating_user=new_user_2, delegate_user=new_user_1) today = datetime.today() week_ago = today - timedelta(6) diff --git a/test/data/test.db b/test/data/test.db index afd5fe238..af85d23e2 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/tools/auditancestry.py b/tools/auditancestry.py index 95e082642..59d636836 100644 --- a/tools/auditancestry.py +++ b/tools/auditancestry.py @@ -1,7 +1,7 @@ import logging import json -from data.database import Image, ImageStorage, Repository, configure +from data.database import Image, ImageStorage, Repository, User, configure from data import model from app import app, storage as store @@ -16,17 +16,18 @@ logging.getLogger('boto').setLevel(logging.CRITICAL) query = (Image - .select(Image, ImageStorage, Repository) - .join(ImageStorage) - .switch(Image) - .join(Repository) - .where(ImageStorage.uploading == False)) + .select(Image, ImageStorage, Repository, User) + .join(ImageStorage) + .switch(Image) + .join(Repository) + .join(User) + .where(ImageStorage.uploading == False)) bad_count = 0 good_count = 0 def resolve_or_create(repo, docker_image_id, new_ancestry): - existing = model.get_repo_image(repo.namespace, repo.name, docker_image_id) + existing = model.get_repo_image(repo.namespace_user.username, repo.name, docker_image_id) if existing: logger.debug('Found existing image: %s, %s', existing.id, docker_image_id) return existing @@ -45,7 +46,7 @@ def resolve_or_create(repo, docker_image_id, new_ancestry): return created except ImageStorage.DoesNotExist: msg = 'No image available anywhere for storage: %s in namespace: %s' - logger.error(msg, docker_image_id, repo.namespace) + logger.error(msg, docker_image_id, repo.namespace_user.username) raise RuntimeError() @@ -62,20 +63,19 @@ def all_ancestors_exist(ancestors): cant_fix = [] for img in query: try: - with_locations = model.get_repo_image(img.repository.namespace, img.repository.name, - img.docker_image_id) + with_locations = model.get_repo_image(img.repository.namespace_user.username, + img.repository.name, img.docker_image_id) ancestry_storage = store.image_ancestry_path(img.storage.uuid) if store.exists(with_locations.storage.locations, ancestry_storage): - full_ancestry = json.loads(store.get_content(with_locations.storage.locations, ancestry_storage))[1:] + full_ancestry = json.loads(store.get_content(with_locations.storage.locations, + ancestry_storage))[1:] full_ancestry.reverse() - ancestor_dbids = [int(anc_id) - for anc_id in img.ancestors.split('/')[1:-1]] + ancestor_dbids = [int(anc_id) for anc_id in img.ancestors.split('/')[1:-1]] if len(full_ancestry) != len(ancestor_dbids) or not all_ancestors_exist(ancestor_dbids): - logger.error('Image has incomplete ancestry: %s, %s, %s, %s' % - (img.id, img.docker_image_id, full_ancestry, - ancestor_dbids)) + logger.error('Image has incomplete ancestry: %s, %s, %s, %s', img.id, img.docker_image_id, + full_ancestry, ancestor_dbids) fixed_ancestry = '/' for ancestor in full_ancestry: @@ -99,5 +99,5 @@ for img in query: len(cant_fix)) for cant in cant_fix: - logger.error('Unable to fix %s in repo %s/%s', cant.id, - cant.repository.namespace, cant.repository.name) \ No newline at end of file + logger.error('Unable to fix %s in repo %s/%s', cant.id, cant.repository.namespace_user.username, + cant.repository.name) diff --git a/tools/relationships.py b/tools/relationships.py index d701a18ce..c2cad5de5 100644 --- a/tools/relationships.py +++ b/tools/relationships.py @@ -9,7 +9,7 @@ with open('outfile.dot', 'w') as outfile: outfile.write('digraph relationships {\n') for repo in Repository.select(): - ns = fix_ident(repo.namespace) + ns = fix_ident(repo.namespace_user.username) outfile.write('%s_%s -> %s\n' % (ns, fix_ident(repo.name), ns)) teams_in_orgs = set()