vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Phase 2 of migrating repo namespaces to referencing user objects, backfilling the rows without a value for namespace_user, and changing all accesses to go through the namespace_user object. All tests are passing, manual testing still required.

Browse source
This commit is contained in:
Jake Moshenko 2014-09-24 18:01:35 -04:00
parent 6070c251ae
commit 03190efde3
19 changed files with 373 additions and 305 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
auth/permissions.py
View file

@ -112,7 +112,7 @@ class QuayDeferredPermissionUser(Identity):
# Add repository permissions
for perm in model.get_all_user_permissions(user_object):
repo_grant = _RepositoryNeed(perm.repository.namespace, perm.repository.name,
repo_grant = _RepositoryNeed(perm.repository.namespace_user.username, perm.repository.name,
self._repo_role_for_scopes(perm.role.name))
logger.debug('User added permission: {0}'.format(repo_grant))
self.provides.add(repo_grant)
@ -239,7 +239,7 @@ def on_identity_loaded(sender, identity):
logger.debug('Loading permissions for token: %s', identity.id)
token_data = model.load_token_data(identity.id)
repo_grant = _RepositoryNeed(token_data.repository.namespace,
repo_grant = _RepositoryNeed(token_data.repository.namespace_user.username,
token_data.repository.name,
token_data.role.name)
logger.debug('Delegate token added permission: {0}'.format(repo_grant))

4
data/database.py
View file

@ -169,7 +169,7 @@ class Visibility(BaseModel):
class Repository(BaseModel):
namespace = CharField()
namespace_user = ForeignKeyField(User, null=True)
namespace_user = ForeignKeyField(User)
name = CharField()
visibility = ForeignKeyField(Visibility)
description = TextField(null=True)
@ -181,7 +181,7 @@ class Repository(BaseModel):
indexes = (
# create a unique index on namespace and name
(('namespace', 'name'), True),
(('namespace_user', 'name'), False),
(('namespace_user', 'name'), True),
)

27
data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py Normal file
View file

@ -0,0 +1,27 @@
"""Backfill the namespace_user fields.
Revision ID: 3f4fe1194671
Revises: 6f2ecf5afcf
Create Date: 2014-09-24 14:29:45.192179
"""
# revision identifiers, used by Alembic.
revision = '3f4fe1194671'
down_revision = '6f2ecf5afcf'
from alembic import op
import sqlalchemy as sa
def upgrade(tables):
conn = op.get_bind()
conn.execute('update repository set namespace_user_id = (select id from user where user.username == repository.namespace) where namespace_user_id is NULL')
op.alter_column('repository', 'namespace_user_id', nullable=False)
op.create_index('repository_namespace_user_id_name', 'repository', ['namespace_user_id', 'name'], unique=True)
def downgrade(tables):
op.drop_index('repository_namespace_user_id_name', table_name='repository')
op.alter_column('repository', 'namespace_user_id', nullable=True)

352
data/model/legacy.py
View file

@ -25,6 +25,9 @@ EXPONENTIAL_BACKOFF_SCALE = timedelta(seconds=1)
PRESUMED_DEAD_BUILD_AGE = timedelta(days=15)
Namespace = User.alias()
logger = logging.getLogger(__name__)
@ -100,13 +103,24 @@ class TooManyLoginAttemptsException(Exception):
super(TooManyLoginAttemptsException, self).__init__(message)
self.retry_after = retry_after
def _get_repository(namespace_name, repository_name):
return (Repository
.select(Repository, Namespace)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace_name, Repository.name == repository_name)
.get())
def hash_password(password, salt=None):
salt = salt or bcrypt.gensalt()
return bcrypt.hashpw(password.encode('utf-8'), salt)
def is_create_user_allowed():
return True
def create_user(username, password, email, auto_verify=False):
""" Creates a regular user, if allowed. """
if not validate_password(password):
@ -122,6 +136,7 @@ def create_user(username, password, email, auto_verify=False):
return created
def _create_user(username, email):
if not validate_email(email):
raise InvalidEmailAddressException('Invalid email address: %s' % email)
@ -733,7 +748,7 @@ def get_visible_repositories(username=None, include_public=True, page=None,
limit=None, sort=False, namespace=None):
query = _visible_repository_query(username=username, include_public=include_public, page=page,
limit=limit, namespace=namespace,
select_models=[Repository, Visibility])
select_models=[Repository, Namespace, Visibility])
if sort:
query = query.order_by(Repository.description.desc())
@ -747,10 +762,12 @@ def get_visible_repositories(username=None, include_public=True, page=None,
def _visible_repository_query(username=None, include_public=True, limit=None,
page=None, namespace=None, select_models=[]):
query = (Repository
.select(*select_models) # Note: We need to leave this blank for the get_count case. Otherwise, MySQL/RDS complains.
.select(*select_models) # MySQL/RDS complains is there are selected models for counts.
.distinct()
.join(Visibility)
.switch(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER))
query = _filter_to_repos_for_user(query, username, namespace, include_public)
@ -782,26 +799,20 @@ def _filter_to_repos_for_user(query, username=None, namespace=None,
.switch(RepositoryPermission)
.join(Team, JOIN_LEFT_OUTER)
.join(TeamMember, JOIN_LEFT_OUTER)
.join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id ==
TeamMember.user))
.join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == TeamMember.user))
.switch(Repository)
.join(Org, JOIN_LEFT_OUTER, on=(Org.username == Repository.namespace))
.join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id ==
AdminTeam.organization))
.join(Org, JOIN_LEFT_OUTER, on=(Repository.namespace_user == Org.id))
.join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id == AdminTeam.organization))
.join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id))
.switch(AdminTeam)
.join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id ==
AdminTeamMember.team))
.join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user ==
AdminUser.id)))
.join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id == AdminTeamMember.team))
.join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user == AdminUser.id)))
where_clause = ((User.username == username) |
(UserThroughTeam.username == username) |
((AdminUser.username == username) &
(TeamRole.name == 'admin')))
where_clause = ((User.username == username) | (UserThroughTeam.username == username) |
((AdminUser.username == username) & (TeamRole.name == 'admin')))
if namespace:
where_clause = where_clause & (Repository.namespace == namespace)
where_clause = where_clause & (Namespace.username == namespace)
if include_public:
new_clause = (Visibility.name == 'public')
@ -820,7 +831,7 @@ def get_matching_repositories(repo_term, username=None):
visible = get_visible_repositories(username)
search_clauses = (Repository.name ** ('%' + name_term + '%') |
Repository.namespace ** ('%' + namespace_term + '%'))
Namespace.username ** ('%' + namespace_term + '%'))
# Handle the case where the user has already entered a namespace path.
if repo_term.find('/') > 0:
@ -829,7 +840,7 @@ def get_matching_repositories(repo_term, username=None):
name_term = parts[-1]
search_clauses = (Repository.name ** ('%' + name_term + '%') &
Repository.namespace ** ('%' + namespace_term + '%'))
Namespace.username ** ('%' + namespace_term + '%'))
final = visible.where(search_clauses).limit(10)
return list(final)
@ -859,22 +870,20 @@ def update_email(user, new_email, auto_verify=False):
def get_all_user_permissions(user):
select = RepositoryPermission.select(RepositoryPermission, Role, Repository)
with_role = select.join(Role)
with_repo = with_role.switch(RepositoryPermission).join(Repository)
through_user = with_repo.switch(RepositoryPermission).join(User,
JOIN_LEFT_OUTER)
as_perm = through_user.switch(RepositoryPermission)
through_team = as_perm.join(Team, JOIN_LEFT_OUTER).join(TeamMember,
JOIN_LEFT_OUTER)
UserThroughTeam = User.alias()
with_team_member = through_team.join(UserThroughTeam, JOIN_LEFT_OUTER,
on=(UserThroughTeam.id ==
TeamMember.user))
return with_team_member.where((User.id == user) |
(UserThroughTeam.id == user))
return (RepositoryPermission
.select(RepositoryPermission, Role, Repository, Namespace)
.join(Role)
.switch(RepositoryPermission)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(RepositoryPermission)
.join(User, JOIN_LEFT_OUTER)
.switch(RepositoryPermission)
.join(Team, JOIN_LEFT_OUTER).join(TeamMember, JOIN_LEFT_OUTER)
.join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == TeamMember.user))
.where((User.id == user) | (UserThroughTeam.id == user)))
def delete_prototype_permission(org, uid):
@ -939,33 +948,37 @@ def get_org_wide_permissions(user):
def get_all_repo_teams(namespace_name, repository_name):
select = RepositoryPermission.select(Team.name.alias('team_name'),
Role.name, RepositoryPermission)
with_team = select.join(Team)
with_role = with_team.switch(RepositoryPermission).join(Role)
with_repo = with_role.switch(RepositoryPermission).join(Repository)
return with_repo.where(Repository.namespace == namespace_name,
Repository.name == repository_name)
return (RepositoryPermission.select(Team.name.alias('team_name'), Role.name, RepositoryPermission)
.join(Team)
.switch(RepositoryPermission)
.join(Role)
.switch(RepositoryPermission)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace_name, Repository.name == repository_name))
def get_all_repo_users(namespace_name, repository_name):
select = RepositoryPermission.select(User.username, User.robot, Role.name,
RepositoryPermission)
with_user = select.join(User)
with_role = with_user.switch(RepositoryPermission).join(Role)
with_repo = with_role.switch(RepositoryPermission).join(Repository)
return with_repo.where(Repository.namespace == namespace_name,
Repository.name == repository_name)
return (RepositoryPermission.select(User.username, User.robot, Role.name, RepositoryPermission)
.join(User)
.switch(RepositoryPermission)
.join(Role)
.switch(RepositoryPermission)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace_name, Repository.name == repository_name))
def get_all_repo_users_transitive_via_teams(namespace_name, repository_name):
select = User.select().distinct()
with_team_member = select.join(TeamMember)
with_team = with_team_member.join(Team)
with_perm = with_team.join(RepositoryPermission)
with_repo = with_perm.join(Repository)
return with_repo.where(Repository.namespace == namespace_name,
Repository.name == repository_name)
return (User
.select()
.distinct()
.join(TeamMember)
.join(Team)
.join(RepositoryPermission)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace_name, Repository.name == repository_name))
def get_all_repo_users_transitive(namespace_name, repository_name):
@ -989,7 +1002,9 @@ def get_all_repo_users_transitive(namespace_name, repository_name):
def get_repository_for_resource(resource_key):
try:
return (Repository
.select()
.select(Repository, Namespace)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Repository)
.join(RepositoryBuild)
.where(RepositoryBuild.resource_key == resource_key)
.get())
@ -1006,8 +1021,7 @@ def lookup_repository(repo_id):
def get_repository(namespace_name, repository_name):
try:
return Repository.get(Repository.name == repository_name,
Repository.namespace == namespace_name)
return _get_repository(namespace_name, repository_name)
except Repository.DoesNotExist:
return None
@ -1024,11 +1038,18 @@ def get_repo_image(namespace_name, repository_name, image_id):
def repository_is_public(namespace_name, repository_name):
joined = Repository.select().join(Visibility)
query = joined.where(Repository.namespace == namespace_name,
Repository.name == repository_name,
try:
(Repository
.select()
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Repository)
.join(Visibility)
.where(Namespace.username == namespace_name, Repository.name == repository_name,
Visibility.name == 'public')
return len(list(query)) > 0
.get())
return True
except Repository.DoesNotExist:
return False
def set_repository_visibility(repo, visibility):
@ -1128,7 +1149,7 @@ def __translate_ancestry(old_ancestry, translations, repository, username, prefe
def find_create_or_link_image(docker_image_id, repository, username, translations,
preferred_location):
with config.app_config['DB_TRANSACTION_FACTORY'](db):
repo_image = get_repo_image(repository.namespace, repository.name,
repo_image = get_repo_image(repository.namespace_user.username, repository.name,
docker_image_id)
if repo_image:
return repo_image
@ -1142,6 +1163,8 @@ def find_create_or_link_image(docker_image_id, repository, username, translation
.join(Visibility)
.switch(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER)
.switch(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(ImageStorage.uploading == False))
query = (_filter_to_repos_for_user(query, username)
@ -1207,10 +1230,10 @@ def set_image_size(docker_image_id, namespace_name, repository_name,
image = (Image
.select(Image, ImageStorage)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Image)
.join(ImageStorage, JOIN_LEFT_OUTER)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
.where(Repository.name == repository_name, Namespace.username == namespace_name,
Image.docker_image_id == docker_image_id)
.get())
@ -1233,10 +1256,10 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
query = (Image
.select(Image, ImageStorage)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Image)
.join(ImageStorage)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
.where(Repository.name == repository_name, Namespace.username == namespace_name,
Image.docker_image_id == docker_image_id))
try:
@ -1267,8 +1290,8 @@ def _get_repository_images_base(namespace_name, repository_name, query_modifier)
.join(ImageStorage, JOIN_LEFT_OUTER)
.join(Image)
.join(Repository)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name))
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.name == repository_name, Namespace.username == namespace_name))
query = query_modifier(query)
@ -1299,11 +1322,13 @@ def get_repository_images(namespace_name, repository_name):
def list_repository_tags(namespace_name, repository_name):
select = RepositoryTag.select(RepositoryTag, Image)
with_repo = select.join(Repository)
with_image = with_repo.switch(RepositoryTag).join(Image)
return with_image.where(Repository.name == repository_name,
Repository.namespace == namespace_name)
return (RepositoryTag
.select(RepositoryTag, Image)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(RepositoryTag)
.join(Image)
.where(Repository.name == repository_name, Namespace.username == namespace_name))
def garbage_collect_repository(namespace_name, repository_name):
@ -1315,8 +1340,8 @@ def garbage_collect_repository(namespace_name, repository_name):
.join(ImageStorage, JOIN_LEFT_OUTER)
.switch(RepositoryTag)
.join(Repository)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name))
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.name == repository_name, Namespace.username == namespace_name))
referenced_anscestors = set()
for tag in tag_query:
@ -1407,22 +1432,17 @@ def get_parent_images(namespace_name, repository_name, image_obj):
def create_or_update_tag(namespace_name, repository_name, tag_name,
tag_docker_image_id):
try:
repo = Repository.get(Repository.name == repository_name,
Repository.namespace == namespace_name)
repo = _get_repository(namespace_name, repository_name)
except Repository.DoesNotExist:
raise DataModelException('Invalid repository %s/%s' %
(namespace_name, repository_name))
raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name))
try:
image = Image.get(Image.docker_image_id == tag_docker_image_id,
Image.repository == repo)
image = Image.get(Image.docker_image_id == tag_docker_image_id, Image.repository == repo)
except Image.DoesNotExist:
raise DataModelException('Invalid image with id: %s' %
tag_docker_image_id)
raise DataModelException('Invalid image with id: %s' % tag_docker_image_id)
try:
tag = RepositoryTag.get(RepositoryTag.repository == repo,
RepositoryTag.name == tag_name)
tag = RepositoryTag.get(RepositoryTag.repository == repo, RepositoryTag.name == tag_name)
tag.image = image
tag.save()
except RepositoryTag.DoesNotExist:
@ -1432,41 +1452,46 @@ def create_or_update_tag(namespace_name, repository_name, tag_name,
def delete_tag(namespace_name, repository_name, tag_name):
joined = RepositoryTag.select().join(Repository)
found = list(joined.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
RepositoryTag.name == tag_name))
try:
found = (RepositoryTag
.select()
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.name == repository_name, Namespace.username == namespace_name,
RepositoryTag.name == tag_name)
.get())
if not found:
except RepositoryTag.DoesNotExist:
msg = ('Invalid repository tag \'%s\' on repository \'%s/%s\'' %
(tag_name, namespace_name, repository_name))
raise DataModelException(msg)
found[0].delete_instance()
found.delete_instance()
def delete_all_repository_tags(namespace_name, repository_name):
try:
repo = Repository.get(Repository.name == repository_name,
Repository.namespace == namespace_name)
repo = _get_repository(namespace_name, repository_name)
except Repository.DoesNotExist:
raise DataModelException('Invalid repository \'%s/%s\'' %
(namespace_name, repository_name))
RepositoryTag.delete().where(RepositoryTag.repository == repo.id).execute()
def __entity_permission_repo_query(entity_id, entity_table,
entity_id_property, namespace_name,
def __entity_permission_repo_query(entity_id, entity_table, entity_id_property, namespace_name,
repository_name):
""" This method works for both users and teams. """
selected = RepositoryPermission.select(entity_table, Repository, Role,
RepositoryPermission)
with_user = selected.join(entity_table)
with_role = with_user.switch(RepositoryPermission).join(Role)
with_repo = with_role.switch(RepositoryPermission).join(Repository)
return with_repo.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
entity_id_property == entity_id)
return (RepositoryPermission
.select(entity_table, Repository, Namespace, Role, RepositoryPermission)
.join(entity_table)
.switch(RepositoryPermission)
.join(Role)
.switch(RepositoryPermission)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.name == repository_name, Namespace.username == namespace_name,
entity_id_property == entity_id))
def get_user_reponame_permission(username, namespace_name, repository_name):
@ -1514,8 +1539,7 @@ def delete_team_permission(team_name, namespace_name, repository_name):
def __set_entity_repo_permission(entity, permission_entity_property,
namespace_name, repository_name, role_name):
repo = Repository.get(Repository.name == repository_name,
Repository.namespace == namespace_name)
repo = _get_repository(namespace_name, repository_name)
new_role = Role.get(Role.name == role_name)
# Fetch any existing permission for this entity on the repo
@ -1566,15 +1590,18 @@ def purge_repository(namespace_name, repository_name):
garbage_collect_repository(namespace_name, repository_name)
# Delete the rest of the repository metadata
fetched = Repository.get(Repository.name == repository_name,
Repository.namespace == namespace_name)
fetched = _get_repository(namespace_name, repository_name)
fetched.delete_instance(recursive=True)
def get_private_repo_count(username):
joined = Repository.select().join(Visibility)
return joined.where(Repository.namespace == username,
Visibility.name == 'private').count()
return (Repository
.select()
.join(Visibility)
.switch(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == username, Visibility.name == 'private')
.count())
def create_access_token(repository, role):
@ -1587,21 +1614,22 @@ def create_access_token(repository, role):
def create_delegate_token(namespace_name, repository_name, friendly_name,
role='read'):
read_only = Role.get(name=role)
repo = Repository.get(Repository.name == repository_name,
Repository.namespace == namespace_name)
repo = _get_repository(namespace_name, repository_name)
new_token = AccessToken.create(repository=repo, role=read_only,
friendly_name=friendly_name, temporary=False)
return new_token
def get_repository_delegate_tokens(namespace_name, repository_name):
return (AccessToken.select(AccessToken, Role)
return (AccessToken
.select(AccessToken, Role)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(AccessToken)
.join(Role)
.switch(AccessToken)
.join(RepositoryBuildTrigger, JOIN_LEFT_OUTER)
.where(Repository.name == repository_name, Repository.namespace == namespace_name,
.where(Repository.name == repository_name, Namespace.username == namespace_name,
AccessToken.temporary == False, RepositoryBuildTrigger.uuid >> None))
@ -1636,14 +1664,17 @@ def delete_delegate_token(namespace_name, repository_name, code):
def load_token_data(code):
""" Load the permissions for any token by code. """
selected = AccessToken.select(AccessToken, Repository, Role)
with_role = selected.join(Role)
with_repo = with_role.switch(AccessToken).join(Repository)
fetched = list(with_repo.where(AccessToken.code == code))
try:
return (AccessToken
.select(AccessToken, Repository, Namespace, Role)
.join(Role)
.switch(AccessToken)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(AccessToken.code == code)
.get())
if fetched:
return fetched[0]
else:
except AccessToken.DoesNotExist:
raise InvalidTokenException('Invalid delegate token code: %s' % code)
@ -1662,11 +1693,11 @@ def list_repository_builds(namespace_name, repository_name, limit,
query = (RepositoryBuild
.select(RepositoryBuild, RepositoryBuildTrigger, BuildTriggerService)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(RepositoryBuild)
.join(RepositoryBuildTrigger, JOIN_LEFT_OUTER)
.join(BuildTriggerService, JOIN_LEFT_OUTER)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name)
.where(Repository.name == repository_name, Namespace.username == namespace_name)
.order_by(RepositoryBuild.started.desc())
.limit(limit))
@ -1732,16 +1763,17 @@ def create_repo_notification(repo, event_name, method_name, config):
def get_repo_notification(namespace_name, repository_name, uuid):
joined = RepositoryNotification.select().join(Repository)
found = list(joined.where(Repository.namespace == namespace_name,
Repository.name == repository_name,
RepositoryNotification.uuid == uuid))
if not found:
try:
return (RepositoryNotification
.select(RepositoryNotification, Repository, Namespace)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace_name, Repository.name == repository_name,
RepositoryNotification.uuid == uuid)
.get())
except RepositoryNotification.DoesNotExist:
raise InvalidNotificationException('No repository notification found with id: %s' % uuid)
return found[0]
def delete_repo_notification(namespace_name, repository_name, uuid):
found = get_repo_notification(namespace_name, repository_name, uuid)
@ -1750,15 +1782,19 @@ def delete_repo_notification(namespace_name, repository_name, uuid):
def list_repo_notifications(namespace_name, repository_name, event_name=None):
joined = RepositoryNotification.select().join(Repository)
where = joined.where(Repository.namespace == namespace_name,
Repository.name == repository_name)
query = (RepositoryNotification
.select(RepositoryNotification, Repository, Namespace)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace_name, Repository.name == repository_name))
if event_name:
event = ExternalNotificationEvent.get(ExternalNotificationEvent.name == event_name)
where = where.where(RepositoryNotification.event == event)
query = (query
.switch(RepositoryNotification)
.join(ExternalNotificationEvent)
.where(ExternalNotificationEvent.name == event_name))
return where
return query
def list_logs(start_time, end_time, performer=None, repository=None, namespace=None):
@ -1802,14 +1838,15 @@ def create_build_trigger(repo, service_name, auth_token, user, pull_robot=None):
def get_build_trigger(namespace_name, repository_name, trigger_uuid):
try:
return (RepositoryBuildTrigger
.select(RepositoryBuildTrigger, BuildTriggerService, Repository)
.select(RepositoryBuildTrigger, BuildTriggerService, Repository, Namespace)
.join(BuildTriggerService)
.switch(RepositoryBuildTrigger)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(RepositoryBuildTrigger)
.join(User)
.where(RepositoryBuildTrigger.uuid == trigger_uuid,
Repository.namespace == namespace_name,
Namespace.username == namespace_name,
Repository.name == repository_name)
.get())
except RepositoryBuildTrigger.DoesNotExist:
@ -1823,8 +1860,8 @@ def list_build_triggers(namespace_name, repository_name):
.join(BuildTriggerService)
.switch(RepositoryBuildTrigger)
.join(Repository)
.where(Repository.namespace == namespace_name,
Repository.name == repository_name))
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace_name, Repository.name == repository_name))
def list_trigger_builds(namespace_name, repository_name, trigger_uuid,
@ -1913,6 +1950,7 @@ def delete_notifications_by_kind(target, kind_name):
Notification.delete().where(Notification.target == target,
Notification.kind == kind_ref).execute()
def delete_matching_notifications(target, kind_name, **kwargs):
kind_ref = NotificationKind.get(name=kind_name)
@ -1943,6 +1981,7 @@ def delete_matching_notifications(target, kind_name, **kwargs):
def get_active_users():
return User.select().where(User.organization == False, User.robot == False)
def get_active_user_count():
return get_active_users().count()
@ -1956,11 +1995,13 @@ def detach_external_login(user, service_name):
FederatedLogin.delete().where(FederatedLogin.user == user,
FederatedLogin.service == service).execute()
def delete_user(user):
user.delete_instance(recursive=True, delete_nullable=True)
# TODO: also delete any repository data associated
def check_health():
# We will connect to the db, check that it contains some log entry kinds
try:
@ -1969,24 +2010,23 @@ def check_health():
except:
return False
def get_email_authorized_for_repo(namespace, repository, email):
found = list(RepositoryAuthorizedEmail.select()
.join(Repository)
.where(Repository.namespace == namespace,
Repository.name == repository,
RepositoryAuthorizedEmail.email == email)
.switch(RepositoryAuthorizedEmail)
.limit(1))
if not found or len(found) < 1:
return None
return found[0]
def get_email_authorized_for_repo(namespace, repository, email):
try:
return (RepositoryAuthorizedEmail
.select(RepositoryAuthorizedEmail, Repository, Namespace)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace, Repository.name == repository,
RepositoryAuthorizedEmail.email == email)
.get())
except RepositoryAuthorizedEmail.DoesNotExist:
return None
def create_email_authorization_for_repo(namespace_name, repository_name, email):
try:
repo = Repository.get(Repository.name == repository_name,
Repository.namespace == namespace_name)
repo = _get_repository(namespace_name, repository_name)
except Repository.DoesNotExist:
raise DataModelException('Invalid repository %s/%s' %
(namespace_name, repository_name))
@ -1996,7 +2036,11 @@ def create_email_authorization_for_repo(namespace_name, repository_name, email):
def confirm_email_authorization_for_repo(code):
try:
found = RepositoryAuthorizedEmail.get(RepositoryAuthorizedEmail.code == code)
found = (RepositoryAuthorizedEmail
.select(RepositoryAuthorizedEmail, Repository, Namespace)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.get(RepositoryAuthorizedEmail.code == code))
except RepositoryAuthorizedEmail.DoesNotExist:
raise DataModelException('Invalid confirmation code.')

2
endpoints/api/build.py
View file

@ -169,7 +169,7 @@ class RepositoryBuildList(RepositoryParamResource):
# was used.
associated_repository = model.get_repository_for_resource(dockerfile_id)
if associated_repository:
if not ModifyRepositoryPermission(associated_repository.namespace,
if not ModifyRepositoryPermission(associated_repository.namespace_user.username,
associated_repository.name):
raise Unauthorized()

2
endpoints/api/repoemail.py
View file

@ -20,7 +20,7 @@ def record_view(record):
return {
'email': record.email,
'repository': record.repository.name,
'namespace': record.repository.namespace,
'namespace': record.repository.namespace_user.username,
'confirmed': record.confirmed
}

8
endpoints/api/repository.py
View file

@ -80,8 +80,7 @@ class RepositoryList(ApiResource):
visibility = req['visibility']
repo = model.create_repository(namespace_name, repository_name, owner,
visibility)
repo = model.create_repository(namespace_name, repository_name, owner, visibility)
repo.description = req['description']
repo.save()
@ -110,7 +109,7 @@ class RepositoryList(ApiResource):
"""Fetch the list of repositories under a variety of situations."""
def repo_view(repo_obj):
return {
'namespace': repo_obj.namespace,
'namespace': repo_obj.namespace_user.username,
'name': repo_obj.name,
'description': repo_obj.description,
'is_public': repo_obj.visibility.name == 'public',
@ -134,7 +133,8 @@ class RepositoryList(ApiResource):
response['repositories'] = [repo_view(repo) for repo in repo_query
if (repo.visibility.name == 'public' or
ReadRepositoryPermission(repo.namespace, repo.name).can())]
ReadRepositoryPermission(repo.namespace_user.username,
repo.name).can())]
return response

4
endpoints/api/search.py
View file

@ -111,7 +111,7 @@ class FindRepositories(ApiResource):
def repo_view(repo):
return {
'namespace': repo.namespace,
'namespace': repo.namespace_user.username,
'name': repo.name,
'description': repo.description
}
@ -125,5 +125,5 @@ class FindRepositories(ApiResource):
return {
'repositories': [repo_view(repo) for repo in matching
if (repo.visibility.name == 'public' or
ReadRepositoryPermission(repo.namespace, repo.name).can())]
ReadRepositoryPermission(repo.namespace_user.username, repo.name).can())]
}

2
endpoints/api/trigger.py
View file

@ -205,7 +205,7 @@ class BuildTriggerActivate(RepositoryParamResource):
'write')
try:
repository_path = '%s/%s' % (trigger.repository.namespace,
repository_path = '%s/%s' % (trigger.repository.namespace_user.username,
trigger.repository.name)
path = url_for('webhooks.build_trigger_webhook',
repository=repository_path, trigger_uuid=trigger.uuid)

13
endpoints/common.py
View file

@ -205,7 +205,7 @@ def check_repository_usage(user_or_org, plan_found):
def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
trigger=None, pull_robot_name=None):
host = urlparse.urlparse(request.url).netloc
repo_path = '%s/%s/%s' % (host, repository.namespace, repository.name)
repo_path = '%s/%s/%s' % (host, repository.namespace_user.username, repository.name)
token = model.create_access_token(repository, 'write')
logger.debug('Creating build %s with repo %s tags %s and dockerfile_id %s',
@ -221,9 +221,9 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
dockerfile_id, build_name,
trigger, pull_robot_name=pull_robot_name)
dockerfile_build_queue.put([repository.namespace, repository.name], json.dumps({
dockerfile_build_queue.put([repository.namespace_user.username, repository.name], json.dumps({
'build_uuid': build_request.uuid,
'namespace': repository.namespace,
'namespace': repository.namespace_user.username,
'repository': repository.name,
'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None
}), retries_remaining=1)
@ -231,7 +231,7 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
# Add the build to the repo's log.
metadata = {
'repo': repository.name,
'namespace': repository.namespace,
'namespace': repository.namespace_user.username,
'fileid': dockerfile_id,
'manual': manual,
}
@ -241,9 +241,8 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
metadata['config'] = json.loads(trigger.config)
metadata['service'] = trigger.service.name
model.log_action('build_dockerfile', repository.namespace,
ip=request.remote_addr, metadata=metadata,
repository=repository)
model.log_action('build_dockerfile', repository.namespace_user.username, ip=request.remote_addr,
metadata=metadata, repository=repository)
# Add notifications for the build queue.
profile.debug('Adding notifications for repository')

4
endpoints/index.py
View file

@ -420,7 +420,7 @@ def put_repository_auth(namespace, repository):
def get_search():
def result_view(repo):
return {
"name": repo.namespace + '/' + repo.name,
"name": repo.namespace_user.username + '/' + repo.name,
"description": repo.description
}
@ -438,7 +438,7 @@ def get_search():
results = [result_view(repo) for repo in matching
if (repo.visibility.name == 'public' or
ReadRepositoryPermission(repo.namespace, repo.name).can())]
ReadRepositoryPermission(repo.namespace_user.username, repo.name).can())]
data = {
"query": query,

4
endpoints/notificationevent.py
View file

@ -1,8 +1,4 @@
import logging
import io
import os.path
import tarfile
import base64
from notificationhelper import build_event_data

11
endpoints/notificationhelper.py
View file

@ -4,7 +4,7 @@ from data import model
import json
def build_event_data(repo, extra_data={}, subpage=None):
repo_string = '%s/%s' % (repo.namespace, repo.name)
repo_string = '%s/%s' % (repo.namespace_user.username, repo.name)
homepage = '%s://%s/repository/%s' % (app.config['PREFERRED_URL_SCHEME'],
app.config['SERVER_HOSTNAME'],
repo_string)
@ -17,7 +17,7 @@ def build_event_data(repo, extra_data={}, subpage=None):
event_data = {
'repository': repo_string,
'namespace': repo.namespace,
'namespace': repo.namespace_user.username,
'name': repo.name,
'docker_url': '%s/%s' % (app.config['SERVER_HOSTNAME'], repo_string),
'homepage': homepage,
@ -30,7 +30,7 @@ def build_event_data(repo, extra_data={}, subpage=None):
def build_notification_data(notification, event_data):
return {
'notification_uuid': notification.uuid,
'repository_namespace': notification.repository.namespace,
'repository_namespace': notification.repository.namespace_user.username,
'repository_name': notification.repository.name,
'event_data': event_data
}
@ -39,8 +39,9 @@ def build_notification_data(notification, event_data):
def spawn_notification(repo, event_name, extra_data={}, subpage=None, pathargs=[]):
event_data = build_event_data(repo, extra_data=extra_data, subpage=subpage)
notifications = model.list_repo_notifications(repo.namespace, repo.name, event_name=event_name)
notifications = model.list_repo_notifications(repo.namespace_user.username, repo.name,
event_name=event_name)
for notification in notifications:
notification_data = build_notification_data(notification, event_data)
path = [repo.namespace, repo.name, event_name] + pathargs
path = [repo.namespace_user.username, repo.name, event_name] + pathargs
notification_queue.put(path, json.dumps(notification_data))

16
endpoints/notificationmethod.py
View file

@ -88,7 +88,7 @@ class QuayNotificationMethod(NotificationMethod):
return (True, 'Unknown organization %s' % target_info['name'], None)
# Only repositories under the organization can cause notifications to that org.
if target_info['name'] != repository.namespace:
if target_info['name'] != repository.namespace_user.username:
return (False, 'Organization name must match repository namespace')
return (True, None, [target])
@ -96,7 +96,7 @@ class QuayNotificationMethod(NotificationMethod):
# Lookup the team.
team = None
try:
team = model.get_organization_team(repository.namespace, target_info['name'])
team = model.get_organization_team(repository.namespace_user.username, target_info['name'])
except model.InvalidTeamException:
# Probably deleted.
return (True, 'Unknown team %s' % target_info['name'], None)
@ -133,7 +133,8 @@ class EmailMethod(NotificationMethod):
if not email:
raise CannotValidateNotificationMethodException('Missing e-mail address')
record = model.get_email_authorized_for_repo(repository.namespace, repository.name, email)
record = model.get_email_authorized_for_repo(repository.namespace_user.username,
repository.name, email)
if not record or not record.confirmed:
raise CannotValidateNotificationMethodException('The specified e-mail address '
'is not authorized to receive '
@ -210,7 +211,7 @@ class FlowdockMethod(NotificationMethod):
if not token:
return
owner = model.get_user(notification.repository.namespace)
owner = model.get_user(notification.repository.namespace_user.username)
if not owner:
# Something went wrong.
return
@ -223,7 +224,8 @@ class FlowdockMethod(NotificationMethod):
'subject': event_handler.get_summary(notification_data['event_data'], notification_data),
'content': event_handler.get_message(notification_data['event_data'], notification_data),
'from_name': owner.username,
'project': notification.repository.namespace + ' ' + notification.repository.name,
'project': (notification.repository.namespace_user.username + ' ' +
notification.repository.name),
'tags': ['#' + event_handler.event_name()],
'link': notification_data['event_data']['homepage']
}
@ -265,7 +267,7 @@ class HipchatMethod(NotificationMethod):
if not token or not room_id:
return
owner = model.get_user(notification.repository.namespace)
owner = model.get_user(notification.repository.namespace_user.username)
if not owner:
# Something went wrong.
return
@ -332,7 +334,7 @@ class SlackMethod(NotificationMethod):
if not token or not subdomain:
return
owner = model.get_user(notification.repository.namespace)
owner = model.get_user(notification.repository.namespace_user.username)
if not owner:
# Something went wrong.
return

4
endpoints/web.py
View file

@ -237,8 +237,8 @@ def confirm_repo_email():
Your E-mail address has been authorized to receive notifications for repository
<a href="%s://%s/repository/%s/%s">%s/%s</a>.
""" % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'],
record.repository.namespace, record.repository.name,
record.repository.namespace, record.repository.name)
record.repository.namespace_user.username, record.repository.name,
record.repository.namespace_user.username, record.repository.name)
return render_page_template('message.html', message=message)

45
initdb.py
View file

@ -51,7 +51,7 @@ def __gen_checksum(image_id):
def __gen_image_id(repo, image_num):
str_to_hash = "%s/%s/%s" % (repo.namespace, repo.name, image_num)
str_to_hash = "%s/%s/%s" % (repo.namespace_user.username, repo.name, image_num)
h = hashlib.md5(str_to_hash)
return h.hexdigest() + h.hexdigest()
@ -79,11 +79,10 @@ def __create_subtree(repo, structure, creator_username, parent):
creation_time = REFERENCE_DATE + timedelta(days=image_num)
command_list = SAMPLE_CMDS[image_num % len(SAMPLE_CMDS)]
command = json.dumps(command_list) if command_list else None
new_image = model.set_image_metadata(docker_image_id, repo.namespace,
repo.name, str(creation_time),
'no comment', command, 0, parent)
new_image = model.set_image_metadata(docker_image_id, repo.namespace_user.username, repo.name,
str(creation_time), 'no comment', command, 0, parent)
model.set_image_size(docker_image_id, repo.namespace, repo.name,
model.set_image_size(docker_image_id, repo.namespace_user.username, repo.name,
random.randrange(1, 1024 * 1024 * 1024))
# Populate the diff file
@ -100,7 +99,7 @@ def __create_subtree(repo, structure, creator_username, parent):
last_node_tags = [last_node_tags]
for tag_name in last_node_tags:
model.create_or_update_tag(repo.namespace, repo.name, tag_name,
model.create_or_update_tag(repo.namespace_user.username, repo.name, tag_name,
new_image.docker_image_id)
for subtree in subtrees:
@ -326,7 +325,8 @@ def populate_database():
outside_org.verified = True
outside_org.save()
model.create_notification('test_notification', new_user_1, metadata={'some': 'value', 'arr': [1,2,3], 'obj': {'a': 1, 'b': 2}})
model.create_notification('test_notification', new_user_1,
metadata={'some':'value', 'arr':[1, 2, 3], 'obj':{'a':1, 'b':2}})
from_date = datetime.utcnow()
to_date = from_date + timedelta(hours=1)
@ -390,18 +390,20 @@ def populate_database():
})
trigger.save()
repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name)
repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace_user.username, building.name)
job_config = {
'repository': repo,
'docker_tags': ['latest'],
'build_subdir': '',
}
record = model.create_email_authorization_for_repo(new_user_1.username, 'simple', 'jschorr@devtable.com')
record = model.create_email_authorization_for_repo(new_user_1.username, 'simple',
'jschorr@devtable.com')
record.confirmed = True
record.save()
model.create_email_authorization_for_repo(new_user_1.username, 'simple', 'jschorr+other@devtable.com')
model.create_email_authorization_for_repo(new_user_1.username, 'simple',
'jschorr+other@devtable.com')
build2 = model.create_repository_build(building, token, job_config,
'68daeebd-a5b9-457f-80a0-4363b882f8ea',
@ -428,12 +430,12 @@ def populate_database():
model.create_robot('coolrobot', org)
oauth.create_application(org, 'Some Test App', 'http://localhost:8000', 'http://localhost:8000/o2c.html',
client_id='deadbeef')
oauth.create_application(org, 'Some Test App', 'http://localhost:8000',
'http://localhost:8000/o2c.html', client_id='deadbeef')
oauth.create_application(org, 'Some Other Test App', 'http://quay.io', 'http://localhost:8000/o2c.html',
client_id='deadpork',
description = 'This is another test application')
oauth.create_application(org, 'Some Other Test App', 'http://quay.io',
'http://localhost:8000/o2c.html', client_id='deadpork',
description='This is another test application')
model.oauth.create_access_token_for_testing(new_user_1, 'deadbeef', 'repo:admin')
@ -455,8 +457,8 @@ def populate_database():
reader_team = model.create_team('readers', org, 'member',
'Readers of orgrepo.')
model.set_team_repo_permission(reader_team.name, org_repo.namespace,
org_repo.name, 'read')
model.set_team_repo_permission(reader_team.name, org_repo.namespace_user.username, org_repo.name,
'read')
model.add_user_to_team(new_user_2, reader_team)
model.add_user_to_team(reader, reader_team)
@ -478,12 +480,9 @@ def populate_database():
(2, [], 'latest17'),
(2, [], 'latest18'),])
model.add_prototype_permission(org, 'read', activating_user=new_user_1,
delegate_user=new_user_2)
model.add_prototype_permission(org, 'read', activating_user=new_user_1,
delegate_team=reader_team)
model.add_prototype_permission(org, 'write', activating_user=new_user_2,
delegate_user=new_user_1)
model.add_prototype_permission(org, 'read', activating_user=new_user_1, delegate_user=new_user_2)
model.add_prototype_permission(org, 'read', activating_user=new_user_1, delegate_team=reader_team)
model.add_prototype_permission(org, 'write', activating_user=new_user_2, delegate_user=new_user_1)
today = datetime.today()
week_ago = today - timedelta(6)

BIN
test/data/test.db
View file

Binary file not shown.

28
tools/auditancestry.py
View file

@ -1,7 +1,7 @@
import logging
import json
from data.database import Image, ImageStorage, Repository, configure
from data.database import Image, ImageStorage, Repository, User, configure
from data import model
from app import app, storage as store
@ -16,17 +16,18 @@ logging.getLogger('boto').setLevel(logging.CRITICAL)
query = (Image
.select(Image, ImageStorage, Repository)
.select(Image, ImageStorage, Repository, User)
.join(ImageStorage)
.switch(Image)
.join(Repository)
.join(User)
.where(ImageStorage.uploading == False))
bad_count = 0
good_count = 0
def resolve_or_create(repo, docker_image_id, new_ancestry):
existing = model.get_repo_image(repo.namespace, repo.name, docker_image_id)
existing = model.get_repo_image(repo.namespace_user.username, repo.name, docker_image_id)
if existing:
logger.debug('Found existing image: %s, %s', existing.id, docker_image_id)
return existing
@ -45,7 +46,7 @@ def resolve_or_create(repo, docker_image_id, new_ancestry):
return created
except ImageStorage.DoesNotExist:
msg = 'No image available anywhere for storage: %s in namespace: %s'
logger.error(msg, docker_image_id, repo.namespace)
logger.error(msg, docker_image_id, repo.namespace_user.username)
raise RuntimeError()
@ -62,20 +63,19 @@ def all_ancestors_exist(ancestors):
cant_fix = []
for img in query:
try:
with_locations = model.get_repo_image(img.repository.namespace, img.repository.name,
img.docker_image_id)
with_locations = model.get_repo_image(img.repository.namespace_user.username,
img.repository.name, img.docker_image_id)
ancestry_storage = store.image_ancestry_path(img.storage.uuid)
if store.exists(with_locations.storage.locations, ancestry_storage):
full_ancestry = json.loads(store.get_content(with_locations.storage.locations, ancestry_storage))[1:]
full_ancestry = json.loads(store.get_content(with_locations.storage.locations,
ancestry_storage))[1:]
full_ancestry.reverse()
ancestor_dbids = [int(anc_id)
for anc_id in img.ancestors.split('/')[1:-1]]
ancestor_dbids = [int(anc_id) for anc_id in img.ancestors.split('/')[1:-1]]
if len(full_ancestry) != len(ancestor_dbids) or not all_ancestors_exist(ancestor_dbids):
logger.error('Image has incomplete ancestry: %s, %s, %s, %s' %
(img.id, img.docker_image_id, full_ancestry,
ancestor_dbids))
logger.error('Image has incomplete ancestry: %s, %s, %s, %s', img.id, img.docker_image_id,
full_ancestry, ancestor_dbids)
fixed_ancestry = '/'
for ancestor in full_ancestry:
@ -99,5 +99,5 @@ for img in query:
len(cant_fix))
for cant in cant_fix:
logger.error('Unable to fix %s in repo %s/%s', cant.id,
cant.repository.namespace, cant.repository.name)
logger.error('Unable to fix %s in repo %s/%s', cant.id, cant.repository.namespace_user.username,
cant.repository.name)

2
tools/relationships.py
View file

@ -9,7 +9,7 @@ with open('outfile.dot', 'w') as outfile:
outfile.write('digraph relationships {\n')
for repo in Repository.select():
ns = fix_ident(repo.namespace)
ns = fix_ident(repo.namespace_user.username)
outfile.write('%s_%s -> %s\n' % (ns, fix_ident(repo.name), ns))
teams_in_orgs = set()