vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Phase 2 of migrating repo namespaces to referencing user objects, backfilling the rows without a value for namespace_user, and changing all accesses to go through the namespace_user object. All tests are passing, manual testing still required.

Browse source
This commit is contained in:
Jake Moshenko 2014-09-24 18:01:35 -04:00
parent 6070c251ae
commit 03190efde3
19 changed files with 373 additions and 305 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
auth/permissions.py
View file

@ -112,7 +112,7 @@ class QuayDeferredPermissionUser(Identity):
# Add repository permissions # Add repository permissions
for perm in model.get_all_user_permissions(user_object): for perm in model.get_all_user_permissions(user_object):
repo_grant = _RepositoryNeed(perm.repository.namespace, perm.repository.name, repo_grant = _RepositoryNeed(perm.repository.namespace_user.username, perm.repository.name,
self._repo_role_for_scopes(perm.role.name)) self._repo_role_for_scopes(perm.role.name))
logger.debug('User added permission: {0}'.format(repo_grant)) logger.debug('User added permission: {0}'.format(repo_grant))
self.provides.add(repo_grant) self.provides.add(repo_grant)
@ -239,7 +239,7 @@ def on_identity_loaded(sender, identity):
logger.debug('Loading permissions for token: %s', identity.id) logger.debug('Loading permissions for token: %s', identity.id)
token_data = model.load_token_data(identity.id) token_data = model.load_token_data(identity.id)
repo_grant = _RepositoryNeed(token_data.repository.namespace, repo_grant = _RepositoryNeed(token_data.repository.namespace_user.username,
token_data.repository.name, token_data.repository.name,
token_data.role.name) token_data.role.name)
logger.debug('Delegate token added permission: {0}'.format(repo_grant)) logger.debug('Delegate token added permission: {0}'.format(repo_grant))

4
data/database.py
View file

@ -169,7 +169,7 @@ class Visibility(BaseModel):
class Repository(BaseModel): class Repository(BaseModel):
namespace = CharField() namespace = CharField()
namespace_user = ForeignKeyField(User, null=True) namespace_user = ForeignKeyField(User)
name = CharField() name = CharField()
visibility = ForeignKeyField(Visibility) visibility = ForeignKeyField(Visibility)
description = TextField(null=True) description = TextField(null=True)
@ -181,7 +181,7 @@ class Repository(BaseModel):
indexes = ( indexes = (
# create a unique index on namespace and name # create a unique index on namespace and name
(('namespace', 'name'), True), (('namespace', 'name'), True),
(('namespace_user', 'name'), False), (('namespace_user', 'name'), True),
) )

27
data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py Normal file
View file

@ -0,0 +1,27 @@
"""Backfill the namespace_user fields.
Revision ID: 3f4fe1194671
Revises: 6f2ecf5afcf
Create Date: 2014-09-24 14:29:45.192179
"""
# revision identifiers, used by Alembic.
revision = '3f4fe1194671'
down_revision = '6f2ecf5afcf'
from alembic import op
import sqlalchemy as sa
def upgrade(tables):
conn = op.get_bind()
conn.execute('update repository set namespace_user_id = (select id from user where user.username == repository.namespace) where namespace_user_id is NULL')
op.alter_column('repository', 'namespace_user_id', nullable=False)
op.create_index('repository_namespace_user_id_name', 'repository', ['namespace_user_id', 'name'], unique=True)
def downgrade(tables):
op.drop_index('repository_namespace_user_id_name', table_name='repository')
op.alter_column('repository', 'namespace_user_id', nullable=True)

490
data/model/legacy.py
View file

@ -25,6 +25,9 @@ EXPONENTIAL_BACKOFF_SCALE = timedelta(seconds=1)
PRESUMED_DEAD_BUILD_AGE = timedelta(days=15) PRESUMED_DEAD_BUILD_AGE = timedelta(days=15)
Namespace = User.alias()
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -100,13 +103,24 @@ class TooManyLoginAttemptsException(Exception):
super(TooManyLoginAttemptsException, self).__init__(message) super(TooManyLoginAttemptsException, self).__init__(message)
self.retry_after = retry_after self.retry_after = retry_after
def _get_repository(namespace_name, repository_name):
return (Repository
.select(Repository, Namespace)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace_name, Repository.name == repository_name)
.get())
def hash_password(password, salt=None): def hash_password(password, salt=None):
salt = salt or bcrypt.gensalt() salt = salt or bcrypt.gensalt()
return bcrypt.hashpw(password.encode('utf-8'), salt) return bcrypt.hashpw(password.encode('utf-8'), salt)
def is_create_user_allowed(): def is_create_user_allowed():
return True return True
def create_user(username, password, email, auto_verify=False): def create_user(username, password, email, auto_verify=False):
""" Creates a regular user, if allowed. """ """ Creates a regular user, if allowed. """
if not validate_password(password): if not validate_password(password):
@ -122,6 +136,7 @@ def create_user(username, password, email, auto_verify=False):
return created return created
def _create_user(username, email): def _create_user(username, email):
if not validate_email(email): if not validate_email(email):
raise InvalidEmailAddressException('Invalid email address: %s' % email) raise InvalidEmailAddressException('Invalid email address: %s' % email)
@ -733,7 +748,7 @@ def get_visible_repositories(username=None, include_public=True, page=None,
limit=None, sort=False, namespace=None): limit=None, sort=False, namespace=None):
query = _visible_repository_query(username=username, include_public=include_public, page=page, query = _visible_repository_query(username=username, include_public=include_public, page=page,
limit=limit, namespace=namespace, limit=limit, namespace=namespace,
select_models=[Repository, Visibility]) select_models=[Repository, Namespace, Visibility])
if sort: if sort:
query = query.order_by(Repository.description.desc()) query = query.order_by(Repository.description.desc())
@ -747,11 +762,13 @@ def get_visible_repositories(username=None, include_public=True, page=None,
def _visible_repository_query(username=None, include_public=True, limit=None, def _visible_repository_query(username=None, include_public=True, limit=None,
page=None, namespace=None, select_models=[]): page=None, namespace=None, select_models=[]):
query = (Repository query = (Repository
.select(*select_models) # Note: We need to leave this blank for the get_count case. Otherwise, MySQL/RDS complains. .select(*select_models) # MySQL/RDS complains is there are selected models for counts.
.distinct() .distinct()
.join(Visibility) .join(Visibility)
.switch(Repository) .switch(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER)) .join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER))
query = _filter_to_repos_for_user(query, username, namespace, include_public) query = _filter_to_repos_for_user(query, username, namespace, include_public)
@ -782,26 +799,20 @@ def _filter_to_repos_for_user(query, username=None, namespace=None,
.switch(RepositoryPermission) .switch(RepositoryPermission)
.join(Team, JOIN_LEFT_OUTER) .join(Team, JOIN_LEFT_OUTER)
.join(TeamMember, JOIN_LEFT_OUTER) .join(TeamMember, JOIN_LEFT_OUTER)
.join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == TeamMember.user))
TeamMember.user))
.switch(Repository) .switch(Repository)
.join(Org, JOIN_LEFT_OUTER, on=(Org.username == Repository.namespace)) .join(Org, JOIN_LEFT_OUTER, on=(Repository.namespace_user == Org.id))
.join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id == .join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id == AdminTeam.organization))
AdminTeam.organization))
.join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id)) .join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id))
.switch(AdminTeam) .switch(AdminTeam)
.join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id == .join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id == AdminTeamMember.team))
AdminTeamMember.team)) .join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user == AdminUser.id)))
.join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user ==
AdminUser.id)))
where_clause = ((User.username == username) | where_clause = ((User.username == username) | (UserThroughTeam.username == username) |
(UserThroughTeam.username == username) | ((AdminUser.username == username) & (TeamRole.name == 'admin')))
((AdminUser.username == username) &
(TeamRole.name == 'admin')))
if namespace: if namespace:
where_clause = where_clause & (Repository.namespace == namespace) where_clause = where_clause & (Namespace.username == namespace)
if include_public: if include_public:
new_clause = (Visibility.name == 'public') new_clause = (Visibility.name == 'public')
@ -820,7 +831,7 @@ def get_matching_repositories(repo_term, username=None):
visible = get_visible_repositories(username) visible = get_visible_repositories(username)
search_clauses = (Repository.name ** ('%' + name_term + '%') | search_clauses = (Repository.name ** ('%' + name_term + '%') |
Repository.namespace ** ('%' + namespace_term + '%')) Namespace.username ** ('%' + namespace_term + '%'))
# Handle the case where the user has already entered a namespace path. # Handle the case where the user has already entered a namespace path.
if repo_term.find('/') > 0: if repo_term.find('/') > 0:
@ -829,7 +840,7 @@ def get_matching_repositories(repo_term, username=None):
name_term = parts[-1] name_term = parts[-1]
search_clauses = (Repository.name ** ('%' + name_term + '%') & search_clauses = (Repository.name ** ('%' + name_term + '%') &
Repository.namespace ** ('%' + namespace_term + '%')) Namespace.username ** ('%' + namespace_term + '%'))
final = visible.where(search_clauses).limit(10) final = visible.where(search_clauses).limit(10)
return list(final) return list(final)
@ -859,22 +870,20 @@ def update_email(user, new_email, auto_verify=False):
def get_all_user_permissions(user): def get_all_user_permissions(user):
select = RepositoryPermission.select(RepositoryPermission, Role, Repository)
with_role = select.join(Role)
with_repo = with_role.switch(RepositoryPermission).join(Repository)
through_user = with_repo.switch(RepositoryPermission).join(User,
JOIN_LEFT_OUTER)
as_perm = through_user.switch(RepositoryPermission)
through_team = as_perm.join(Team, JOIN_LEFT_OUTER).join(TeamMember,
JOIN_LEFT_OUTER)
UserThroughTeam = User.alias() UserThroughTeam = User.alias()
with_team_member = through_team.join(UserThroughTeam, JOIN_LEFT_OUTER,
on=(UserThroughTeam.id ==
TeamMember.user))
return with_team_member.where((User.id == user) | return (RepositoryPermission
(UserThroughTeam.id == user)) .select(RepositoryPermission, Role, Repository, Namespace)
.join(Role)
.switch(RepositoryPermission)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(RepositoryPermission)
.join(User, JOIN_LEFT_OUTER)
.switch(RepositoryPermission)
.join(Team, JOIN_LEFT_OUTER).join(TeamMember, JOIN_LEFT_OUTER)
.join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == TeamMember.user))
.where((User.id == user) | (UserThroughTeam.id == user)))
def delete_prototype_permission(org, uid): def delete_prototype_permission(org, uid):
@ -939,33 +948,37 @@ def get_org_wide_permissions(user):
def get_all_repo_teams(namespace_name, repository_name): def get_all_repo_teams(namespace_name, repository_name):
select = RepositoryPermission.select(Team.name.alias('team_name'), return (RepositoryPermission.select(Team.name.alias('team_name'), Role.name, RepositoryPermission)
Role.name, RepositoryPermission) .join(Team)
with_team = select.join(Team) .switch(RepositoryPermission)
with_role = with_team.switch(RepositoryPermission).join(Role) .join(Role)
with_repo = with_role.switch(RepositoryPermission).join(Repository) .switch(RepositoryPermission)
return with_repo.where(Repository.namespace == namespace_name, .join(Repository)
Repository.name == repository_name) .join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace_name, Repository.name == repository_name))
def get_all_repo_users(namespace_name, repository_name): def get_all_repo_users(namespace_name, repository_name):
select = RepositoryPermission.select(User.username, User.robot, Role.name, return (RepositoryPermission.select(User.username, User.robot, Role.name, RepositoryPermission)
RepositoryPermission) .join(User)
with_user = select.join(User) .switch(RepositoryPermission)
with_role = with_user.switch(RepositoryPermission).join(Role) .join(Role)
with_repo = with_role.switch(RepositoryPermission).join(Repository) .switch(RepositoryPermission)
return with_repo.where(Repository.namespace == namespace_name, .join(Repository)
Repository.name == repository_name) .join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace_name, Repository.name == repository_name))
def get_all_repo_users_transitive_via_teams(namespace_name, repository_name): def get_all_repo_users_transitive_via_teams(namespace_name, repository_name):
select = User.select().distinct() return (User
with_team_member = select.join(TeamMember) .select()
with_team = with_team_member.join(Team) .distinct()
with_perm = with_team.join(RepositoryPermission) .join(TeamMember)
with_repo = with_perm.join(Repository) .join(Team)
return with_repo.where(Repository.namespace == namespace_name, .join(RepositoryPermission)
Repository.name == repository_name) .join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace_name, Repository.name == repository_name))
def get_all_repo_users_transitive(namespace_name, repository_name): def get_all_repo_users_transitive(namespace_name, repository_name):
@ -989,10 +1002,12 @@ def get_all_repo_users_transitive(namespace_name, repository_name):
def get_repository_for_resource(resource_key): def get_repository_for_resource(resource_key):
try: try:
return (Repository return (Repository
.select() .select(Repository, Namespace)
.join(RepositoryBuild) .join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(RepositoryBuild.resource_key == resource_key) .switch(Repository)
.get()) .join(RepositoryBuild)
.where(RepositoryBuild.resource_key == resource_key)
.get())
except Repository.DoesNotExist: except Repository.DoesNotExist:
return None return None
@ -1006,8 +1021,7 @@ def lookup_repository(repo_id):
def get_repository(namespace_name, repository_name): def get_repository(namespace_name, repository_name):
try: try:
return Repository.get(Repository.name == repository_name, return _get_repository(namespace_name, repository_name)
Repository.namespace == namespace_name)
except Repository.DoesNotExist: except Repository.DoesNotExist:
return None return None
@ -1024,11 +1038,18 @@ def get_repo_image(namespace_name, repository_name, image_id):
def repository_is_public(namespace_name, repository_name): def repository_is_public(namespace_name, repository_name):
joined = Repository.select().join(Visibility) try:
query = joined.where(Repository.namespace == namespace_name, (Repository
Repository.name == repository_name, .select()
Visibility.name == 'public') .join(Namespace, on=(Repository.namespace_user == Namespace.id))
return len(list(query)) > 0 .switch(Repository)
.join(Visibility)
.where(Namespace.username == namespace_name, Repository.name == repository_name,
Visibility.name == 'public')
.get())
return True
except Repository.DoesNotExist:
return False
def set_repository_visibility(repo, visibility): def set_repository_visibility(repo, visibility):
@ -1128,7 +1149,7 @@ def __translate_ancestry(old_ancestry, translations, repository, username, prefe
def find_create_or_link_image(docker_image_id, repository, username, translations, def find_create_or_link_image(docker_image_id, repository, username, translations,
preferred_location): preferred_location):
with config.app_config['DB_TRANSACTION_FACTORY'](db): with config.app_config['DB_TRANSACTION_FACTORY'](db):
repo_image = get_repo_image(repository.namespace, repository.name, repo_image = get_repo_image(repository.namespace_user.username, repository.name,
docker_image_id) docker_image_id)
if repo_image: if repo_image:
return repo_image return repo_image
@ -1142,6 +1163,8 @@ def find_create_or_link_image(docker_image_id, repository, username, translation
.join(Visibility) .join(Visibility)
.switch(Repository) .switch(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER) .join(RepositoryPermission, JOIN_LEFT_OUTER)
.switch(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(ImageStorage.uploading == False)) .where(ImageStorage.uploading == False))
query = (_filter_to_repos_for_user(query, username) query = (_filter_to_repos_for_user(query, username)
@ -1186,11 +1209,11 @@ def find_create_or_link_image(docker_image_id, repository, username, translation
def get_storage_by_uuid(storage_uuid): def get_storage_by_uuid(storage_uuid):
placements = list(ImageStoragePlacement placements = list(ImageStoragePlacement
.select(ImageStoragePlacement, ImageStorage, ImageStorageLocation) .select(ImageStoragePlacement, ImageStorage, ImageStorageLocation)
.join(ImageStorageLocation) .join(ImageStorageLocation)
.switch(ImageStoragePlacement) .switch(ImageStoragePlacement)
.join(ImageStorage) .join(ImageStorage)
.where(ImageStorage.uuid == storage_uuid)) .where(ImageStorage.uuid == storage_uuid))
if not placements: if not placements:
raise InvalidImageException('No storage found with uuid: %s', storage_uuid) raise InvalidImageException('No storage found with uuid: %s', storage_uuid)
@ -1205,14 +1228,14 @@ def set_image_size(docker_image_id, namespace_name, repository_name,
image_size): image_size):
try: try:
image = (Image image = (Image
.select(Image, ImageStorage) .select(Image, ImageStorage)
.join(Repository) .join(Repository)
.switch(Image) .join(Namespace, on=(Repository.namespace_user == Namespace.id))
.join(ImageStorage, JOIN_LEFT_OUTER) .switch(Image)
.where(Repository.name == repository_name, .join(ImageStorage, JOIN_LEFT_OUTER)
Repository.namespace == namespace_name, .where(Repository.name == repository_name, Namespace.username == namespace_name,
Image.docker_image_id == docker_image_id) Image.docker_image_id == docker_image_id)
.get()) .get())
except Image.DoesNotExist: except Image.DoesNotExist:
raise DataModelException('No image with specified id and repository') raise DataModelException('No image with specified id and repository')
@ -1231,13 +1254,13 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
command, uncompressed_size, parent=None): command, uncompressed_size, parent=None):
with config.app_config['DB_TRANSACTION_FACTORY'](db): with config.app_config['DB_TRANSACTION_FACTORY'](db):
query = (Image query = (Image
.select(Image, ImageStorage) .select(Image, ImageStorage)
.join(Repository) .join(Repository)
.switch(Image) .join(Namespace, on=(Repository.namespace_user == Namespace.id))
.join(ImageStorage) .switch(Image)
.where(Repository.name == repository_name, .join(ImageStorage)
Repository.namespace == namespace_name, .where(Repository.name == repository_name, Namespace.username == namespace_name,
Image.docker_image_id == docker_image_id)) Image.docker_image_id == docker_image_id))
try: try:
fetched = query.get() fetched = query.get()
@ -1248,7 +1271,7 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
fetched.storage.checksum = None fetched.storage.checksum = None
fetched.storage.created = dateutil.parser.parse(created_date_str).replace(tzinfo=None) fetched.storage.created = dateutil.parser.parse(created_date_str).replace(tzinfo=None)
fetched.storage.comment = comment fetched.storage.comment = comment
fetched.storage.command = command fetched.storage.command = command
fetched.storage.uncompressed_size = uncompressed_size fetched.storage.uncompressed_size = uncompressed_size
if parent: if parent:
@ -1261,14 +1284,14 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
def _get_repository_images_base(namespace_name, repository_name, query_modifier): def _get_repository_images_base(namespace_name, repository_name, query_modifier):
query = (ImageStoragePlacement query = (ImageStoragePlacement
.select(ImageStoragePlacement, Image, ImageStorage, ImageStorageLocation) .select(ImageStoragePlacement, Image, ImageStorage, ImageStorageLocation)
.join(ImageStorageLocation) .join(ImageStorageLocation)
.switch(ImageStoragePlacement) .switch(ImageStoragePlacement)
.join(ImageStorage, JOIN_LEFT_OUTER) .join(ImageStorage, JOIN_LEFT_OUTER)
.join(Image) .join(Image)
.join(Repository) .join(Repository)
.where(Repository.name == repository_name, .join(Namespace, on=(Repository.namespace_user == Namespace.id))
Repository.namespace == namespace_name)) .where(Repository.name == repository_name, Namespace.username == namespace_name))
query = query_modifier(query) query = query_modifier(query)
@ -1299,24 +1322,26 @@ def get_repository_images(namespace_name, repository_name):
def list_repository_tags(namespace_name, repository_name): def list_repository_tags(namespace_name, repository_name):
select = RepositoryTag.select(RepositoryTag, Image) return (RepositoryTag
with_repo = select.join(Repository) .select(RepositoryTag, Image)
with_image = with_repo.switch(RepositoryTag).join(Image) .join(Repository)
return with_image.where(Repository.name == repository_name, .join(Namespace, on=(Repository.namespace_user == Namespace.id))
Repository.namespace == namespace_name) .switch(RepositoryTag)
.join(Image)
.where(Repository.name == repository_name, Namespace.username == namespace_name))
def garbage_collect_repository(namespace_name, repository_name): def garbage_collect_repository(namespace_name, repository_name):
with config.app_config['DB_TRANSACTION_FACTORY'](db): with config.app_config['DB_TRANSACTION_FACTORY'](db):
# Get a list of all images used by tags in the repository # Get a list of all images used by tags in the repository
tag_query = (RepositoryTag tag_query = (RepositoryTag
.select(RepositoryTag, Image, ImageStorage) .select(RepositoryTag, Image, ImageStorage)
.join(Image) .join(Image)
.join(ImageStorage, JOIN_LEFT_OUTER) .join(ImageStorage, JOIN_LEFT_OUTER)
.switch(RepositoryTag) .switch(RepositoryTag)
.join(Repository) .join(Repository)
.where(Repository.name == repository_name, .join(Namespace, on=(Repository.namespace_user == Namespace.id))
Repository.namespace == namespace_name)) .where(Repository.name == repository_name, Namespace.username == namespace_name))
referenced_anscestors = set() referenced_anscestors = set()
for tag in tag_query: for tag in tag_query:
@ -1344,11 +1369,11 @@ def garbage_collect_repository(namespace_name, repository_name):
if uuids_to_check_for_gc: if uuids_to_check_for_gc:
storage_to_remove = (ImageStorage storage_to_remove = (ImageStorage
.select() .select()
.join(Image, JOIN_LEFT_OUTER) .join(Image, JOIN_LEFT_OUTER)
.group_by(ImageStorage) .group_by(ImageStorage)
.where(ImageStorage.uuid << list(uuids_to_check_for_gc)) .where(ImageStorage.uuid << list(uuids_to_check_for_gc))
.having(fn.Count(Image.id) == 0)) .having(fn.Count(Image.id) == 0))
for storage in storage_to_remove: for storage in storage_to_remove:
logger.debug('Garbage collecting image storage: %s', storage.uuid) logger.debug('Garbage collecting image storage: %s', storage.uuid)
@ -1367,9 +1392,9 @@ def garbage_collect_repository(namespace_name, repository_name):
def get_tag_image(namespace_name, repository_name, tag_name): def get_tag_image(namespace_name, repository_name, tag_name):
def limit_to_tag(query): def limit_to_tag(query):
return (query return (query
.switch(Image) .switch(Image)
.join(RepositoryTag) .join(RepositoryTag)
.where(RepositoryTag.name == tag_name)) .where(RepositoryTag.name == tag_name))
images = _get_repository_images_base(namespace_name, repository_name, limit_to_tag) images = _get_repository_images_base(namespace_name, repository_name, limit_to_tag)
if not images: if not images:
@ -1407,22 +1432,17 @@ def get_parent_images(namespace_name, repository_name, image_obj):
def create_or_update_tag(namespace_name, repository_name, tag_name, def create_or_update_tag(namespace_name, repository_name, tag_name,
tag_docker_image_id): tag_docker_image_id):
try: try:
repo = Repository.get(Repository.name == repository_name, repo = _get_repository(namespace_name, repository_name)
Repository.namespace == namespace_name)
except Repository.DoesNotExist: except Repository.DoesNotExist:
raise DataModelException('Invalid repository %s/%s' % raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name))
(namespace_name, repository_name))
try: try:
image = Image.get(Image.docker_image_id == tag_docker_image_id, image = Image.get(Image.docker_image_id == tag_docker_image_id, Image.repository == repo)
Image.repository == repo)
except Image.DoesNotExist: except Image.DoesNotExist:
raise DataModelException('Invalid image with id: %s' % raise DataModelException('Invalid image with id: %s' % tag_docker_image_id)
tag_docker_image_id)
try: try:
tag = RepositoryTag.get(RepositoryTag.repository == repo, tag = RepositoryTag.get(RepositoryTag.repository == repo, RepositoryTag.name == tag_name)
RepositoryTag.name == tag_name)
tag.image = image tag.image = image
tag.save() tag.save()
except RepositoryTag.DoesNotExist: except RepositoryTag.DoesNotExist:
@ -1432,41 +1452,46 @@ def create_or_update_tag(namespace_name, repository_name, tag_name,
def delete_tag(namespace_name, repository_name, tag_name): def delete_tag(namespace_name, repository_name, tag_name):
joined = RepositoryTag.select().join(Repository) try:
found = list(joined.where(Repository.name == repository_name, found = (RepositoryTag
Repository.namespace == namespace_name, .select()
RepositoryTag.name == tag_name)) .join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.name == repository_name, Namespace.username == namespace_name,
RepositoryTag.name == tag_name)
.get())
if not found: except RepositoryTag.DoesNotExist:
msg = ('Invalid repository tag \'%s\' on repository \'%s/%s\'' % msg = ('Invalid repository tag \'%s\' on repository \'%s/%s\'' %
(tag_name, namespace_name, repository_name)) (tag_name, namespace_name, repository_name))
raise DataModelException(msg) raise DataModelException(msg)
found[0].delete_instance() found.delete_instance()
def delete_all_repository_tags(namespace_name, repository_name): def delete_all_repository_tags(namespace_name, repository_name):
try: try:
repo = Repository.get(Repository.name == repository_name, repo = _get_repository(namespace_name, repository_name)
Repository.namespace == namespace_name)
except Repository.DoesNotExist: except Repository.DoesNotExist:
raise DataModelException('Invalid repository \'%s/%s\'' % raise DataModelException('Invalid repository \'%s/%s\'' %
(namespace_name, repository_name)) (namespace_name, repository_name))
RepositoryTag.delete().where(RepositoryTag.repository == repo.id).execute() RepositoryTag.delete().where(RepositoryTag.repository == repo.id).execute()
def __entity_permission_repo_query(entity_id, entity_table, def __entity_permission_repo_query(entity_id, entity_table, entity_id_property, namespace_name,
entity_id_property, namespace_name,
repository_name): repository_name):
""" This method works for both users and teams. """ """ This method works for both users and teams. """
selected = RepositoryPermission.select(entity_table, Repository, Role,
RepositoryPermission) return (RepositoryPermission
with_user = selected.join(entity_table) .select(entity_table, Repository, Namespace, Role, RepositoryPermission)
with_role = with_user.switch(RepositoryPermission).join(Role) .join(entity_table)
with_repo = with_role.switch(RepositoryPermission).join(Repository) .switch(RepositoryPermission)
return with_repo.where(Repository.name == repository_name, .join(Role)
Repository.namespace == namespace_name, .switch(RepositoryPermission)
entity_id_property == entity_id) .join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.name == repository_name, Namespace.username == namespace_name,
entity_id_property == entity_id))
def get_user_reponame_permission(username, namespace_name, repository_name): def get_user_reponame_permission(username, namespace_name, repository_name):
@ -1514,8 +1539,7 @@ def delete_team_permission(team_name, namespace_name, repository_name):
def __set_entity_repo_permission(entity, permission_entity_property, def __set_entity_repo_permission(entity, permission_entity_property,
namespace_name, repository_name, role_name): namespace_name, repository_name, role_name):
repo = Repository.get(Repository.name == repository_name, repo = _get_repository(namespace_name, repository_name)
Repository.namespace == namespace_name)
new_role = Role.get(Role.name == role_name) new_role = Role.get(Role.name == role_name)
# Fetch any existing permission for this entity on the repo # Fetch any existing permission for this entity on the repo
@ -1566,15 +1590,18 @@ def purge_repository(namespace_name, repository_name):
garbage_collect_repository(namespace_name, repository_name) garbage_collect_repository(namespace_name, repository_name)
# Delete the rest of the repository metadata # Delete the rest of the repository metadata
fetched = Repository.get(Repository.name == repository_name, fetched = _get_repository(namespace_name, repository_name)
Repository.namespace == namespace_name)
fetched.delete_instance(recursive=True) fetched.delete_instance(recursive=True)
def get_private_repo_count(username): def get_private_repo_count(username):
joined = Repository.select().join(Visibility) return (Repository
return joined.where(Repository.namespace == username, .select()
Visibility.name == 'private').count() .join(Visibility)
.switch(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == username, Visibility.name == 'private')
.count())
def create_access_token(repository, role): def create_access_token(repository, role):
@ -1587,22 +1614,23 @@ def create_access_token(repository, role):
def create_delegate_token(namespace_name, repository_name, friendly_name, def create_delegate_token(namespace_name, repository_name, friendly_name,
role='read'): role='read'):
read_only = Role.get(name=role) read_only = Role.get(name=role)
repo = Repository.get(Repository.name == repository_name, repo = _get_repository(namespace_name, repository_name)
Repository.namespace == namespace_name)
new_token = AccessToken.create(repository=repo, role=read_only, new_token = AccessToken.create(repository=repo, role=read_only,
friendly_name=friendly_name, temporary=False) friendly_name=friendly_name, temporary=False)
return new_token return new_token
def get_repository_delegate_tokens(namespace_name, repository_name): def get_repository_delegate_tokens(namespace_name, repository_name):
return (AccessToken.select(AccessToken, Role) return (AccessToken
.join(Repository) .select(AccessToken, Role)
.switch(AccessToken) .join(Repository)
.join(Role) .join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(AccessToken) .switch(AccessToken)
.join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) .join(Role)
.where(Repository.name == repository_name, Repository.namespace == namespace_name, .switch(AccessToken)
AccessToken.temporary == False, RepositoryBuildTrigger.uuid >> None)) .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER)
.where(Repository.name == repository_name, Namespace.username == namespace_name,
AccessToken.temporary == False, RepositoryBuildTrigger.uuid >> None))
def get_repo_delegate_token(namespace_name, repository_name, code): def get_repo_delegate_token(namespace_name, repository_name, code):
@ -1636,14 +1664,17 @@ def delete_delegate_token(namespace_name, repository_name, code):
def load_token_data(code): def load_token_data(code):
""" Load the permissions for any token by code. """ """ Load the permissions for any token by code. """
selected = AccessToken.select(AccessToken, Repository, Role) try:
with_role = selected.join(Role) return (AccessToken
with_repo = with_role.switch(AccessToken).join(Repository) .select(AccessToken, Repository, Namespace, Role)
fetched = list(with_repo.where(AccessToken.code == code)) .join(Role)
.switch(AccessToken)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(AccessToken.code == code)
.get())
if fetched: except AccessToken.DoesNotExist:
return fetched[0]
else:
raise InvalidTokenException('Invalid delegate token code: %s' % code) raise InvalidTokenException('Invalid delegate token code: %s' % code)
@ -1660,15 +1691,15 @@ def get_repository_build(namespace_name, repository_name, build_uuid):
def list_repository_builds(namespace_name, repository_name, limit, def list_repository_builds(namespace_name, repository_name, limit,
include_inactive=True): include_inactive=True):
query = (RepositoryBuild query = (RepositoryBuild
.select(RepositoryBuild, RepositoryBuildTrigger, BuildTriggerService) .select(RepositoryBuild, RepositoryBuildTrigger, BuildTriggerService)
.join(Repository) .join(Repository)
.switch(RepositoryBuild) .join(Namespace, on=(Repository.namespace_user == Namespace.id))
.join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) .switch(RepositoryBuild)
.join(BuildTriggerService, JOIN_LEFT_OUTER) .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER)
.where(Repository.name == repository_name, .join(BuildTriggerService, JOIN_LEFT_OUTER)
Repository.namespace == namespace_name) .where(Repository.name == repository_name, Namespace.username == namespace_name)
.order_by(RepositoryBuild.started.desc()) .order_by(RepositoryBuild.started.desc())
.limit(limit)) .limit(limit))
if not include_inactive: if not include_inactive:
query = query.where(RepositoryBuild.phase != 'error', query = query.where(RepositoryBuild.phase != 'error',
@ -1732,16 +1763,17 @@ def create_repo_notification(repo, event_name, method_name, config):
def get_repo_notification(namespace_name, repository_name, uuid): def get_repo_notification(namespace_name, repository_name, uuid):
joined = RepositoryNotification.select().join(Repository) try:
found = list(joined.where(Repository.namespace == namespace_name, return (RepositoryNotification
Repository.name == repository_name, .select(RepositoryNotification, Repository, Namespace)
RepositoryNotification.uuid == uuid)) .join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
if not found: .where(Namespace.username == namespace_name, Repository.name == repository_name,
RepositoryNotification.uuid == uuid)
.get())
except RepositoryNotification.DoesNotExist:
raise InvalidNotificationException('No repository notification found with id: %s' % uuid) raise InvalidNotificationException('No repository notification found with id: %s' % uuid)
return found[0]
def delete_repo_notification(namespace_name, repository_name, uuid): def delete_repo_notification(namespace_name, repository_name, uuid):
found = get_repo_notification(namespace_name, repository_name, uuid) found = get_repo_notification(namespace_name, repository_name, uuid)
@ -1750,15 +1782,19 @@ def delete_repo_notification(namespace_name, repository_name, uuid):
def list_repo_notifications(namespace_name, repository_name, event_name=None): def list_repo_notifications(namespace_name, repository_name, event_name=None):
joined = RepositoryNotification.select().join(Repository) query = (RepositoryNotification
where = joined.where(Repository.namespace == namespace_name, .select(RepositoryNotification, Repository, Namespace)
Repository.name == repository_name) .join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace_name, Repository.name == repository_name))
if event_name: if event_name:
event = ExternalNotificationEvent.get(ExternalNotificationEvent.name == event_name) query = (query
where = where.where(RepositoryNotification.event == event) .switch(RepositoryNotification)
.join(ExternalNotificationEvent)
.where(ExternalNotificationEvent.name == event_name))
return where return query
def list_logs(start_time, end_time, performer=None, repository=None, namespace=None): def list_logs(start_time, end_time, performer=None, repository=None, namespace=None):
@ -1802,16 +1838,17 @@ def create_build_trigger(repo, service_name, auth_token, user, pull_robot=None):
def get_build_trigger(namespace_name, repository_name, trigger_uuid): def get_build_trigger(namespace_name, repository_name, trigger_uuid):
try: try:
return (RepositoryBuildTrigger return (RepositoryBuildTrigger
.select(RepositoryBuildTrigger, BuildTriggerService, Repository) .select(RepositoryBuildTrigger, BuildTriggerService, Repository, Namespace)
.join(BuildTriggerService) .join(BuildTriggerService)
.switch(RepositoryBuildTrigger) .switch(RepositoryBuildTrigger)
.join(Repository) .join(Repository)
.switch(RepositoryBuildTrigger) .join(Namespace, on=(Repository.namespace_user == Namespace.id))
.join(User) .switch(RepositoryBuildTrigger)
.where(RepositoryBuildTrigger.uuid == trigger_uuid, .join(User)
Repository.namespace == namespace_name, .where(RepositoryBuildTrigger.uuid == trigger_uuid,
Repository.name == repository_name) Namespace.username == namespace_name,
.get()) Repository.name == repository_name)
.get())
except RepositoryBuildTrigger.DoesNotExist: except RepositoryBuildTrigger.DoesNotExist:
msg = 'No build trigger with uuid: %s' % trigger_uuid msg = 'No build trigger with uuid: %s' % trigger_uuid
raise InvalidBuildTriggerException(msg) raise InvalidBuildTriggerException(msg)
@ -1819,12 +1856,12 @@ def get_build_trigger(namespace_name, repository_name, trigger_uuid):
def list_build_triggers(namespace_name, repository_name): def list_build_triggers(namespace_name, repository_name):
return (RepositoryBuildTrigger return (RepositoryBuildTrigger
.select(RepositoryBuildTrigger, BuildTriggerService, Repository) .select(RepositoryBuildTrigger, BuildTriggerService, Repository)
.join(BuildTriggerService) .join(BuildTriggerService)
.switch(RepositoryBuildTrigger) .switch(RepositoryBuildTrigger)
.join(Repository) .join(Repository)
.where(Repository.namespace == namespace_name, .join(Namespace, on=(Repository.namespace_user == Namespace.id))
Repository.name == repository_name)) .where(Namespace.username == namespace_name, Repository.name == repository_name))
def list_trigger_builds(namespace_name, repository_name, trigger_uuid, def list_trigger_builds(namespace_name, repository_name, trigger_uuid,
@ -1913,6 +1950,7 @@ def delete_notifications_by_kind(target, kind_name):
Notification.delete().where(Notification.target == target, Notification.delete().where(Notification.target == target,
Notification.kind == kind_ref).execute() Notification.kind == kind_ref).execute()
def delete_matching_notifications(target, kind_name, **kwargs): def delete_matching_notifications(target, kind_name, **kwargs):
kind_ref = NotificationKind.get(name=kind_name) kind_ref = NotificationKind.get(name=kind_name)
@ -1943,6 +1981,7 @@ def delete_matching_notifications(target, kind_name, **kwargs):
def get_active_users(): def get_active_users():
return User.select().where(User.organization == False, User.robot == False) return User.select().where(User.organization == False, User.robot == False)
def get_active_user_count(): def get_active_user_count():
return get_active_users().count() return get_active_users().count()
@ -1956,11 +1995,13 @@ def detach_external_login(user, service_name):
FederatedLogin.delete().where(FederatedLogin.user == user, FederatedLogin.delete().where(FederatedLogin.user == user,
FederatedLogin.service == service).execute() FederatedLogin.service == service).execute()
def delete_user(user): def delete_user(user):
user.delete_instance(recursive=True, delete_nullable=True) user.delete_instance(recursive=True, delete_nullable=True)
# TODO: also delete any repository data associated # TODO: also delete any repository data associated
def check_health(): def check_health():
# We will connect to the db, check that it contains some log entry kinds # We will connect to the db, check that it contains some log entry kinds
try: try:
@ -1969,24 +2010,23 @@ def check_health():
except: except:
return False return False
def get_email_authorized_for_repo(namespace, repository, email):
found = list(RepositoryAuthorizedEmail.select()
.join(Repository)
.where(Repository.namespace == namespace,
Repository.name == repository,
RepositoryAuthorizedEmail.email == email)
.switch(RepositoryAuthorizedEmail)
.limit(1))
if not found or len(found) < 1:
return None
return found[0] def get_email_authorized_for_repo(namespace, repository, email):
try:
return (RepositoryAuthorizedEmail
.select(RepositoryAuthorizedEmail, Repository, Namespace)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace, Repository.name == repository,
RepositoryAuthorizedEmail.email == email)
.get())
except RepositoryAuthorizedEmail.DoesNotExist:
return None
def create_email_authorization_for_repo(namespace_name, repository_name, email): def create_email_authorization_for_repo(namespace_name, repository_name, email):
try: try:
repo = Repository.get(Repository.name == repository_name, repo = _get_repository(namespace_name, repository_name)
Repository.namespace == namespace_name)
except Repository.DoesNotExist: except Repository.DoesNotExist:
raise DataModelException('Invalid repository %s/%s' % raise DataModelException('Invalid repository %s/%s' %
(namespace_name, repository_name)) (namespace_name, repository_name))
@ -1996,7 +2036,11 @@ def create_email_authorization_for_repo(namespace_name, repository_name, email):
def confirm_email_authorization_for_repo(code): def confirm_email_authorization_for_repo(code):
try: try:
found = RepositoryAuthorizedEmail.get(RepositoryAuthorizedEmail.code == code) found = (RepositoryAuthorizedEmail
.select(RepositoryAuthorizedEmail, Repository, Namespace)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.get(RepositoryAuthorizedEmail.code == code))
except RepositoryAuthorizedEmail.DoesNotExist: except RepositoryAuthorizedEmail.DoesNotExist:
raise DataModelException('Invalid confirmation code.') raise DataModelException('Invalid confirmation code.')

2
endpoints/api/build.py
View file

@ -169,7 +169,7 @@ class RepositoryBuildList(RepositoryParamResource):
# was used. # was used.
associated_repository = model.get_repository_for_resource(dockerfile_id) associated_repository = model.get_repository_for_resource(dockerfile_id)
if associated_repository: if associated_repository:
if not ModifyRepositoryPermission(associated_repository.namespace, if not ModifyRepositoryPermission(associated_repository.namespace_user.username,
associated_repository.name): associated_repository.name):
raise Unauthorized() raise Unauthorized()

2
endpoints/api/repoemail.py
View file

@ -20,7 +20,7 @@ def record_view(record):
return { return {
'email': record.email, 'email': record.email,
'repository': record.repository.name, 'repository': record.repository.name,
'namespace': record.repository.namespace, 'namespace': record.repository.namespace_user.username,
'confirmed': record.confirmed 'confirmed': record.confirmed
} }

8
endpoints/api/repository.py
View file

@ -80,8 +80,7 @@ class RepositoryList(ApiResource):
visibility = req['visibility'] visibility = req['visibility']
repo = model.create_repository(namespace_name, repository_name, owner, repo = model.create_repository(namespace_name, repository_name, owner, visibility)
visibility)
repo.description = req['description'] repo.description = req['description']
repo.save() repo.save()
@ -110,7 +109,7 @@ class RepositoryList(ApiResource):
"""Fetch the list of repositories under a variety of situations.""" """Fetch the list of repositories under a variety of situations."""
def repo_view(repo_obj): def repo_view(repo_obj):
return { return {
'namespace': repo_obj.namespace, 'namespace': repo_obj.namespace_user.username,
'name': repo_obj.name, 'name': repo_obj.name,
'description': repo_obj.description, 'description': repo_obj.description,
'is_public': repo_obj.visibility.name == 'public', 'is_public': repo_obj.visibility.name == 'public',
@ -134,7 +133,8 @@ class RepositoryList(ApiResource):
response['repositories'] = [repo_view(repo) for repo in repo_query response['repositories'] = [repo_view(repo) for repo in repo_query
if (repo.visibility.name == 'public' or if (repo.visibility.name == 'public' or
ReadRepositoryPermission(repo.namespace, repo.name).can())] ReadRepositoryPermission(repo.namespace_user.username,
repo.name).can())]
return response return response

4
endpoints/api/search.py
View file

@ -111,7 +111,7 @@ class FindRepositories(ApiResource):
def repo_view(repo): def repo_view(repo):
return { return {
'namespace': repo.namespace, 'namespace': repo.namespace_user.username,
'name': repo.name, 'name': repo.name,
'description': repo.description 'description': repo.description
} }
@ -125,5 +125,5 @@ class FindRepositories(ApiResource):
return { return {
'repositories': [repo_view(repo) for repo in matching 'repositories': [repo_view(repo) for repo in matching
if (repo.visibility.name == 'public' or if (repo.visibility.name == 'public' or
ReadRepositoryPermission(repo.namespace, repo.name).can())] ReadRepositoryPermission(repo.namespace_user.username, repo.name).can())]
} }

2
endpoints/api/trigger.py
View file

@ -205,7 +205,7 @@ class BuildTriggerActivate(RepositoryParamResource):
'write') 'write')
try: try:
repository_path = '%s/%s' % (trigger.repository.namespace, repository_path = '%s/%s' % (trigger.repository.namespace_user.username,
trigger.repository.name) trigger.repository.name)
path = url_for('webhooks.build_trigger_webhook', path = url_for('webhooks.build_trigger_webhook',
repository=repository_path, trigger_uuid=trigger.uuid) repository=repository_path, trigger_uuid=trigger.uuid)

13
endpoints/common.py
View file

@ -205,7 +205,7 @@ def check_repository_usage(user_or_org, plan_found):
def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
trigger=None, pull_robot_name=None): trigger=None, pull_robot_name=None):
host = urlparse.urlparse(request.url).netloc host = urlparse.urlparse(request.url).netloc
repo_path = '%s/%s/%s' % (host, repository.namespace, repository.name) repo_path = '%s/%s/%s' % (host, repository.namespace_user.username, repository.name)
token = model.create_access_token(repository, 'write') token = model.create_access_token(repository, 'write')
logger.debug('Creating build %s with repo %s tags %s and dockerfile_id %s', logger.debug('Creating build %s with repo %s tags %s and dockerfile_id %s',
@ -221,9 +221,9 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
dockerfile_id, build_name, dockerfile_id, build_name,
trigger, pull_robot_name=pull_robot_name) trigger, pull_robot_name=pull_robot_name)
dockerfile_build_queue.put([repository.namespace, repository.name], json.dumps({ dockerfile_build_queue.put([repository.namespace_user.username, repository.name], json.dumps({
'build_uuid': build_request.uuid, 'build_uuid': build_request.uuid,
'namespace': repository.namespace, 'namespace': repository.namespace_user.username,
'repository': repository.name, 'repository': repository.name,
'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None 'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None
}), retries_remaining=1) }), retries_remaining=1)
@ -231,7 +231,7 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
# Add the build to the repo's log. # Add the build to the repo's log.
metadata = { metadata = {
'repo': repository.name, 'repo': repository.name,
'namespace': repository.namespace, 'namespace': repository.namespace_user.username,
'fileid': dockerfile_id, 'fileid': dockerfile_id,
'manual': manual, 'manual': manual,
} }
@ -241,9 +241,8 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
metadata['config'] = json.loads(trigger.config) metadata['config'] = json.loads(trigger.config)
metadata['service'] = trigger.service.name metadata['service'] = trigger.service.name
model.log_action('build_dockerfile', repository.namespace, model.log_action('build_dockerfile', repository.namespace_user.username, ip=request.remote_addr,
ip=request.remote_addr, metadata=metadata, metadata=metadata, repository=repository)
repository=repository)
# Add notifications for the build queue. # Add notifications for the build queue.
profile.debug('Adding notifications for repository') profile.debug('Adding notifications for repository')

4
endpoints/index.py
View file

@ -420,7 +420,7 @@ def put_repository_auth(namespace, repository):
def get_search(): def get_search():
def result_view(repo): def result_view(repo):
return { return {
"name": repo.namespace + '/' + repo.name, "name": repo.namespace_user.username + '/' + repo.name,
"description": repo.description "description": repo.description
} }
@ -438,7 +438,7 @@ def get_search():
results = [result_view(repo) for repo in matching results = [result_view(repo) for repo in matching
if (repo.visibility.name == 'public' or if (repo.visibility.name == 'public' or
ReadRepositoryPermission(repo.namespace, repo.name).can())] ReadRepositoryPermission(repo.namespace_user.username, repo.name).can())]
data = { data = {
"query": query, "query": query,

4
endpoints/notificationevent.py
View file

@ -1,8 +1,4 @@
import logging import logging
import io
import os.path
import tarfile
import base64
from notificationhelper import build_event_data from notificationhelper import build_event_data

11
endpoints/notificationhelper.py
View file

@ -4,7 +4,7 @@ from data import model
import json import json
def build_event_data(repo, extra_data={}, subpage=None): def build_event_data(repo, extra_data={}, subpage=None):
repo_string = '%s/%s' % (repo.namespace, repo.name) repo_string = '%s/%s' % (repo.namespace_user.username, repo.name)
homepage = '%s://%s/repository/%s' % (app.config['PREFERRED_URL_SCHEME'], homepage = '%s://%s/repository/%s' % (app.config['PREFERRED_URL_SCHEME'],
app.config['SERVER_HOSTNAME'], app.config['SERVER_HOSTNAME'],
repo_string) repo_string)
@ -17,7 +17,7 @@ def build_event_data(repo, extra_data={}, subpage=None):
event_data = { event_data = {
'repository': repo_string, 'repository': repo_string,
'namespace': repo.namespace, 'namespace': repo.namespace_user.username,
'name': repo.name, 'name': repo.name,
'docker_url': '%s/%s' % (app.config['SERVER_HOSTNAME'], repo_string), 'docker_url': '%s/%s' % (app.config['SERVER_HOSTNAME'], repo_string),
'homepage': homepage, 'homepage': homepage,
@ -30,7 +30,7 @@ def build_event_data(repo, extra_data={}, subpage=None):
def build_notification_data(notification, event_data): def build_notification_data(notification, event_data):
return { return {
'notification_uuid': notification.uuid, 'notification_uuid': notification.uuid,
'repository_namespace': notification.repository.namespace, 'repository_namespace': notification.repository.namespace_user.username,
'repository_name': notification.repository.name, 'repository_name': notification.repository.name,
'event_data': event_data 'event_data': event_data
} }
@ -39,8 +39,9 @@ def build_notification_data(notification, event_data):
def spawn_notification(repo, event_name, extra_data={}, subpage=None, pathargs=[]): def spawn_notification(repo, event_name, extra_data={}, subpage=None, pathargs=[]):
event_data = build_event_data(repo, extra_data=extra_data, subpage=subpage) event_data = build_event_data(repo, extra_data=extra_data, subpage=subpage)
notifications = model.list_repo_notifications(repo.namespace, repo.name, event_name=event_name) notifications = model.list_repo_notifications(repo.namespace_user.username, repo.name,
event_name=event_name)
for notification in notifications: for notification in notifications:
notification_data = build_notification_data(notification, event_data) notification_data = build_notification_data(notification, event_data)
path = [repo.namespace, repo.name, event_name] + pathargs path = [repo.namespace_user.username, repo.name, event_name] + pathargs
notification_queue.put(path, json.dumps(notification_data)) notification_queue.put(path, json.dumps(notification_data))

16
endpoints/notificationmethod.py
View file

@ -88,7 +88,7 @@ class QuayNotificationMethod(NotificationMethod):
return (True, 'Unknown organization %s' % target_info['name'], None) return (True, 'Unknown organization %s' % target_info['name'], None)
# Only repositories under the organization can cause notifications to that org. # Only repositories under the organization can cause notifications to that org.
if target_info['name'] != repository.namespace: if target_info['name'] != repository.namespace_user.username:
return (False, 'Organization name must match repository namespace') return (False, 'Organization name must match repository namespace')
return (True, None, [target]) return (True, None, [target])
@ -96,7 +96,7 @@ class QuayNotificationMethod(NotificationMethod):
# Lookup the team. # Lookup the team.
team = None team = None
try: try:
team = model.get_organization_team(repository.namespace, target_info['name']) team = model.get_organization_team(repository.namespace_user.username, target_info['name'])
except model.InvalidTeamException: except model.InvalidTeamException:
# Probably deleted. # Probably deleted.
return (True, 'Unknown team %s' % target_info['name'], None) return (True, 'Unknown team %s' % target_info['name'], None)
@ -133,7 +133,8 @@ class EmailMethod(NotificationMethod):
if not email: if not email:
raise CannotValidateNotificationMethodException('Missing e-mail address') raise CannotValidateNotificationMethodException('Missing e-mail address')
record = model.get_email_authorized_for_repo(repository.namespace, repository.name, email) record = model.get_email_authorized_for_repo(repository.namespace_user.username,
repository.name, email)
if not record or not record.confirmed: if not record or not record.confirmed:
raise CannotValidateNotificationMethodException('The specified e-mail address ' raise CannotValidateNotificationMethodException('The specified e-mail address '
'is not authorized to receive ' 'is not authorized to receive '
@ -210,7 +211,7 @@ class FlowdockMethod(NotificationMethod):
if not token: if not token:
return return
owner = model.get_user(notification.repository.namespace) owner = model.get_user(notification.repository.namespace_user.username)
if not owner: if not owner:
# Something went wrong. # Something went wrong.
return return
@ -223,7 +224,8 @@ class FlowdockMethod(NotificationMethod):
'subject': event_handler.get_summary(notification_data['event_data'], notification_data), 'subject': event_handler.get_summary(notification_data['event_data'], notification_data),
'content': event_handler.get_message(notification_data['event_data'], notification_data), 'content': event_handler.get_message(notification_data['event_data'], notification_data),
'from_name': owner.username, 'from_name': owner.username,
'project': notification.repository.namespace + ' ' + notification.repository.name, 'project': (notification.repository.namespace_user.username + ' ' +
notification.repository.name),
'tags': ['#' + event_handler.event_name()], 'tags': ['#' + event_handler.event_name()],
'link': notification_data['event_data']['homepage'] 'link': notification_data['event_data']['homepage']
} }
@ -265,7 +267,7 @@ class HipchatMethod(NotificationMethod):
if not token or not room_id: if not token or not room_id:
return return
owner = model.get_user(notification.repository.namespace) owner = model.get_user(notification.repository.namespace_user.username)
if not owner: if not owner:
# Something went wrong. # Something went wrong.
return return
@ -332,7 +334,7 @@ class SlackMethod(NotificationMethod):
if not token or not subdomain: if not token or not subdomain:
return return
owner = model.get_user(notification.repository.namespace) owner = model.get_user(notification.repository.namespace_user.username)
if not owner: if not owner:
# Something went wrong. # Something went wrong.
return return

4
endpoints/web.py
View file

@ -237,8 +237,8 @@ def confirm_repo_email():
Your E-mail address has been authorized to receive notifications for repository Your E-mail address has been authorized to receive notifications for repository
<a href="%s://%s/repository/%s/%s">%s/%s</a>. <a href="%s://%s/repository/%s/%s">%s/%s</a>.
""" % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'], """ % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'],
record.repository.namespace, record.repository.name, record.repository.namespace_user.username, record.repository.name,
record.repository.namespace, record.repository.name) record.repository.namespace_user.username, record.repository.name)
return render_page_template('message.html', message=message) return render_page_template('message.html', message=message)

45
initdb.py
View file

@ -51,7 +51,7 @@ def __gen_checksum(image_id):
def __gen_image_id(repo, image_num): def __gen_image_id(repo, image_num):
str_to_hash = "%s/%s/%s" % (repo.namespace, repo.name, image_num) str_to_hash = "%s/%s/%s" % (repo.namespace_user.username, repo.name, image_num)
h = hashlib.md5(str_to_hash) h = hashlib.md5(str_to_hash)
return h.hexdigest() + h.hexdigest() return h.hexdigest() + h.hexdigest()
@ -79,11 +79,10 @@ def __create_subtree(repo, structure, creator_username, parent):
creation_time = REFERENCE_DATE + timedelta(days=image_num) creation_time = REFERENCE_DATE + timedelta(days=image_num)
command_list = SAMPLE_CMDS[image_num % len(SAMPLE_CMDS)] command_list = SAMPLE_CMDS[image_num % len(SAMPLE_CMDS)]
command = json.dumps(command_list) if command_list else None command = json.dumps(command_list) if command_list else None
new_image = model.set_image_metadata(docker_image_id, repo.namespace, new_image = model.set_image_metadata(docker_image_id, repo.namespace_user.username, repo.name,
repo.name, str(creation_time), str(creation_time), 'no comment', command, 0, parent)
'no comment', command, 0, parent)
model.set_image_size(docker_image_id, repo.namespace, repo.name, model.set_image_size(docker_image_id, repo.namespace_user.username, repo.name,
random.randrange(1, 1024 * 1024 * 1024)) random.randrange(1, 1024 * 1024 * 1024))
# Populate the diff file # Populate the diff file
@ -100,7 +99,7 @@ def __create_subtree(repo, structure, creator_username, parent):
last_node_tags = [last_node_tags] last_node_tags = [last_node_tags]
for tag_name in last_node_tags: for tag_name in last_node_tags:
model.create_or_update_tag(repo.namespace, repo.name, tag_name, model.create_or_update_tag(repo.namespace_user.username, repo.name, tag_name,
new_image.docker_image_id) new_image.docker_image_id)
for subtree in subtrees: for subtree in subtrees:
@ -326,7 +325,8 @@ def populate_database():
outside_org.verified = True outside_org.verified = True
outside_org.save() outside_org.save()
model.create_notification('test_notification', new_user_1, metadata={'some': 'value', 'arr': [1,2,3], 'obj': {'a': 1, 'b': 2}}) model.create_notification('test_notification', new_user_1,
metadata={'some':'value', 'arr':[1, 2, 3], 'obj':{'a':1, 'b':2}})
from_date = datetime.utcnow() from_date = datetime.utcnow()
to_date = from_date + timedelta(hours=1) to_date = from_date + timedelta(hours=1)
@ -390,18 +390,20 @@ def populate_database():
}) })
trigger.save() trigger.save()
repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name) repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace_user.username, building.name)
job_config = { job_config = {
'repository': repo, 'repository': repo,
'docker_tags': ['latest'], 'docker_tags': ['latest'],
'build_subdir': '', 'build_subdir': '',
} }
record = model.create_email_authorization_for_repo(new_user_1.username, 'simple', 'jschorr@devtable.com') record = model.create_email_authorization_for_repo(new_user_1.username, 'simple',
'jschorr@devtable.com')
record.confirmed = True record.confirmed = True
record.save() record.save()
model.create_email_authorization_for_repo(new_user_1.username, 'simple', 'jschorr+other@devtable.com') model.create_email_authorization_for_repo(new_user_1.username, 'simple',
'jschorr+other@devtable.com')
build2 = model.create_repository_build(building, token, job_config, build2 = model.create_repository_build(building, token, job_config,
'68daeebd-a5b9-457f-80a0-4363b882f8ea', '68daeebd-a5b9-457f-80a0-4363b882f8ea',
@ -428,12 +430,12 @@ def populate_database():
model.create_robot('coolrobot', org) model.create_robot('coolrobot', org)
oauth.create_application(org, 'Some Test App', 'http://localhost:8000', 'http://localhost:8000/o2c.html', oauth.create_application(org, 'Some Test App', 'http://localhost:8000',
client_id='deadbeef') 'http://localhost:8000/o2c.html', client_id='deadbeef')
oauth.create_application(org, 'Some Other Test App', 'http://quay.io', 'http://localhost:8000/o2c.html', oauth.create_application(org, 'Some Other Test App', 'http://quay.io',
client_id='deadpork', 'http://localhost:8000/o2c.html', client_id='deadpork',
description = 'This is another test application') description='This is another test application')
model.oauth.create_access_token_for_testing(new_user_1, 'deadbeef', 'repo:admin') model.oauth.create_access_token_for_testing(new_user_1, 'deadbeef', 'repo:admin')
@ -455,8 +457,8 @@ def populate_database():
reader_team = model.create_team('readers', org, 'member', reader_team = model.create_team('readers', org, 'member',
'Readers of orgrepo.') 'Readers of orgrepo.')
model.set_team_repo_permission(reader_team.name, org_repo.namespace, model.set_team_repo_permission(reader_team.name, org_repo.namespace_user.username, org_repo.name,
org_repo.name, 'read') 'read')
model.add_user_to_team(new_user_2, reader_team) model.add_user_to_team(new_user_2, reader_team)
model.add_user_to_team(reader, reader_team) model.add_user_to_team(reader, reader_team)
@ -478,12 +480,9 @@ def populate_database():
(2, [], 'latest17'), (2, [], 'latest17'),
(2, [], 'latest18'),]) (2, [], 'latest18'),])
model.add_prototype_permission(org, 'read', activating_user=new_user_1, model.add_prototype_permission(org, 'read', activating_user=new_user_1, delegate_user=new_user_2)
delegate_user=new_user_2) model.add_prototype_permission(org, 'read', activating_user=new_user_1, delegate_team=reader_team)
model.add_prototype_permission(org, 'read', activating_user=new_user_1, model.add_prototype_permission(org, 'write', activating_user=new_user_2, delegate_user=new_user_1)
delegate_team=reader_team)
model.add_prototype_permission(org, 'write', activating_user=new_user_2,
delegate_user=new_user_1)
today = datetime.today() today = datetime.today()
week_ago = today - timedelta(6) week_ago = today - timedelta(6)

BIN
test/data/test.db
View file

Binary file not shown.

36
tools/auditancestry.py
View file

@ -1,7 +1,7 @@
import logging import logging
import json import json
from data.database import Image, ImageStorage, Repository, configure from data.database import Image, ImageStorage, Repository, User, configure
from data import model from data import model
from app import app, storage as store from app import app, storage as store
@ -16,17 +16,18 @@ logging.getLogger('boto').setLevel(logging.CRITICAL)
query = (Image query = (Image
.select(Image, ImageStorage, Repository) .select(Image, ImageStorage, Repository, User)
.join(ImageStorage) .join(ImageStorage)
.switch(Image) .switch(Image)
.join(Repository) .join(Repository)
.where(ImageStorage.uploading == False)) .join(User)
.where(ImageStorage.uploading == False))
bad_count = 0 bad_count = 0
good_count = 0 good_count = 0
def resolve_or_create(repo, docker_image_id, new_ancestry): def resolve_or_create(repo, docker_image_id, new_ancestry):
existing = model.get_repo_image(repo.namespace, repo.name, docker_image_id) existing = model.get_repo_image(repo.namespace_user.username, repo.name, docker_image_id)
if existing: if existing:
logger.debug('Found existing image: %s, %s', existing.id, docker_image_id) logger.debug('Found existing image: %s, %s', existing.id, docker_image_id)
return existing return existing
@ -45,7 +46,7 @@ def resolve_or_create(repo, docker_image_id, new_ancestry):
return created return created
except ImageStorage.DoesNotExist: except ImageStorage.DoesNotExist:
msg = 'No image available anywhere for storage: %s in namespace: %s' msg = 'No image available anywhere for storage: %s in namespace: %s'
logger.error(msg, docker_image_id, repo.namespace) logger.error(msg, docker_image_id, repo.namespace_user.username)
raise RuntimeError() raise RuntimeError()
@ -62,20 +63,19 @@ def all_ancestors_exist(ancestors):
cant_fix = [] cant_fix = []
for img in query: for img in query:
try: try:
with_locations = model.get_repo_image(img.repository.namespace, img.repository.name, with_locations = model.get_repo_image(img.repository.namespace_user.username,
img.docker_image_id) img.repository.name, img.docker_image_id)
ancestry_storage = store.image_ancestry_path(img.storage.uuid) ancestry_storage = store.image_ancestry_path(img.storage.uuid)
if store.exists(with_locations.storage.locations, ancestry_storage): if store.exists(with_locations.storage.locations, ancestry_storage):
full_ancestry = json.loads(store.get_content(with_locations.storage.locations, ancestry_storage))[1:] full_ancestry = json.loads(store.get_content(with_locations.storage.locations,
ancestry_storage))[1:]
full_ancestry.reverse() full_ancestry.reverse()
ancestor_dbids = [int(anc_id) ancestor_dbids = [int(anc_id) for anc_id in img.ancestors.split('/')[1:-1]]
for anc_id in img.ancestors.split('/')[1:-1]]
if len(full_ancestry) != len(ancestor_dbids) or not all_ancestors_exist(ancestor_dbids): if len(full_ancestry) != len(ancestor_dbids) or not all_ancestors_exist(ancestor_dbids):
logger.error('Image has incomplete ancestry: %s, %s, %s, %s' % logger.error('Image has incomplete ancestry: %s, %s, %s, %s', img.id, img.docker_image_id,
(img.id, img.docker_image_id, full_ancestry, full_ancestry, ancestor_dbids)
ancestor_dbids))
fixed_ancestry = '/' fixed_ancestry = '/'
for ancestor in full_ancestry: for ancestor in full_ancestry:
@ -99,5 +99,5 @@ for img in query:
len(cant_fix)) len(cant_fix))
for cant in cant_fix: for cant in cant_fix:
logger.error('Unable to fix %s in repo %s/%s', cant.id, logger.error('Unable to fix %s in repo %s/%s', cant.id, cant.repository.namespace_user.username,
cant.repository.namespace, cant.repository.name) cant.repository.name)

2
tools/relationships.py
View file

@ -9,7 +9,7 @@ with open('outfile.dot', 'w') as outfile:
outfile.write('digraph relationships {\n') outfile.write('digraph relationships {\n')
for repo in Repository.select(): for repo in Repository.select():
ns = fix_ident(repo.namespace) ns = fix_ident(repo.namespace_user.username)
outfile.write('%s_%s -> %s\n' % (ns, fix_ident(repo.name), ns)) outfile.write('%s_%s -> %s\n' % (ns, fix_ident(repo.name), ns))
teams_in_orgs = set() teams_in_orgs = set()