vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #2447 from jzelinskie/cnr-step2

Browse source 
CNR Step 2
This commit is contained in:
Jimmy Zelinskie 2017-03-22 18:45:51 -04:00 • committed by GitHub
commit 3ccf3c5f33
40 changed files with 790 additions and 136 deletions
Download patch file Download diff file Expand all files Collapse all files

6
data/database.py
View file

@ -341,7 +341,6 @@ class QuayUserField(ForeignKeyField):
super(QuayUserField, self).__init__(*args, **kwargs)
# @TODO: Generates client-side enum
class EnumField(ForeignKeyField):
""" Create a cached python Enum from an EnumTable """
def __init__(self, rel_model, enum_key_field='name', *args, **kwargs):
@ -549,12 +548,17 @@ class Visibility(BaseModel):
name = CharField(index=True, unique=True)
class RepositoryKind(BaseModel):
name = CharField(index=True, unique=True)
class Repository(BaseModel):
namespace_user = QuayUserField(null=True)
name = FullIndexedCharField(match_function=db_match_func)
visibility = ForeignKeyField(Visibility)
description = FullIndexedTextField(match_function=db_match_func, null=True)
badge_token = CharField(default=uuid_generator)
kind = EnumField(RepositoryKind)
class Meta:
database = db

16
data/interfaces/v1.py
View file

@ -10,9 +10,15 @@ from util.morecollections import AttrDict
class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description',
'is_public'])):
'is_public', 'kind'])):
"""
Repository represents a namespaced collection of tags.
:type id: int
:type name: string
:type namespace_name: string
:type description: string
:type is_public: bool
:type kind: string
"""
@ -383,8 +389,9 @@ class PreOCIModel(DockerRegistryV1DataInterface):
return bool(model.oauth.validate_access_token(token))
def get_sorted_matching_repositories(self, search_term, filter_username=None, offset=0, limit=25):
repos = model.repository.get_filtered_matching_repositories(search_term, filter_username,
offset, limit)
repos = model.repository.get_filtered_matching_repositories(search_term,
filter_username=filter_username,
offset=offset, limit=limit)
return [_repository_for_repo(repo) for repo in repos]
@ -395,7 +402,8 @@ def _repository_for_repo(repo):
name=repo.name,
namespace_name=repo.namespace_user.username,
description=repo.description,
is_public=model.repository.is_repository_public(repo)
is_public=model.repository.is_repository_public(repo),
kind=model.repository.get_repo_kind_name(repo),
)

25
data/interfaces/v2.py
View file

@ -13,9 +13,15 @@ _MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v1+prettyjws"
class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description',
'is_public'])):
'is_public', 'kind'])):
"""
Repository represents a namespaced collection of tags.
:type id: int
:type name: string
:type namespace_name: string
:type description: string
:type is_public: bool
:type kind: string
"""
class ManifestJSON(namedtuple('ManifestJSON', ['digest', 'json', 'media_type'])):
@ -70,14 +76,6 @@ class DockerRegistryV2DataInterface(object):
"""
pass
@abstractmethod
def repository_is_public(self, namespace_name, repo_name):
"""
Returns true if the repository with the given name under the given namespace has public
visibility.
"""
pass
@abstractmethod
def get_repository(self, namespace_name, repo_name):
"""
@ -271,9 +269,6 @@ class PreOCIModel(DockerRegistryV2DataInterface):
def create_repository(self, namespace_name, repo_name, creating_user=None):
return model.repository.create_repository(namespace_name, repo_name, creating_user)
def repository_is_public(self, namespace_name, repo_name):
return model.repository.repository_is_public(namespace_name, repo_name)
def get_repository(self, namespace_name, repo_name):
repo = model.repository.get_repository(namespace_name, repo_name)
if repo is None:
@ -392,7 +387,8 @@ class PreOCIModel(DockerRegistryV2DataInterface):
return [_tag_view(tag) for tag in tags_query]
def get_visible_repositories(self, username, limit, offset):
query = model.repository.get_visible_repositories(username, include_public=(username is None))
query = model.repository.get_visible_repositories(username, repo_kind='image',
include_public=(username is None))
query = query.limit(limit).offset(offset)
return [_repository_for_repo(repo) for repo in query]
@ -538,7 +534,8 @@ def _repository_for_repo(repo):
name=repo.name,
namespace_name=repo.namespace_user.username,
description=repo.description,
is_public=model.repository.is_repository_public(repo)
is_public=model.repository.is_repository_public(repo),
kind=model.repository.get_repo_kind_name(repo),
)

37
data/interfaces/verbs.py
View file

@ -9,6 +9,19 @@ from data import model
from image.docker.v1 import DockerV1Metadata
class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description',
'is_public', 'kind'])):
"""
Repository represents a namespaced collection of tags.
:type id: int
:type name: string
:type namespace_name: string
:type description: string
:type is_public: bool
:type kind: string
"""
class DerivedImage(namedtuple('DerivedImage', ['ref', 'blob', 'internal_source_image_db_id'])):
"""
DerivedImage represents a user-facing alias for an image which was derived from another image.
@ -43,9 +56,10 @@ class VerbsDataInterface(object):
verbs.
"""
@abstractmethod
def repository_is_public(self, namespace_name, repo_name):
def get_repository(self, namespace_name, repo_name):
"""
Returns a boolean for whether the repository with the given name and namespace is public.
Returns a repository tuple for the repository with the given name under the given namespace.
Returns None if no such repository was found.
"""
pass
@ -144,8 +158,12 @@ class PreOCIModel(VerbsDataInterface):
before it was changed to support the OCI specification.
"""
def repository_is_public(self, namespace_name, repo_name):
return model.repository.repository_is_public(namespace_name, repo_name)
def get_repository(self, namespace_name, repo_name):
repo = model.repository.get_repository(namespace_name, repo_name)
if repo is None:
return None
return _repository_for_repo(repo)
def get_manifest_layers_with_blobs(self, repo_image):
repo_image_record = model.image.get_image_by_id(repo_image.repository.namespace_name,
@ -320,3 +338,14 @@ def _blob(blob_record):
uploading=blob_record.uploading,
locations=locations,
)
def _repository_for_repo(repo):
""" Returns a Repository object representing the Pre-OCI data model repo instance given. """
return Repository(
id=repo.id,
name=repo.name,
namespace_name=repo.namespace_user.username,
description=repo.description,
is_public=model.repository.is_repository_public(repo),
kind=model.repository.get_repo_kind_name(repo),
)

16
data/migrations/migration.sh
View file

@ -10,9 +10,9 @@ up_mysql() {
# Run a SQL database on port 3306 inside of Docker.
docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mysql
# Sleep for 10s to get MySQL get started.
echo 'Sleeping for 20...'
sleep 20
# Sleep for 25s to get MySQL get started.
echo 'Sleeping for 25...'
sleep 25
# Add the database to mysql.
docker run --rm --link mysql:mysql mysql sh -c 'echo "create database genschema" | mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -ppassword'
@ -27,9 +27,9 @@ up_mariadb() {
# Run a SQL database on port 3306 inside of Docker.
docker run --name mariadb -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mariadb
# Sleep for 20s to get MySQL get started.
echo 'Sleeping for 20...'
sleep 20
# Sleep for 25s to get MySQL get started.
echo 'Sleeping for 25...'
sleep 25
# Add the database to mysql.
docker run --rm --link mariadb:mariadb mariadb sh -c 'echo "create database genschema" | mysql -h"$MARIADB_PORT_3306_TCP_ADDR" -P"$MARIADB_PORT_3306_TCP_PORT" -uroot -ppassword'
@ -45,8 +45,8 @@ up_percona() {
docker run --name percona -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d percona
# Sleep for 20s
echo 'Sleeping for 20...'
sleep 20
echo 'Sleeping for 25...'
sleep 25
# Add the daabase to mysql.
docker run --rm --link percona:percona percona sh -c 'echo "create database genschema" | mysql -h $PERCONA_PORT_3306_TCP_ADDR -uroot -ppassword'

6
data/migrations/versions/7a525c68eb13_add_oci_app_models.py
View file

@ -300,9 +300,9 @@ def upgrade(tables):
op.bulk_insert(
tables.tagkind,
[
{'name': 'tag', 'id': 1},
{'name': 'release', 'id': 2},
{'name': 'channel', 'id': 3},
{'id': 1, 'name': 'tag'},
{'id': 2, 'name': 'release'},
{'id': 3, 'name': 'channel'},
]
)

44
data/migrations/versions/b4df55dea4b3_add_repository_kind.py Normal file
View file

@ -0,0 +1,44 @@
"""add repository kind
Revision ID: b4df55dea4b3
Revises: 7a525c68eb13
Create Date: 2017-03-19 12:59:41.484430
"""
# revision identifiers, used by Alembic.
revision = 'b4df55dea4b3'
down_revision = 'b8ae68ad3e52'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
def upgrade(tables):
op.create_table(
'repositorykind',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_repositorykind'))
)
op.create_index('repositorykind_name', 'repositorykind', ['name'], unique=True)
op.bulk_insert(
tables.repositorykind,
[
{'id': 1, 'name': 'image'},
{'id': 2, 'name': 'application'},
],
)
op.add_column(u'repository', sa.Column('kind_id', sa.Integer(), nullable=False, server_default='1'))
op.create_index('repository_kind_id', 'repository', ['kind_id'], unique=False)
op.create_foreign_key(op.f('fk_repository_kind_id_repositorykind'), 'repository', 'repositorykind', ['kind_id'], ['id'])
def downgrade(tables):
op.drop_constraint(op.f('fk_repository_kind_id_repositorykind'), 'repository', type_='foreignkey')
op.drop_index('repository_kind_id', table_name='repository')
op.drop_column(u'repository', 'kind_id')
op.drop_table('repositorykind')

24
data/model/_basequery.py
View file

@ -3,14 +3,23 @@ from cachetools import lru_cache
from data.model import DataModelException
from data.database import (Repository, User, Team, TeamMember, RepositoryPermission, TeamRole,
Namespace, Visibility, ImageStorage, Image, db_for_update)
Namespace, Visibility, ImageStorage, Image, RepositoryKind,
db_for_update)
def get_existing_repository(namespace_name, repository_name, for_update=False):
def get_existing_repository(namespace_name, repository_name, for_update=False, kind_filter=None):
query = (Repository
.select(Repository, Namespace)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace_name, Repository.name == repository_name))
.where(Namespace.username == namespace_name,
Repository.name == repository_name))
if kind_filter:
query = (query
.switch(Repository)
.join(RepositoryKind)
.where(RepositoryKind.name == kind_filter))
if for_update:
query = db_for_update(query)
@ -27,11 +36,14 @@ def _lookup_team_role(name):
return TeamRole.get(name=name)
def filter_to_repos_for_user(query, username=None, namespace=None, include_public=True,
start_id=None):
def filter_to_repos_for_user(query, username=None, namespace=None, repo_kind='image',
include_public=True, start_id=None):
if not include_public and not username:
return Repository.select().where(Repository.id == '-1')
# Filter on the type of repository.
query = query.where(Repository.kind == Repository.kind.get_id(repo_kind))
# Add the start ID if necessary.
if start_id is not None:
query = query.where(Repository.id >= start_id)
@ -121,5 +133,3 @@ def calculate_image_aggregate_size(ancestors_str, image_size, parent_image):
return None
return ancestor_size + image_size

52
data/model/repository.py
View file

@ -18,6 +18,11 @@ from util.itertoolrecipes import take
logger = logging.getLogger(__name__)
def get_repo_kind_name(repo):
return Repository.kind.get_name(repo.kind_id)
def get_repository_count():
return Repository.select().count()
@ -26,10 +31,11 @@ def get_public_repo_visibility():
return _basequery.get_public_repo_visibility()
def create_repository(namespace, name, creating_user, visibility='private'):
def create_repository(namespace, name, creating_user, visibility='private', repo_kind='image'):
private = Visibility.get(name=visibility)
namespace_user = User.get(username=namespace)
repo = Repository.create(name=name, visibility=private, namespace_user=namespace_user)
repo = Repository.create(name=name, visibility=private, namespace_user=namespace_user,
kind=Repository.kind.get_id(repo_kind))
admin = Role.get(name='admin')
yesterday = datetime.now() - timedelta(days=1)
@ -44,9 +50,10 @@ def create_repository(namespace, name, creating_user, visibility='private'):
return repo
def get_repository(namespace_name, repository_name):
def get_repository(namespace_name, repository_name, kind_filter=None):
try:
return _basequery.get_existing_repository(namespace_name, repository_name)
return _basequery.get_existing_repository(namespace_name, repository_name,
kind_filter=kind_filter)
except Repository.DoesNotExist:
return None
@ -103,7 +110,7 @@ def _get_gc_expiration_policies():
def get_random_gc_policy():
""" Return a single random policy from the database to use when garbage collecting.
"""
"""
return random.choice(_get_gc_expiration_policies())
@ -259,7 +266,7 @@ def unstar_repository(user, repository):
raise DataModelException('Star not found.')
def get_user_starred_repositories(user):
def get_user_starred_repositories(user, repo_kind='image'):
""" Retrieves all of the repositories a user has starred. """
query = (Repository
.select(Repository, User, Visibility, Repository.id.alias('rid'))
@ -268,7 +275,8 @@ def get_user_starred_repositories(user):
.join(User)
.switch(Repository)
.join(Visibility)
.where(Star.user == user))
.where(Star.user == user,
Repository.kind == Repository.kind.get_id(repo_kind)))
return query
@ -302,8 +310,8 @@ def get_when_last_modified(repository_ids):
return last_modified_map
def get_visible_repositories(username, namespace=None, include_public=False, start_id=None,
limit=None):
def get_visible_repositories(username, namespace=None, repo_kind='image', include_public=False,
start_id=None, limit=None):
""" Returns the repositories visible to the given user (if any).
"""
if not include_public and not username:
@ -313,7 +321,8 @@ def get_visible_repositories(username, namespace=None, include_public=False, sta
query = (Repository
.select(Repository.name, Repository.id.alias('rid'),
Repository.description, Namespace.username, Repository.visibility)
Repository.description, Namespace.username, Repository.visibility,
Repository.kind)
.switch(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id)))
@ -321,7 +330,7 @@ def get_visible_repositories(username, namespace=None, include_public=False, sta
# Note: We only need the permissions table if we will filter based on a user's permissions.
query = query.switch(Repository).distinct().join(RepositoryPermission, JOIN_LEFT_OUTER)
query = _basequery.filter_to_repos_for_user(query, username, namespace, include_public,
query = _basequery.filter_to_repos_for_user(query, username, namespace, repo_kind, include_public,
start_id=start_id)
if limit is not None:
@ -330,14 +339,15 @@ def get_visible_repositories(username, namespace=None, include_public=False, sta
return query
def get_filtered_matching_repositories(lookup_value, filter_username=None, offset=0, limit=25):
def get_filtered_matching_repositories(lookup_value, filter_username=None, repo_kind='image',
offset=0, limit=25):
""" Returns an iterator of all repositories matching the given lookup value, with optional
filtering to a specific user. If the user is unspecified, only public repositories will
be returned.
"""
# Build the unfiltered search query.
unfiltered_query = _get_sorted_matching_repositories(lookup_value,
unfiltered_query = _get_sorted_matching_repositories(lookup_value, repo_kind=repo_kind,
include_private=filter_username is not None)
# Add a filter to the iterator, if necessary.
@ -395,7 +405,7 @@ def _filter_repositories_visible_to_username(unfiltered_query, filter_username,
iteration_count = iteration_count + 1
def _get_sorted_matching_repositories(lookup_value, include_private=False):
def _get_sorted_matching_repositories(lookup_value, repo_kind='image', include_private=False):
""" Returns a query of repositories matching the given lookup string, with optional inclusion of
private repositories. Note that this method does *not* filter results based on visibility
to users.
@ -405,7 +415,8 @@ def _get_sorted_matching_repositories(lookup_value, include_private=False):
query = (Repository
.select(Repository, Namespace)
.join(Namespace, on=(Namespace.id == Repository.namespace_user))
.where(Repository.name.match(lookup_value) | Repository.description.match(lookup_value))
.where(Repository.name.match(lookup_value) | Repository.description.match(lookup_value),
Repository.kind == Repository.kind.get_id(repo_kind))
.group_by(Repository.id, Namespace.id))
if not include_private:
@ -438,7 +449,8 @@ def repository_is_public(namespace_name, repository_name):
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Repository)
.join(Visibility)
.where(Namespace.username == namespace_name, Repository.name == repository_name,
.where(Namespace.username == namespace_name,
Repository.name == repository_name,
Visibility.name == 'public')
.get())
return True
@ -461,7 +473,8 @@ def get_email_authorized_for_repo(namespace, repository, email):
.select(RepositoryAuthorizedEmail, Repository, Namespace)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace, Repository.name == repository,
.where(Namespace.username == namespace,
Repository.name == repository,
RepositoryAuthorizedEmail.email == email)
.get())
except RepositoryAuthorizedEmail.DoesNotExist:
@ -495,7 +508,7 @@ def confirm_email_authorization_for_repo(code):
return found
def list_popular_public_repos(action_count_threshold, time_span):
def list_popular_public_repos(action_count_threshold, time_span, repo_kind='image'):
cutoff = datetime.now() - time_span
return (Repository
.select(Namespace.username, Repository.name)
@ -503,7 +516,8 @@ def list_popular_public_repos(action_count_threshold, time_span):
.switch(Repository)
.join(RepositoryActionCount)
.where(RepositoryActionCount.date >= cutoff,
Repository.visibility == get_public_repo_visibility())
Repository.visibility == get_public_repo_visibility(),
Repository.kind == Repository.kind.get_id(repo_kind))
.group_by(RepositoryActionCount.repository, Repository.name, Namespace.username)
.having(fn.Sum(RepositoryActionCount.count) >= action_count_threshold)
.tuples())

14
data/model/test/test_repository.py Normal file
View file

@ -0,0 +1,14 @@
import pytest
from peewee import IntegrityError
from endpoints.test.fixtures import database_uri, init_db_path, sqlitedb_file
from data.model.repository import create_repository
def test_duplicate_repository_different_kinds(database_uri):
# Create an image repo.
create_repository('devtable', 'somenewrepo', None, repo_kind='image')
# Try to create an app repo with the same name, which should fail.
with pytest.raises(IntegrityError):
create_repository('devtable', 'somenewrepo', None, repo_kind='application')