vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'ldapper'

Browse source 
Conflicts:
	Dockerfile
	app.py
	data/database.py
	endpoints/index.py
	test/data/test.db
This commit is contained in:
Jake Moshenko 2014-05-22 12:13:41 -04:00
commit f6726bd0a4
25 changed files with 1157 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
Dockerfile.web
View file

@ -17,6 +17,9 @@ RUN apt-get install -y nodejs npm
RUN ln -s /usr/bin/nodejs /usr/bin/node RUN ln -s /usr/bin/nodejs /usr/bin/node
RUN npm install -g grunt-cli RUN npm install -g grunt-cli
# LDAP
RUN apt-get install -y libldap2-dev libsasl2-dev
ADD binary_dependencies binary_dependencies ADD binary_dependencies binary_dependencies
RUN gdebi --n binary_dependencies/*.deb RUN gdebi --n binary_dependencies/*.deb
@ -26,15 +29,19 @@ ADD requirements.txt requirements.txt
RUN virtualenv --distribute venv RUN virtualenv --distribute venv
RUN venv/bin/pip install -r requirements.txt RUN venv/bin/pip install -r requirements.txt
# Add the static assets and run grunt
ADD grunt grunt
ADD static static
RUN cd grunt && npm install
RUN cd grunt && grunt
# Add the backend assets
ADD auth auth ADD auth auth
ADD buildstatus buildstatus ADD buildstatus buildstatus
ADD conf conf
ADD data data ADD data data
ADD endpoints endpoints ADD endpoints endpoints
ADD features features ADD features features
ADD grunt grunt
ADD screenshots screenshots ADD screenshots screenshots
ADD static static
ADD storage storage ADD storage storage
ADD templates templates ADD templates templates
ADD util util ADD util util
@ -44,23 +51,34 @@ ADD app.py app.py
ADD application.py application.py ADD application.py application.py
ADD config.py config.py ADD config.py config.py
ADD initdb.py initdb.py ADD initdb.py initdb.py
ADD external_libraries.py external_libraries.py
ADD alembic.ini alembic.ini
# Add the config
ADD conf conf
RUN rm -rf /conf/stack
ADD conf/init/svlogd_config /svlogd_config ADD conf/init/svlogd_config /svlogd_config
ADD conf/init/preplogsdir.sh /etc/my_init.d/ ADD conf/init/preplogsdir.sh /etc/my_init.d/
ADD conf/init/runmigration.sh /etc/my_init.d/
ADD conf/init/gunicorn /etc/service/gunicorn ADD conf/init/gunicorn /etc/service/gunicorn
ADD conf/init/nginx /etc/service/nginx ADD conf/init/nginx /etc/service/nginx
ADD conf/init/diffsworker /etc/service/diffsworker ADD conf/init/diffsworker /etc/service/diffsworker
ADD conf/init/webhookworker /etc/service/webhookworker ADD conf/init/webhookworker /etc/service/webhookworker
RUN cd grunt && npm install # Download any external libs.
RUN cd grunt && grunt RUN mkdir static/fonts
RUN mkdir static/ldn
RUN venv/bin/python -m external_libraries
# Add the tests last because they're prone to accidental changes, then run them # Add the tests last because they're prone to accidental changes, then run them
ADD test test ADD test test
RUN TEST=true venv/bin/python -m unittest discover RUN TEST=true venv/bin/python -m unittest discover
RUN rm -rf /conf/stack RUN rm -rf /conf/stack
VOLUME ["/conf/stack", "/var/log"] VOLUME ["/conf/stack", "/var/log", "/datastorage"]
EXPOSE 443 80 EXPOSE 443 80

15
app.py
View file

@ -9,7 +9,10 @@ from flask.ext.mail import Mail
import features import features
from storage import Storage from storage import Storage
from data import model
from data import database
from data.userfiles import Userfiles from data.userfiles import Userfiles
from data.users import UserAuthentication
from util.analytics import Analytics from util.analytics import Analytics
from util.exceptionlog import Sentry from util.exceptionlog import Sentry
from util.queuemetrics import QueueMetrics from util.queuemetrics import QueueMetrics
@ -51,8 +54,14 @@ billing = Billing(app)
sentry = Sentry(app) sentry = Sentry(app)
build_logs = BuildLogs(app) build_logs = BuildLogs(app)
queue_metrics = QueueMetrics(app) queue_metrics = QueueMetrics(app)
authentication = UserAuthentication(app)
image_diff_queue = WorkQueue(app.config['DIFFS_QUEUE_NAME']) tf = app.config['DB_TRANSACTION_FACTORY']
dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], image_diff_queue = WorkQueue(app.config['DIFFS_QUEUE_NAME'], tf)
dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf,
reporter=queue_metrics.report) reporter=queue_metrics.report)
webhook_queue = WorkQueue(app.config['WEBHOOK_QUEUE_NAME']) webhook_queue = WorkQueue(app.config['WEBHOOK_QUEUE_NAME'], tf)
database.configure(app.config)
model.config.app_config = app.config
model.config.store = storage

6
auth/auth.py
View file

@ -11,7 +11,7 @@ import scopes
from data import model from data import model
from data.model import oauth from data.model import oauth
from app import app from app import app, authentication
from permissions import QuayDeferredPermissionUser from permissions import QuayDeferredPermissionUser
from auth_context import (set_authenticated_user, set_validated_token, from auth_context import (set_authenticated_user, set_validated_token,
set_authenticated_user_deferred, set_validated_oauth_token) set_authenticated_user_deferred, set_validated_oauth_token)
@ -70,7 +70,7 @@ def process_basic_auth(auth):
logger.debug('Invalid basic auth format.') logger.debug('Invalid basic auth format.')
return return
credentials = b64decode(normalized[1]).split(':', 1) credentials = [part.decode('utf-8') for part in b64decode(normalized[1]).split(':', 1)]
if len(credentials) != 2: if len(credentials) != 2:
logger.debug('Invalid basic auth credential format.') logger.debug('Invalid basic auth credential format.')
@ -108,7 +108,7 @@ def process_basic_auth(auth):
logger.debug('Invalid robot or password for robot: %s' % credentials[0]) logger.debug('Invalid robot or password for robot: %s' % credentials[0])
else: else:
authenticated = model.verify_user(credentials[0], credentials[1]) authenticated = authentication.verify_user(credentials[0], credentials[1])
if authenticated: if authenticated:
logger.debug('Successfully validated user: %s' % authenticated.username) logger.debug('Successfully validated user: %s' % authenticated.username)

5
conf/init/runmigration.sh Executable file
View file

@ -0,0 +1,5 @@
#! /bin/bash
set -e
# Run the database migration
PYTHONPATH=. venv/bin/alembic upgrade head

7
config.py
View file

@ -68,10 +68,17 @@ class DefaultConfig(object):
DB_TRANSACTION_FACTORY = create_transaction DB_TRANSACTION_FACTORY = create_transaction
# If true, CDN URLs will be used for our external dependencies, rather than the local
# copies.
USE_CDN = True
# Data storage # Data storage
STORAGE_TYPE = 'LocalStorage' STORAGE_TYPE = 'LocalStorage'
STORAGE_PATH = 'test/data/registry' STORAGE_PATH = 'test/data/registry'
# Authentication
AUTHENTICATION_TYPE = 'Database'
# Build logs # Build logs
BUILDLOGS_OPTIONS = ['logs.quay.io'] BUILDLOGS_OPTIONS = ['logs.quay.io']

12
data/database.py
View file

@ -8,19 +8,19 @@ from peewee import *
from sqlalchemy.engine.url import make_url from sqlalchemy.engine.url import make_url
from urlparse import urlparse from urlparse import urlparse
from app import app
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
SCHEME_DRIVERS = { SCHEME_DRIVERS = {
'mysql': MySQLDatabase, 'mysql': MySQLDatabase,
'mysql+pymysql': MySQLDatabase,
'sqlite': SqliteDatabase, 'sqlite': SqliteDatabase,
} }
db = Proxy()
def generate_db(config_object): def configure(config_object):
db_kwargs = dict(config_object['DB_CONNECTION_ARGS']) db_kwargs = dict(config_object['DB_CONNECTION_ARGS'])
parsed_url = make_url(config_object['DB_URI']) parsed_url = make_url(config_object['DB_URI'])
@ -33,10 +33,8 @@ def generate_db(config_object):
if parsed_url.password: if parsed_url.password:
db_kwargs['passwd'] = parsed_url.password db_kwargs['passwd'] = parsed_url.password
return SCHEME_DRIVERS[parsed_url.drivername](parsed_url.database, **db_kwargs) real_db = SCHEME_DRIVERS[parsed_url.drivername](parsed_url.database, **db_kwargs)
db.initialize(real_db)
db = generate_db(app.config)
def random_string_generator(length=16): def random_string_generator(length=16):

15
data/migrations/env.py
View file

@ -2,15 +2,17 @@ from __future__ import with_statement
from alembic import context from alembic import context
from sqlalchemy import engine_from_config, pool from sqlalchemy import engine_from_config, pool
from logging.config import fileConfig from logging.config import fileConfig
from urllib import unquote
from peewee import SqliteDatabase
from data.database import all_models from data.database import all_models, db
from app import app from app import app
from data.model.sqlalchemybridge import gen_sqlalchemy_metadata from data.model.sqlalchemybridge import gen_sqlalchemy_metadata
# this is the Alembic Config object, which provides # this is the Alembic Config object, which provides
# access to the values within the .ini file in use. # access to the values within the .ini file in use.
config = context.config config = context.config
config.set_main_option('sqlalchemy.url', app.config['DB_URI']) config.set_main_option('sqlalchemy.url', unquote(app.config['DB_URI']))
# Interpret the config file for Python logging. # Interpret the config file for Python logging.
# This line sets up loggers basically. # This line sets up loggers basically.
@ -39,8 +41,8 @@ def run_migrations_offline():
script output. script output.
""" """
url = app.config['DB_CONNECTION'] url = unquote(app.config['DB_URI'])
context.configure(url=url, target_metadata=target_metadata) context.configure(url=url, target_metadata=target_metadata, transactional_ddl=True)
with context.begin_transaction(): with context.begin_transaction():
context.run_migrations() context.run_migrations()
@ -52,6 +54,11 @@ def run_migrations_online():
and associate a connection with the context. and associate a connection with the context.
""" """
if isinstance(db.obj, SqliteDatabase):
print ('Skipping Sqlite migration!')
return
engine = engine_from_config( engine = engine_from_config(
config.get_section(config.config_ini_section), config.get_section(config.config_ini_section),
prefix='sqlalchemy.', prefix='sqlalchemy.',

607
data/migrations/versions/5a07499ce53f_set_up_initial_database.py Normal file
View file

@ -0,0 +1,607 @@
"""Set up initial database
Revision ID: 5a07499ce53f
Revises: None
Create Date: 2014-05-13 11:26:51.808426
"""
# revision identifiers, used by Alembic.
revision = '5a07499ce53f'
down_revision = None
from alembic import op
from data.model.sqlalchemybridge import gen_sqlalchemy_metadata
from data.database import all_models
import sqlalchemy as sa
def upgrade():
schema = gen_sqlalchemy_metadata(all_models)
### commands auto generated by Alembic - please adjust! ###
op.create_table('loginservice',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_index('loginservice_name', 'loginservice', ['name'], unique=True)
op.bulk_insert(schema.tables['loginservice'],
[
{'id':1, 'name':'github'},
{'id':2, 'name':'quayrobot'},
{'id':3, 'name':'ldap'},
])
op.create_table('imagestorage',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=255), nullable=False),
sa.Column('checksum', sa.String(length=255), nullable=True),
sa.Column('created', sa.DateTime(), nullable=True),
sa.Column('comment', sa.Text(), nullable=True),
sa.Column('command', sa.Text(), nullable=True),
sa.Column('image_size', sa.BigInteger(), nullable=True),
sa.Column('uploading', sa.Boolean(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('queueitem',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('queue_name', sa.String(length=1024), nullable=False),
sa.Column('body', sa.Text(), nullable=False),
sa.Column('available_after', sa.DateTime(), nullable=False),
sa.Column('available', sa.Boolean(), nullable=False),
sa.Column('processing_expires', sa.DateTime(), nullable=True),
sa.Column('retries_remaining', sa.Integer(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_index('queueitem_available', 'queueitem', ['available'], unique=False)
op.create_index('queueitem_available_after', 'queueitem', ['available_after'], unique=False)
op.create_index('queueitem_processing_expires', 'queueitem', ['processing_expires'], unique=False)
op.create_index('queueitem_queue_name', 'queueitem', ['queue_name'], unique=False)
op.create_table('role',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_index('role_name', 'role', ['name'], unique=False)
op.bulk_insert(schema.tables['role'],
[
{'id':1, 'name':'admin'},
{'id':2, 'name':'write'},
{'id':3, 'name':'read'},
])
op.create_table('logentrykind',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_index('logentrykind_name', 'logentrykind', ['name'], unique=False)
op.bulk_insert(schema.tables['logentrykind'],
[
{'id':1, 'name':'account_change_plan'},
{'id':2, 'name':'account_change_cc'},
{'id':3, 'name':'account_change_password'},
{'id':4, 'name':'account_convert'},
{'id':5, 'name':'create_robot'},
{'id':6, 'name':'delete_robot'},
{'id':7, 'name':'create_repo'},
{'id':8, 'name':'push_repo'},
{'id':9, 'name':'pull_repo'},
{'id':10, 'name':'delete_repo'},
{'id':11, 'name':'create_tag'},
{'id':12, 'name':'move_tag'},
{'id':13, 'name':'delete_tag'},
{'id':14, 'name':'add_repo_permission'},
{'id':15, 'name':'change_repo_permission'},
{'id':16, 'name':'delete_repo_permission'},
{'id':17, 'name':'change_repo_visibility'},
{'id':18, 'name':'add_repo_accesstoken'},
{'id':19, 'name':'delete_repo_accesstoken'},
{'id':20, 'name':'add_repo_webhook'},
{'id':21, 'name':'delete_repo_webhook'},
{'id':22, 'name':'set_repo_description'},
{'id':23, 'name':'build_dockerfile'},
{'id':24, 'name':'org_create_team'},
{'id':25, 'name':'org_delete_team'},
{'id':26, 'name':'org_add_team_member'},
{'id':27, 'name':'org_remove_team_member'},
{'id':28, 'name':'org_set_team_description'},
{'id':29, 'name':'org_set_team_role'},
{'id':30, 'name':'create_prototype_permission'},
{'id':31, 'name':'modify_prototype_permission'},
{'id':32, 'name':'delete_prototype_permission'},
{'id':33, 'name':'setup_repo_trigger'},
{'id':34, 'name':'delete_repo_trigger'},
{'id':35, 'name':'create_application'},
{'id':36, 'name':'update_application'},
{'id':37, 'name':'delete_application'},
{'id':38, 'name':'reset_application_client_secret'},
])
op.create_table('notificationkind',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_index('notificationkind_name', 'notificationkind', ['name'], unique=False)
op.bulk_insert(schema.tables['notificationkind'],
[
{'id':1, 'name':'password_required'},
{'id':2, 'name':'over_private_usage'},
])
op.create_table('teamrole',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_index('teamrole_name', 'teamrole', ['name'], unique=False)
op.bulk_insert(schema.tables['teamrole'],
[
{'id':1, 'name':'admin'},
{'id':2, 'name':'creator'},
{'id':3, 'name':'member'},
])
op.create_table('visibility',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_index('visibility_name', 'visibility', ['name'], unique=False)
op.bulk_insert(schema.tables['visibility'],
[
{'id':1, 'name':'public'},
{'id':2, 'name':'private'},
])
op.create_table('user',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('username', sa.String(length=255), nullable=False),
sa.Column('password_hash', sa.String(length=255), nullable=True),
sa.Column('email', sa.String(length=255), nullable=False),
sa.Column('verified', sa.Boolean(), nullable=False),
sa.Column('stripe_id', sa.String(length=255), nullable=True),
sa.Column('organization', sa.Boolean(), nullable=False),
sa.Column('robot', sa.Boolean(), nullable=False),
sa.Column('invoice_email', sa.Boolean(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_index('user_email', 'user', ['email'], unique=True)
op.create_index('user_organization', 'user', ['organization'], unique=False)
op.create_index('user_robot', 'user', ['robot'], unique=False)
op.create_index('user_stripe_id', 'user', ['stripe_id'], unique=False)
op.create_index('user_username', 'user', ['username'], unique=True)
op.create_table('buildtriggerservice',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_index('buildtriggerservice_name', 'buildtriggerservice', ['name'], unique=False)
op.bulk_insert(schema.tables['buildtriggerservice'],
[
{'id':1, 'name':'github'},
])
op.create_table('federatedlogin',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('user_id', sa.Integer(), nullable=False),
sa.Column('service_id', sa.Integer(), nullable=False),
sa.Column('service_ident', sa.String(length=255, collation='utf8_general_ci'), nullable=False),
sa.ForeignKeyConstraint(['service_id'], ['loginservice.id'], ),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('federatedlogin_service_id', 'federatedlogin', ['service_id'], unique=False)
op.create_index('federatedlogin_service_id_service_ident', 'federatedlogin', ['service_id', 'service_ident'], unique=True)
op.create_index('federatedlogin_service_id_user_id', 'federatedlogin', ['service_id', 'user_id'], unique=True)
op.create_index('federatedlogin_user_id', 'federatedlogin', ['user_id'], unique=False)
op.create_table('oauthapplication',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('client_id', sa.String(length=255), nullable=False),
sa.Column('client_secret', sa.String(length=255), nullable=False),
sa.Column('redirect_uri', sa.String(length=255), nullable=False),
sa.Column('application_uri', sa.String(length=255), nullable=False),
sa.Column('organization_id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.Column('description', sa.Text(), nullable=False),
sa.Column('gravatar_email', sa.String(length=255), nullable=True),
sa.ForeignKeyConstraint(['organization_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('oauthapplication_client_id', 'oauthapplication', ['client_id'], unique=False)
op.create_index('oauthapplication_organization_id', 'oauthapplication', ['organization_id'], unique=False)
op.create_table('notification',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=255), nullable=False),
sa.Column('kind_id', sa.Integer(), nullable=False),
sa.Column('target_id', sa.Integer(), nullable=False),
sa.Column('metadata_json', sa.Text(), nullable=False),
sa.Column('created', sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(['kind_id'], ['notificationkind.id'], ),
sa.ForeignKeyConstraint(['target_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('notification_created', 'notification', ['created'], unique=False)
op.create_index('notification_kind_id', 'notification', ['kind_id'], unique=False)
op.create_index('notification_target_id', 'notification', ['target_id'], unique=False)
op.create_index('notification_uuid', 'notification', ['uuid'], unique=False)
op.create_table('emailconfirmation',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('code', sa.String(length=255), nullable=False),
sa.Column('user_id', sa.Integer(), nullable=False),
sa.Column('pw_reset', sa.Boolean(), nullable=False),
sa.Column('new_email', sa.String(length=255), nullable=True),
sa.Column('email_confirm', sa.Boolean(), nullable=False),
sa.Column('created', sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('emailconfirmation_code', 'emailconfirmation', ['code'], unique=True)
op.create_index('emailconfirmation_user_id', 'emailconfirmation', ['user_id'], unique=False)
op.create_table('team',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.Column('organization_id', sa.Integer(), nullable=False),
sa.Column('role_id', sa.Integer(), nullable=False),
sa.Column('description', sa.Text(), nullable=False),
sa.ForeignKeyConstraint(['organization_id'], ['user.id'], ),
sa.ForeignKeyConstraint(['role_id'], ['teamrole.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('team_name', 'team', ['name'], unique=False)
op.create_index('team_name_organization_id', 'team', ['name', 'organization_id'], unique=True)
op.create_index('team_organization_id', 'team', ['organization_id'], unique=False)
op.create_index('team_role_id', 'team', ['role_id'], unique=False)
op.create_table('repository',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('namespace', sa.String(length=255), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.Column('visibility_id', sa.Integer(), nullable=False),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('badge_token', sa.String(length=255), nullable=False),
sa.ForeignKeyConstraint(['visibility_id'], ['visibility.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('repository_namespace_name', 'repository', ['namespace', 'name'], unique=True)
op.create_index('repository_visibility_id', 'repository', ['visibility_id'], unique=False)
op.create_table('accesstoken',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('friendly_name', sa.String(length=255), nullable=True),
sa.Column('code', sa.String(length=255), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('created', sa.DateTime(), nullable=False),
sa.Column('role_id', sa.Integer(), nullable=False),
sa.Column('temporary', sa.Boolean(), nullable=False),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], ),
sa.ForeignKeyConstraint(['role_id'], ['role.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('accesstoken_code', 'accesstoken', ['code'], unique=True)
op.create_index('accesstoken_repository_id', 'accesstoken', ['repository_id'], unique=False)
op.create_index('accesstoken_role_id', 'accesstoken', ['role_id'], unique=False)
op.create_table('repositorypermission',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('team_id', sa.Integer(), nullable=True),
sa.Column('user_id', sa.Integer(), nullable=True),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('role_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], ),
sa.ForeignKeyConstraint(['role_id'], ['role.id'], ),
sa.ForeignKeyConstraint(['team_id'], ['team.id'], ),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('repositorypermission_repository_id', 'repositorypermission', ['repository_id'], unique=False)
op.create_index('repositorypermission_role_id', 'repositorypermission', ['role_id'], unique=False)
op.create_index('repositorypermission_team_id', 'repositorypermission', ['team_id'], unique=False)
op.create_index('repositorypermission_team_id_repository_id', 'repositorypermission', ['team_id', 'repository_id'], unique=True)
op.create_index('repositorypermission_user_id', 'repositorypermission', ['user_id'], unique=False)
op.create_index('repositorypermission_user_id_repository_id', 'repositorypermission', ['user_id', 'repository_id'], unique=True)
op.create_table('oauthaccesstoken',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=255), nullable=False),
sa.Column('application_id', sa.Integer(), nullable=False),
sa.Column('authorized_user_id', sa.Integer(), nullable=False),
sa.Column('scope', sa.String(length=255), nullable=False),
sa.Column('access_token', sa.String(length=255), nullable=False),
sa.Column('token_type', sa.String(length=255), nullable=False),
sa.Column('expires_at', sa.DateTime(), nullable=False),
sa.Column('refresh_token', sa.String(length=255), nullable=True),
sa.Column('data', sa.Text(), nullable=False),
sa.ForeignKeyConstraint(['application_id'], ['oauthapplication.id'], ),
sa.ForeignKeyConstraint(['authorized_user_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('oauthaccesstoken_access_token', 'oauthaccesstoken', ['access_token'], unique=False)
op.create_index('oauthaccesstoken_application_id', 'oauthaccesstoken', ['application_id'], unique=False)
op.create_index('oauthaccesstoken_authorized_user_id', 'oauthaccesstoken', ['authorized_user_id'], unique=False)
op.create_index('oauthaccesstoken_refresh_token', 'oauthaccesstoken', ['refresh_token'], unique=False)
op.create_index('oauthaccesstoken_uuid', 'oauthaccesstoken', ['uuid'], unique=False)
op.create_table('teammember',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('user_id', sa.Integer(), nullable=False),
sa.Column('team_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['team_id'], ['team.id'], ),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('teammember_team_id', 'teammember', ['team_id'], unique=False)
op.create_index('teammember_user_id', 'teammember', ['user_id'], unique=False)
op.create_index('teammember_user_id_team_id', 'teammember', ['user_id', 'team_id'], unique=True)
op.create_table('webhook',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('public_id', sa.String(length=255), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('parameters', sa.Text(), nullable=False),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('webhook_public_id', 'webhook', ['public_id'], unique=True)
op.create_index('webhook_repository_id', 'webhook', ['repository_id'], unique=False)
op.create_table('oauthauthorizationcode',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('application_id', sa.Integer(), nullable=False),
sa.Column('code', sa.String(length=255), nullable=False),
sa.Column('scope', sa.String(length=255), nullable=False),
sa.Column('data', sa.Text(), nullable=False),
sa.ForeignKeyConstraint(['application_id'], ['oauthapplication.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('oauthauthorizationcode_application_id', 'oauthauthorizationcode', ['application_id'], unique=False)
op.create_index('oauthauthorizationcode_code', 'oauthauthorizationcode', ['code'], unique=False)
op.create_table('image',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('docker_image_id', sa.String(length=255), nullable=False),
sa.Column('checksum', sa.String(length=255), nullable=True),
sa.Column('created', sa.DateTime(), nullable=True),
sa.Column('comment', sa.Text(), nullable=True),
sa.Column('command', sa.Text(), nullable=True),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('image_size', sa.BigInteger(), nullable=True),
sa.Column('ancestors', sa.String(length=60535, collation='latin1_swedish_ci'), nullable=True),
sa.Column('storage_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], ),
sa.ForeignKeyConstraint(['storage_id'], ['imagestorage.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('image_ancestors', 'image', ['ancestors'], unique=False)
op.create_index('image_repository_id', 'image', ['repository_id'], unique=False)
op.create_index('image_repository_id_docker_image_id', 'image', ['repository_id', 'docker_image_id'], unique=False)
op.create_index('image_storage_id', 'image', ['storage_id'], unique=False)
op.create_table('permissionprototype',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('org_id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=255), nullable=False),
sa.Column('activating_user_id', sa.Integer(), nullable=True),
sa.Column('delegate_user_id', sa.Integer(), nullable=True),
sa.Column('delegate_team_id', sa.Integer(), nullable=True),
sa.Column('role_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['activating_user_id'], ['user.id'], ),
sa.ForeignKeyConstraint(['delegate_team_id'], ['team.id'], ),
sa.ForeignKeyConstraint(['delegate_user_id'], ['user.id'], ),
sa.ForeignKeyConstraint(['org_id'], ['user.id'], ),
sa.ForeignKeyConstraint(['role_id'], ['role.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('permissionprototype_activating_user_id', 'permissionprototype', ['activating_user_id'], unique=False)
op.create_index('permissionprototype_delegate_team_id', 'permissionprototype', ['delegate_team_id'], unique=False)
op.create_index('permissionprototype_delegate_user_id', 'permissionprototype', ['delegate_user_id'], unique=False)
op.create_index('permissionprototype_org_id', 'permissionprototype', ['org_id'], unique=False)
op.create_index('permissionprototype_org_id_activating_user_id', 'permissionprototype', ['org_id', 'activating_user_id'], unique=False)
op.create_index('permissionprototype_role_id', 'permissionprototype', ['role_id'], unique=False)
op.create_table('repositorytag',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.Column('image_id', sa.Integer(), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['image_id'], ['image.id'], ),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('repositorytag_image_id', 'repositorytag', ['image_id'], unique=False)
op.create_index('repositorytag_repository_id', 'repositorytag', ['repository_id'], unique=False)
op.create_index('repositorytag_repository_id_name', 'repositorytag', ['repository_id', 'name'], unique=True)
op.create_table('logentry',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('kind_id', sa.Integer(), nullable=False),
sa.Column('account_id', sa.Integer(), nullable=False),
sa.Column('performer_id', sa.Integer(), nullable=True),
sa.Column('repository_id', sa.Integer(), nullable=True),
sa.Column('access_token_id', sa.Integer(), nullable=True),
sa.Column('datetime', sa.DateTime(), nullable=False),
sa.Column('ip', sa.String(length=255), nullable=True),
sa.Column('metadata_json', sa.Text(), nullable=False),
sa.ForeignKeyConstraint(['access_token_id'], ['accesstoken.id'], ),
sa.ForeignKeyConstraint(['account_id'], ['user.id'], ),
sa.ForeignKeyConstraint(['kind_id'], ['logentrykind.id'], ),
sa.ForeignKeyConstraint(['performer_id'], ['user.id'], ),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('logentry_access_token_id', 'logentry', ['access_token_id'], unique=False)
op.create_index('logentry_account_id', 'logentry', ['account_id'], unique=False)
op.create_index('logentry_datetime', 'logentry', ['datetime'], unique=False)
op.create_index('logentry_kind_id', 'logentry', ['kind_id'], unique=False)
op.create_index('logentry_performer_id', 'logentry', ['performer_id'], unique=False)
op.create_index('logentry_repository_id', 'logentry', ['repository_id'], unique=False)
op.create_table('repositorybuildtrigger',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=255), nullable=False),
sa.Column('service_id', sa.Integer(), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('connected_user_id', sa.Integer(), nullable=False),
sa.Column('auth_token', sa.String(length=255), nullable=False),
sa.Column('config', sa.Text(), nullable=False),
sa.Column('write_token_id', sa.Integer(), nullable=True),
sa.Column('pull_robot_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['connected_user_id'], ['user.id'], ),
sa.ForeignKeyConstraint(['pull_robot_id'], ['user.id'], ),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], ),
sa.ForeignKeyConstraint(['service_id'], ['buildtriggerservice.id'], ),
sa.ForeignKeyConstraint(['write_token_id'], ['accesstoken.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('repositorybuildtrigger_connected_user_id', 'repositorybuildtrigger', ['connected_user_id'], unique=False)
op.create_index('repositorybuildtrigger_pull_robot_id', 'repositorybuildtrigger', ['pull_robot_id'], unique=False)
op.create_index('repositorybuildtrigger_repository_id', 'repositorybuildtrigger', ['repository_id'], unique=False)
op.create_index('repositorybuildtrigger_service_id', 'repositorybuildtrigger', ['service_id'], unique=False)
op.create_index('repositorybuildtrigger_write_token_id', 'repositorybuildtrigger', ['write_token_id'], unique=False)
op.create_table('repositorybuild',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=255), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('access_token_id', sa.Integer(), nullable=False),
sa.Column('resource_key', sa.String(length=255), nullable=False),
sa.Column('job_config', sa.Text(), nullable=False),
sa.Column('phase', sa.String(length=255), nullable=False),
sa.Column('started', sa.DateTime(), nullable=False),
sa.Column('display_name', sa.String(length=255), nullable=False),
sa.Column('trigger_id', sa.Integer(), nullable=True),
sa.Column('pull_robot_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['access_token_id'], ['accesstoken.id'], ),
sa.ForeignKeyConstraint(['pull_robot_id'], ['user.id'], ),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], ),
sa.ForeignKeyConstraint(['trigger_id'], ['repositorybuildtrigger.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('repositorybuild_access_token_id', 'repositorybuild', ['access_token_id'], unique=False)
op.create_index('repositorybuild_pull_robot_id', 'repositorybuild', ['pull_robot_id'], unique=False)
op.create_index('repositorybuild_repository_id', 'repositorybuild', ['repository_id'], unique=False)
op.create_index('repositorybuild_resource_key', 'repositorybuild', ['resource_key'], unique=False)
op.create_index('repositorybuild_trigger_id', 'repositorybuild', ['trigger_id'], unique=False)
op.create_index('repositorybuild_uuid', 'repositorybuild', ['uuid'], unique=False)
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_index('repositorybuild_uuid', table_name='repositorybuild')
op.drop_index('repositorybuild_trigger_id', table_name='repositorybuild')
op.drop_index('repositorybuild_resource_key', table_name='repositorybuild')
op.drop_index('repositorybuild_repository_id', table_name='repositorybuild')
op.drop_index('repositorybuild_pull_robot_id', table_name='repositorybuild')
op.drop_index('repositorybuild_access_token_id', table_name='repositorybuild')
op.drop_table('repositorybuild')
op.drop_index('repositorybuildtrigger_write_token_id', table_name='repositorybuildtrigger')
op.drop_index('repositorybuildtrigger_service_id', table_name='repositorybuildtrigger')
op.drop_index('repositorybuildtrigger_repository_id', table_name='repositorybuildtrigger')
op.drop_index('repositorybuildtrigger_pull_robot_id', table_name='repositorybuildtrigger')
op.drop_index('repositorybuildtrigger_connected_user_id', table_name='repositorybuildtrigger')
op.drop_table('repositorybuildtrigger')
op.drop_index('logentry_repository_id', table_name='logentry')
op.drop_index('logentry_performer_id', table_name='logentry')
op.drop_index('logentry_kind_id', table_name='logentry')
op.drop_index('logentry_datetime', table_name='logentry')
op.drop_index('logentry_account_id', table_name='logentry')
op.drop_index('logentry_access_token_id', table_name='logentry')
op.drop_table('logentry')
op.drop_index('repositorytag_repository_id_name', table_name='repositorytag')
op.drop_index('repositorytag_repository_id', table_name='repositorytag')
op.drop_index('repositorytag_image_id', table_name='repositorytag')
op.drop_table('repositorytag')
op.drop_index('permissionprototype_role_id', table_name='permissionprototype')
op.drop_index('permissionprototype_org_id_activating_user_id', table_name='permissionprototype')
op.drop_index('permissionprototype_org_id', table_name='permissionprototype')
op.drop_index('permissionprototype_delegate_user_id', table_name='permissionprototype')
op.drop_index('permissionprototype_delegate_team_id', table_name='permissionprototype')
op.drop_index('permissionprototype_activating_user_id', table_name='permissionprototype')
op.drop_table('permissionprototype')
op.drop_index('image_storage_id', table_name='image')
op.drop_index('image_repository_id_docker_image_id', table_name='image')
op.drop_index('image_repository_id', table_name='image')
op.drop_index('image_ancestors', table_name='image')
op.drop_table('image')
op.drop_index('oauthauthorizationcode_code', table_name='oauthauthorizationcode')
op.drop_index('oauthauthorizationcode_application_id', table_name='oauthauthorizationcode')
op.drop_table('oauthauthorizationcode')
op.drop_index('webhook_repository_id', table_name='webhook')
op.drop_index('webhook_public_id', table_name='webhook')
op.drop_table('webhook')
op.drop_index('teammember_user_id_team_id', table_name='teammember')
op.drop_index('teammember_user_id', table_name='teammember')
op.drop_index('teammember_team_id', table_name='teammember')
op.drop_table('teammember')
op.drop_index('oauthaccesstoken_uuid', table_name='oauthaccesstoken')
op.drop_index('oauthaccesstoken_refresh_token', table_name='oauthaccesstoken')
op.drop_index('oauthaccesstoken_authorized_user_id', table_name='oauthaccesstoken')
op.drop_index('oauthaccesstoken_application_id', table_name='oauthaccesstoken')
op.drop_index('oauthaccesstoken_access_token', table_name='oauthaccesstoken')
op.drop_table('oauthaccesstoken')
op.drop_index('repositorypermission_user_id_repository_id', table_name='repositorypermission')
op.drop_index('repositorypermission_user_id', table_name='repositorypermission')
op.drop_index('repositorypermission_team_id_repository_id', table_name='repositorypermission')
op.drop_index('repositorypermission_team_id', table_name='repositorypermission')
op.drop_index('repositorypermission_role_id', table_name='repositorypermission')
op.drop_index('repositorypermission_repository_id', table_name='repositorypermission')
op.drop_table('repositorypermission')
op.drop_index('accesstoken_role_id', table_name='accesstoken')
op.drop_index('accesstoken_repository_id', table_name='accesstoken')
op.drop_index('accesstoken_code', table_name='accesstoken')
op.drop_table('accesstoken')
op.drop_index('repository_visibility_id', table_name='repository')
op.drop_index('repository_namespace_name', table_name='repository')
op.drop_table('repository')
op.drop_index('team_role_id', table_name='team')
op.drop_index('team_organization_id', table_name='team')
op.drop_index('team_name_organization_id', table_name='team')
op.drop_index('team_name', table_name='team')
op.drop_table('team')
op.drop_index('emailconfirmation_user_id', table_name='emailconfirmation')
op.drop_index('emailconfirmation_code', table_name='emailconfirmation')
op.drop_table('emailconfirmation')
op.drop_index('notification_uuid', table_name='notification')
op.drop_index('notification_target_id', table_name='notification')
op.drop_index('notification_kind_id', table_name='notification')
op.drop_index('notification_created', table_name='notification')
op.drop_table('notification')
op.drop_index('oauthapplication_organization_id', table_name='oauthapplication')
op.drop_index('oauthapplication_client_id', table_name='oauthapplication')
op.drop_table('oauthapplication')
op.drop_index('federatedlogin_user_id', table_name='federatedlogin')
op.drop_index('federatedlogin_service_id_user_id', table_name='federatedlogin')
op.drop_index('federatedlogin_service_id_service_ident', table_name='federatedlogin')
op.drop_index('federatedlogin_service_id', table_name='federatedlogin')
op.drop_table('federatedlogin')
op.drop_index('buildtriggerservice_name', table_name='buildtriggerservice')
op.drop_table('buildtriggerservice')
op.drop_index('user_username', table_name='user')
op.drop_index('user_stripe_id', table_name='user')
op.drop_index('user_robot', table_name='user')
op.drop_index('user_organization', table_name='user')
op.drop_index('user_email', table_name='user')
op.drop_table('user')
op.drop_index('visibility_name', table_name='visibility')
op.drop_table('visibility')
op.drop_index('teamrole_name', table_name='teamrole')
op.drop_table('teamrole')
op.drop_index('notificationkind_name', table_name='notificationkind')
op.drop_table('notificationkind')
op.drop_index('logentrykind_name', table_name='logentrykind')
op.drop_table('logentrykind')
op.drop_index('role_name', table_name='role')
op.drop_table('role')
op.drop_index('queueitem_queue_name', table_name='queueitem')
op.drop_index('queueitem_processing_expires', table_name='queueitem')
op.drop_index('queueitem_available_after', table_name='queueitem')
op.drop_index('queueitem_available', table_name='queueitem')
op.drop_table('queueitem')
op.drop_table('imagestorage')
op.drop_index('loginservice_name', table_name='loginservice')
op.drop_table('loginservice')
### end Alembic commands ###

67
data/model/legacy.py
View file

@ -8,11 +8,17 @@ from data.database import *
from util.validation import * from util.validation import *
from util.names import format_robot_username from util.names import format_robot_username
from app import storage as store
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
transaction_factory = app.config['DB_TRANSACTION_FACTORY']
class Config(object):
def __init__(self):
self.app_config = None
self.store = None
config = Config()
class DataModelException(Exception): class DataModelException(Exception):
pass pass
@ -58,7 +64,7 @@ class InvalidBuildTriggerException(DataModelException):
pass pass
def create_user(username, password, email, is_organization=False): def create_user(username, password, email, add_change_pw_notification=True):
if not validate_email(email): if not validate_email(email):
raise InvalidEmailAddressException('Invalid email address: %s' % email) raise InvalidEmailAddressException('Invalid email address: %s' % email)
@ -97,7 +103,7 @@ def create_user(username, password, email, is_organization=False):
# If the password is None, then add a notification for the user to change # If the password is None, then add a notification for the user to change
# their password ASAP. # their password ASAP.
if not pw_hash and not is_organization: if not pw_hash and add_change_pw_notification:
create_notification('password_required', new_user) create_notification('password_required', new_user)
return new_user return new_user
@ -105,10 +111,18 @@ def create_user(username, password, email, is_organization=False):
raise DataModelException(ex.message) raise DataModelException(ex.message)
def is_username_unique(test_username):
try:
User.get((User.username == test_username))
return False
except User.DoesNotExist:
return True
def create_organization(name, email, creating_user): def create_organization(name, email, creating_user):
try: try:
# Create the org # Create the org
new_org = create_user(name, None, email, is_organization=True) new_org = create_user(name, None, email, add_change_pw_notification=False)
new_org.organization = True new_org.organization = True
new_org.save() new_org.save()
@ -340,17 +354,15 @@ def attach_federated_login(user, service_name, service_id):
def verify_federated_login(service_name, service_id): def verify_federated_login(service_name, service_id):
selected = FederatedLogin.select(FederatedLogin, User) try:
with_service = selected.join(LoginService) found = (FederatedLogin
with_user = with_service.switch(FederatedLogin).join(User) .select(FederatedLogin, User)
found = with_user.where(FederatedLogin.service_ident == service_id, .join(LoginService)
LoginService.name == service_name) .switch(FederatedLogin).join(User)
.where(FederatedLogin.service_ident == service_id, LoginService.name == service_name)
found_list = list(found) .get())
return found.user
if found_list: except FederatedLogin.DoesNotExist:
return found_list[0].user
return None return None
@ -935,7 +947,7 @@ def __translate_ancestry(old_ancestry, translations, repository, username):
def find_create_or_link_image(docker_image_id, repository, username, def find_create_or_link_image(docker_image_id, repository, username,
translations): translations):
with transaction_factory(db): with config.app_config['DB_TRANSACTION_FACTORY'](db):
repo_image = get_repo_image(repository.namespace, repository.name, repo_image = get_repo_image(repository.namespace, repository.name,
docker_image_id) docker_image_id)
if repo_image: if repo_image:
@ -1018,7 +1030,7 @@ def set_image_size(docker_image_id, namespace_name, repository_name,
def set_image_metadata(docker_image_id, namespace_name, repository_name, def set_image_metadata(docker_image_id, namespace_name, repository_name,
created_date_str, comment, command, parent=None): created_date_str, comment, command, parent=None):
with transaction_factory(db): with config.app_config['DB_TRANSACTION_FACTORY'](db):
query = (Image query = (Image
.select(Image, ImageStorage) .select(Image, ImageStorage)
.join(Repository) .join(Repository)
@ -1064,7 +1076,7 @@ def list_repository_tags(namespace_name, repository_name):
def garbage_collect_repository(namespace_name, repository_name): def garbage_collect_repository(namespace_name, repository_name):
with transaction_factory(db): with config.app_config['DB_TRANSACTION_FACTORY'](db):
# Get a list of all images used by tags in the repository # Get a list of all images used by tags in the repository
tag_query = (RepositoryTag tag_query = (RepositoryTag
.select(RepositoryTag, Image, ImageStorage) .select(RepositoryTag, Image, ImageStorage)
@ -1098,10 +1110,10 @@ def garbage_collect_repository(namespace_name, repository_name):
image_to_remove.storage.uuid) image_to_remove.storage.uuid)
uuids_to_check_for_gc.add(image_to_remove.storage.uuid) uuids_to_check_for_gc.add(image_to_remove.storage.uuid)
else: else:
image_path = store.image_path(namespace_name, repository_name, image_path = config.store.image_path(namespace_name, repository_name,
image_to_remove.docker_image_id, None) image_to_remove.docker_image_id, None)
logger.debug('Deleting image storage: %s', image_path) logger.debug('Deleting image storage: %s', image_path)
store.remove(image_path) config.store.remove(image_path)
image_to_remove.delete_instance() image_to_remove.delete_instance()
@ -1116,10 +1128,9 @@ def garbage_collect_repository(namespace_name, repository_name):
for storage in storage_to_remove: for storage in storage_to_remove:
logger.debug('Garbage collecting image storage: %s', storage.uuid) logger.debug('Garbage collecting image storage: %s', storage.uuid)
storage.delete_instance() storage.delete_instance()
image_path = store.image_path(namespace_name, repository_name, image_path = config.store.image_path(namespace_name, repository_name,
image_to_remove.docker_image_id, image_to_remove.docker_image_id, storage.uuid)
storage.uuid) config.store.remove(image_path)
store.remove(image_path)
return len(to_remove) return len(to_remove)
@ -1489,8 +1500,8 @@ def get_pull_credentials(robotname):
return { return {
'username': robot.username, 'username': robot.username,
'password': login_info.service_ident, 'password': login_info.service_ident,
'registry': '%s://%s/v1/' % (app.config['PREFERRED_URL_SCHEME'], 'registry': '%s://%s/v1/' % (config.app_config['PREFERRED_URL_SCHEME'],
app.config['SERVER_HOSTNAME']), config.app_config['SERVER_HOSTNAME']),
} }

18
data/queue.py
View file

@ -1,19 +1,17 @@
from datetime import datetime, timedelta from datetime import datetime, timedelta
from data.database import QueueItem, db from data.database import QueueItem, db
from app import app
transaction_factory = app.config['DB_TRANSACTION_FACTORY']
MINIMUM_EXTENSION = timedelta(seconds=20) MINIMUM_EXTENSION = timedelta(seconds=20)
class WorkQueue(object): class WorkQueue(object):
def __init__(self, queue_name, canonical_name_match_list=None, reporter=None): def __init__(self, queue_name, transaction_factory,
canonical_name_match_list=None, reporter=None):
self._queue_name = queue_name self._queue_name = queue_name
self._reporter = reporter self._reporter = reporter
self._transaction_factory = transaction_factory
if canonical_name_match_list is None: if canonical_name_match_list is None:
self._canonical_name_match_list = [] self._canonical_name_match_list = []
@ -55,7 +53,7 @@ class WorkQueue(object):
self._reporter(running, total_jobs) self._reporter(running, total_jobs)
def update_metrics(self): def update_metrics(self):
with transaction_factory(db): with self._transaction_factory(db):
self._report_queue_metrics() self._report_queue_metrics()
def put(self, canonical_name_list, message, available_after=0, retries_remaining=5): def put(self, canonical_name_list, message, available_after=0, retries_remaining=5):
@ -74,7 +72,7 @@ class WorkQueue(object):
available_date = datetime.now() + timedelta(seconds=available_after) available_date = datetime.now() + timedelta(seconds=available_after)
params['available_after'] = available_date params['available_after'] = available_date
with transaction_factory(db): with self._transaction_factory(db):
QueueItem.create(**params) QueueItem.create(**params)
self._report_queue_metrics() self._report_queue_metrics()
@ -87,7 +85,7 @@ class WorkQueue(object):
name_match_query = self._name_match_query() name_match_query = self._name_match_query()
with transaction_factory(db): with self._transaction_factory(db):
running = self._running_jobs(now, name_match_query) running = self._running_jobs(now, name_match_query)
avail = QueueItem.select().where(QueueItem.queue_name ** name_match_query, avail = QueueItem.select().where(QueueItem.queue_name ** name_match_query,
@ -113,12 +111,12 @@ class WorkQueue(object):
return item return item
def complete(self, completed_item): def complete(self, completed_item):
with transaction_factory(db): with self._transaction_factory(db):
completed_item.delete_instance() completed_item.delete_instance()
self._report_queue_metrics() self._report_queue_metrics()
def incomplete(self, incomplete_item, retry_after=300, restore_retry=False): def incomplete(self, incomplete_item, retry_after=300, restore_retry=False):
with transaction_factory(db): with self._transaction_factory(db):
retry_date = datetime.now() + timedelta(seconds=retry_after) retry_date = datetime.now() + timedelta(seconds=retry_after)
incomplete_item.available_after = retry_date incomplete_item.available_after = retry_date
incomplete_item.available = True incomplete_item.available = True

144
data/users.py Normal file
View file

@ -0,0 +1,144 @@
import ldap
import logging
from util.validation import generate_valid_usernames
from data import model
logger = logging.getLogger(__name__)
class DatabaseUsers(object):
def verify_user(self, username_or_email, password):
""" Simply delegate to the model implementation. """
return model.verify_user(username_or_email, password)
def user_exists(self, username):
return model.get_user(username) is not None
class LDAPConnection(object):
def __init__(self, ldap_uri, user_dn, user_pw):
self._ldap_uri = ldap_uri
self._user_dn = user_dn
self._user_pw = user_pw
self._conn = None
def __enter__(self):
self._conn = ldap.initialize(self._ldap_uri)
self._conn.simple_bind_s(self._user_dn, self._user_pw)
return self._conn
def __exit__(self, exc_type, value, tb):
self._conn.unbind_s()
class LDAPUsers(object):
def __init__(self, ldap_uri, base_dn, admin_dn, admin_passwd, user_rdn, uid_attr, email_attr):
self._ldap_conn = LDAPConnection(ldap_uri, admin_dn, admin_passwd)
self._ldap_uri = ldap_uri
self._base_dn = base_dn
self._user_rdn = user_rdn
self._uid_attr = uid_attr
self._email_attr = email_attr
def _ldap_user_search(self, username_or_email):
with self._ldap_conn as conn:
logger.debug('Incoming username or email param: %s', username_or_email.__repr__())
user_search_dn = ','.join(self._user_rdn + self._base_dn)
query = u'(|({0}={2})({1}={2}))'.format(self._uid_attr, self._email_attr,
username_or_email)
user = conn.search_s(user_search_dn, ldap.SCOPE_SUBTREE, query.encode('utf-8'))
if len(user) != 1:
return None
return user[0]
def verify_user(self, username_or_email, password):
""" Verify the credentials with LDAP and if they are valid, create or update the user
in our database. """
# Make sure that even if the server supports anonymous binds, we don't allow it
if not password:
return None
found_user = self._ldap_user_search(username_or_email)
if found_user is None:
return None
found_dn, found_response = found_user
# First validate the password by binding as the user
try:
with LDAPConnection(self._ldap_uri, found_dn, password.encode('utf-8')):
pass
except ldap.INVALID_CREDENTIALS:
return None
# Now check if we have a federated login for this user
username = found_response[self._uid_attr][0].decode('utf-8')
email = found_response[self._email_attr][0]
db_user = model.verify_federated_login('ldap', username)
if not db_user:
# We must create the user in our db
valid_username = None
for valid_username in generate_valid_usernames(username):
if model.is_username_unique(valid_username):
break
if not valid_username:
logger.error('Unable to pick a username for user: %s', username)
return None
db_user = model.create_user(valid_username, None, email, add_change_pw_notification=False)
db_user.verified = True
model.attach_federated_login(db_user, 'ldap', username)
else:
# Update the db attributes from ldap
db_user.email = email
db_user.save()
return db_user
def user_exists(self, username):
found_user = self._ldap_user_search(username)
return found_user is not None
class UserAuthentication(object):
def __init__(self, app=None):
self.app = app
if app is not None:
self.state = self.init_app(app)
else:
self.state = None
def init_app(self, app):
authentication_type = app.config.get('AUTHENTICATION_TYPE', 'Database')
if authentication_type == 'Database':
users = DatabaseUsers()
elif authentication_type == 'LDAP':
ldap_uri = app.config.get('LDAP_URI', 'ldap://localhost')
base_dn = app.config.get('LDAP_BASE_DN')
admin_dn = app.config.get('LDAP_ADMIN_DN')
admin_passwd = app.config.get('LDAP_ADMIN_PASSWD')
user_rdn = app.config.get('LDAP_USER_RDN', [])
uid_attr = app.config.get('LDAP_UID_ATTR', 'uid')
email_attr = app.config.get('LDAP_EMAIL_ATTR', 'mail')
users = LDAPUsers(ldap_uri, base_dn, admin_dn, admin_passwd, user_rdn, uid_attr, email_attr)
else:
raise RuntimeError('Unknown authentication type: %s' % authentication_type)
# register extension with app
app.extensions = getattr(app, 'extensions', {})
app.extensions['authentication'] = users
return users
def __getattr__(self, name):
return getattr(self.state, name, None)

6
endpoints/api/user.py
View file

@ -5,7 +5,7 @@ from flask import request
from flask.ext.login import logout_user from flask.ext.login import logout_user
from flask.ext.principal import identity_changed, AnonymousIdentity from flask.ext.principal import identity_changed, AnonymousIdentity
from app import app, billing as stripe from app import app, billing as stripe, authentication
from endpoints.api import (ApiResource, nickname, resource, validate_json_request, request_error, from endpoints.api import (ApiResource, nickname, resource, validate_json_request, request_error,
log_action, internal_only, NotFound, require_user_admin, log_action, internal_only, NotFound, require_user_admin,
InvalidToken, require_scope, format_date, hide_if, show_if) InvalidToken, require_scope, format_date, hide_if, show_if)
@ -227,7 +227,7 @@ def conduct_signin(username_or_email, password):
needs_email_verification = False needs_email_verification = False
invalid_credentials = False invalid_credentials = False
verified = model.verify_user(username_or_email, password) verified = authentication.verify_user(username_or_email, password)
if verified: if verified:
if common_login(verified): if common_login(verified):
return {'success': True} return {'success': True}
@ -289,7 +289,7 @@ class ConvertToOrganization(ApiResource):
# Ensure that the sign in credentials work. # Ensure that the sign in credentials work.
admin_password = convert_data['adminPassword'] admin_password = convert_data['adminPassword']
if not model.verify_user(admin_username, admin_password): if not authentication.verify_user(admin_username, admin_password):
raise request_error(reason='invaliduser', raise request_error(reason='invaliduser',
message='The admin user credentials are not valid') message='The admin user credentials are not valid')

6
endpoints/common.py
View file

@ -16,6 +16,7 @@ from endpoints.api.discovery import swagger_route_data
from werkzeug.routing import BaseConverter from werkzeug.routing import BaseConverter
from functools import wraps from functools import wraps
from config import getFrontendVisibleConfig from config import getFrontendVisibleConfig
from external_libraries import get_external_javascript, get_external_css
import features import features
@ -146,7 +147,12 @@ def render_page_template(name, **kwargs):
main_scripts = ['dist/quay-frontend.min.js'] main_scripts = ['dist/quay-frontend.min.js']
cache_buster = random_string() cache_buster = random_string()
external_styles = get_external_css(local=not app.config.get('USE_CDN', True))
external_scripts = get_external_javascript(local=not app.config.get('USE_CDN', True))
resp = make_response(render_template(name, route_data=json.dumps(get_route_data()), resp = make_response(render_template(name, route_data=json.dumps(get_route_data()),
external_styles=external_styles,
external_scripts=external_scripts,
main_styles=main_styles, main_styles=main_styles,
library_styles=library_styles, library_styles=library_styles,
main_scripts=main_scripts, main_scripts=main_scripts,

7
endpoints/index.py
View file

@ -8,7 +8,7 @@ from collections import OrderedDict
from data import model from data import model
from data.model import oauth from data.model import oauth
from app import analytics, app, webhook_queue from app import analytics, app, webhook_queue, authentication
from auth.auth import process_auth from auth.auth import process_auth
from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token
from util.names import parse_repository_name from util.names import parse_repository_name
@ -94,9 +94,8 @@ def create_user():
abort(400, 'Invalid robot account or password.', abort(400, 'Invalid robot account or password.',
issue='robot-login-failure') issue='robot-login-failure')
existing_user = model.get_user(username) if authentication.user_exists(username):
if existing_user: verified = authentication.verify_user(username, password)
verified = model.verify_user(username, password)
if verified: if verified:
# Mark that the user was logged in. # Mark that the user was logged in.
event = app.config['USER_EVENTS'].get_event(username) event = app.config['USER_EVENTS'].get_event(username)

77
external_libraries.py Normal file
View file

@ -0,0 +1,77 @@
import urllib2
import re
import os
LOCAL_DIRECTORY = 'static/ldn/'
EXTERNAL_JS = [
'code.jquery.com/jquery.js',
'netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js',
'ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular.min.js',
'ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular-route.min.js',
'ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular-sanitize.min.js',
'ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular-animate.min.js',
'cdnjs.cloudflare.com/ajax/libs/bootstrap-datepicker/1.2.0/js/bootstrap-datepicker.min.js',
'cdn.jsdelivr.net/g/bootbox@4.1.0,underscorejs@1.5.2,restangular@1.2.0,d3js@3.3.3,momentjs',
'cdn.ravenjs.com/1.1.14/jquery,native/raven.min.js',
'checkout.stripe.com/checkout.js',
]
EXTERNAL_CSS = [
'netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css',
'netdna.bootstrapcdn.com/bootstrap/3.0.0/css/bootstrap.no-icons.min.css',
'fonts.googleapis.com/css?family=Droid+Sans:400,700',
]
EXTERNAL_FONTS = [
'netdna.bootstrapcdn.com/font-awesome/4.0.3/fonts/fontawesome-webfont.woff?v=4.0.3',
'netdna.bootstrapcdn.com/font-awesome/4.0.3/fonts/fontawesome-webfont.ttf?v=4.0.3',
'netdna.bootstrapcdn.com/font-awesome/4.0.3/fonts/fontawesome-webfont.svg?v=4.0.3',
]
def get_external_javascript(local=False):
if local:
return [LOCAL_DIRECTORY + format_local_name(src) for src in EXTERNAL_JS]
return ['//' + src for src in EXTERNAL_JS]
def get_external_css(local=False):
if local:
return [LOCAL_DIRECTORY + format_local_name(src) for src in EXTERNAL_CSS]
return ['//' + src for src in EXTERNAL_CSS]
def format_local_name(url):
filename = url.split('/')[-1]
filename = re.sub(r'[+,?@=:]', '', filename)
if not filename.endswith('.css') and not filename.endswith('.js'):
if filename.find('css') >= 0:
filename = filename + '.css'
else:
filename = filename + '.js'
return filename
if __name__ == '__main__':
for url in EXTERNAL_JS + EXTERNAL_CSS:
print 'Downloading %s' % url
response = urllib2.urlopen('https://' + url)
contents = response.read()
filename = format_local_name(url)
print 'Writing %s' % filename
with open(LOCAL_DIRECTORY + filename, 'w') as f:
f.write(contents)
for url in EXTERNAL_FONTS:
print 'Downloading %s' % url
response = urllib2.urlopen('https://' + url)
filename = os.path.basename(url).split('?')[0]
with open('static/fonts/' + filename, "wb") as local_file:
local_file.write(response.read())

5
initdb.py
View file

@ -148,7 +148,7 @@ def setup_database_for_testing(testcase):
# Sanity check to make sure we're not killing our prod db # Sanity check to make sure we're not killing our prod db
db = model.db db = model.db
if not isinstance(model.db, SqliteDatabase): if not isinstance(model.db.obj, SqliteDatabase):
raise RuntimeError('Attempted to wipe production database!') raise RuntimeError('Attempted to wipe production database!')
global db_initialized_for_testing global db_initialized_for_testing
@ -181,6 +181,7 @@ def initialize_database():
Visibility.create(name='private') Visibility.create(name='private')
LoginService.create(name='github') LoginService.create(name='github')
LoginService.create(name='quayrobot') LoginService.create(name='quayrobot')
LoginService.create(name='ldap')
BuildTriggerService.create(name='github') BuildTriggerService.create(name='github')
@ -241,7 +242,7 @@ def wipe_database():
# Sanity check to make sure we're not killing our prod db # Sanity check to make sure we're not killing our prod db
db = model.db db = model.db
if not isinstance(model.db, SqliteDatabase): if not isinstance(model.db.obj, SqliteDatabase):
raise RuntimeError('Attempted to wipe production database!') raise RuntimeError('Attempted to wipe production database!')
drop_model_tables(all_models, fail_silently=True) drop_model_tables(all_models, fail_silently=True)

2
requirements-nover.txt
View file

@ -32,3 +32,5 @@ python-magic
reportlab==2.7 reportlab==2.7
blinker blinker
raven raven
python-ldap
unidecode

2
requirements.txt
View file

@ -12,6 +12,7 @@ PyGithub==1.24.1
PyMySQL==0.6.2 PyMySQL==0.6.2
PyPDF2==1.21 PyPDF2==1.21
SQLAlchemy==0.9.4 SQLAlchemy==0.9.4
Unidecode==0.04.16
Werkzeug==0.9.4 Werkzeug==0.9.4
alembic==0.6.4 alembic==0.6.4
aniso8601==0.82 aniso8601==0.82
@ -40,6 +41,7 @@ pycrypto==2.6.1
python-daemon==1.6 python-daemon==1.6
python-dateutil==2.2 python-dateutil==2.2
python-digitalocean==0.7 python-digitalocean==0.7
python-ldap==2.4.15
python-magic==0.4.6 python-magic==0.4.6
pytz==2014.2 pytz==2014.2
raven==4.2.1 raven==4.2.1

23
templates/base.html
View file

@ -11,9 +11,9 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="stylesheet" href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css"> {% for style_url in external_styles %}
<link rel="stylesheet" href="//netdna.bootstrapcdn.com/bootstrap/3.0.0/css/bootstrap.no-icons.min.css"> <link rel="stylesheet" href="{{ style_url }}" type="text/css">
<link href='//fonts.googleapis.com/css?family=Droid+Sans:400,700' rel='stylesheet' type='text/css'> {% endfor %}
<!-- Icons --> <!-- Icons -->
<link rel="shortcut icon" href="/static/img/favicon.ico" type="image/x-icon" /> <link rel="shortcut icon" href="/static/img/favicon.ico" type="image/x-icon" />
@ -47,20 +47,9 @@
window.__token = '{{ csrf_token() }}'; window.__token = '{{ csrf_token() }}';
</script> </script>
<script src="//code.jquery.com/jquery.js"></script> {% for script_url in external_scripts %}
<script src="//netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script> <script src="{{ script_url }}"></script>
{% endfor %}
<script src="//ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular.min.js"></script>
<script src="//ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular-route.min.js"></script>
<script src="//ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular-sanitize.min.js"></script>
<script src="//ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular-animate.min.js"></script>
<script src="//cdnjs.cloudflare.com/ajax/libs/bootstrap-datepicker/1.2.0/js/bootstrap-datepicker.min.js"></script>
<script src="//cdn.jsdelivr.net/g/bootbox@4.1.0,underscorejs@1.5.2,restangular@1.2.0,d3js@3.3.3,momentjs"></script>
<script src="//cdn.ravenjs.com/1.1.14/jquery,native/raven.min.js"></script>
<script src="https://checkout.stripe.com/checkout.js"></script>
{% for script_path in library_scripts %} {% for script_path in library_scripts %}
<script src="/static/{{ script_path }}?v={{ cache_buster }}"></script> <script src="/static/{{ script_path }}?v={{ cache_buster }}"></script>

BIN
test/data/test.db
View file

Binary file not shown.

85
test/test_api_security.py
View file

@ -36,6 +36,9 @@ from endpoints.api.repository import RepositoryList, RepositoryVisibility, Repos
from endpoints.api.permission import (RepositoryUserPermission, RepositoryTeamPermission, from endpoints.api.permission import (RepositoryUserPermission, RepositoryTeamPermission,
RepositoryTeamPermissionList, RepositoryUserPermissionList) RepositoryTeamPermissionList, RepositoryUserPermissionList)
from endpoints.api.superuser import SuperUserLogs, SeatUsage, SuperUserList, SuperUserManagement
try: try:
app.register_blueprint(api_bp, url_prefix='/api') app.register_blueprint(api_bp, url_prefix='/api')
except ValueError: except ValueError:
@ -3275,5 +3278,87 @@ class TestUserAuthorization(ApiTestCase):
self._run_test('DELETE', 404, 'devtable', None) self._run_test('DELETE', 404, 'devtable', None)
class TestSuperUserLogs(ApiTestCase):
def setUp(self):
ApiTestCase.setUp(self)
self._set_url(SuperUserLogs)
def test_get_anonymous(self):
self._run_test('GET', 403, None, None)
def test_get_freshuser(self):
self._run_test('GET', 403, 'freshuser', None)
def test_get_reader(self):
self._run_test('GET', 403, 'reader', None)
def test_get_devtable(self):
self._run_test('GET', 200, 'devtable', None)
class TestSuperUserList(ApiTestCase):
def setUp(self):
ApiTestCase.setUp(self)
self._set_url(SuperUserList)
def test_get_anonymous(self):
self._run_test('GET', 403, None, None)
def test_get_freshuser(self):
self._run_test('GET', 403, 'freshuser', None)
def test_get_reader(self):
self._run_test('GET', 403, 'reader', None)
def test_get_devtable(self):
self._run_test('GET', 200, 'devtable', None)
class TestSuperUserManagement(ApiTestCase):
def setUp(self):
ApiTestCase.setUp(self)
self._set_url(SuperUserManagement, username='freshuser')
def test_get_anonymous(self):
self._run_test('GET', 403, None, None)
def test_get_freshuser(self):
self._run_test('GET', 403, 'freshuser', None)
def test_get_reader(self):
self._run_test('GET', 403, 'reader', None)
def test_get_devtable(self):
self._run_test('GET', 200, 'devtable', None)
def test_put_anonymous(self):
self._run_test('PUT', 403, None, {})
def test_put_freshuser(self):
self._run_test('PUT', 403, 'freshuser', {})
def test_put_reader(self):
self._run_test('PUT', 403, 'reader', {})
def test_put_devtable(self):
self._run_test('PUT', 200, 'devtable', {})
def test_delete_anonymous(self):
self._run_test('DELETE', 403, None, None)
def test_delete_freshuser(self):
self._run_test('DELETE', 403, 'freshuser', None)
def test_delete_reader(self):
self._run_test('DELETE', 403, 'reader', None)
def test_delete_devtable(self):
self._run_test('DELETE', 204, 'devtable', None)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

62
test/test_api_usage.py
View file

@ -38,6 +38,7 @@ from endpoints.api.organization import (OrganizationList, OrganizationMember,
from endpoints.api.repository import RepositoryList, RepositoryVisibility, Repository from endpoints.api.repository import RepositoryList, RepositoryVisibility, Repository
from endpoints.api.permission import (RepositoryUserPermission, RepositoryTeamPermission, from endpoints.api.permission import (RepositoryUserPermission, RepositoryTeamPermission,
RepositoryTeamPermissionList, RepositoryUserPermissionList) RepositoryTeamPermissionList, RepositoryUserPermissionList)
from endpoints.api.superuser import SuperUserLogs, SeatUsage, SuperUserList, SuperUserManagement
try: try:
app.register_blueprint(api_bp, url_prefix='/api') app.register_blueprint(api_bp, url_prefix='/api')
@ -1939,5 +1940,66 @@ class TestUserAuthorizations(ApiTestCase):
self.getJsonResponse(UserAuthorization, params=dict(access_token_uuid = authorization['uuid']), self.getJsonResponse(UserAuthorization, params=dict(access_token_uuid = authorization['uuid']),
expected_code=404) expected_code=404)
class TestSuperUserLogs(ApiTestCase):
def test_get_logs(self):
self.login(ADMIN_ACCESS_USER)
json = self.getJsonResponse(SuperUserLogs)
assert 'logs' in json
assert len(json['logs']) > 0
class TestSuperUserList(ApiTestCase):
def test_get_users(self):
self.login(ADMIN_ACCESS_USER)
json = self.getJsonResponse(SuperUserList)
assert 'users' in json
assert len(json['users']) > 0
class TestSuperUserManagement(ApiTestCase):
def test_get_user(self):
self.login(ADMIN_ACCESS_USER)
json = self.getJsonResponse(SuperUserManagement, params=dict(username = 'freshuser'))
self.assertEquals('freshuser', json['username'])
self.assertEquals('no@thanks.com', json['email'])
self.assertEquals(False, json['super_user'])
def test_delete_user(self):
self.login(ADMIN_ACCESS_USER)
# Verify the user exists.
json = self.getJsonResponse(SuperUserManagement, params=dict(username = 'freshuser'))
self.assertEquals('freshuser', json['username'])
# Delete the user.
self.deleteResponse(SuperUserManagement, params=dict(username = 'freshuser'), expected_code=204)
# Verify the user no longer exists.
self.getResponse(SuperUserManagement, params=dict(username = 'freshuser'), expected_code=404)
def test_update_user(self):
self.login(ADMIN_ACCESS_USER)
# Verify the user exists.
json = self.getJsonResponse(SuperUserManagement, params=dict(username = 'freshuser'))
self.assertEquals('freshuser', json['username'])
self.assertEquals('no@thanks.com', json['email'])
# Update the user.
self.putJsonResponse(SuperUserManagement, params=dict(username='freshuser'), data=dict(email='foo@bar.com'))
# Verify the user was updated.
json = self.getJsonResponse(SuperUserManagement, params=dict(username = 'freshuser'))
self.assertEquals('freshuser', json['username'])
self.assertEquals('foo@bar.com', json['email'])
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

6
test/test_queue.py
View file

@ -2,6 +2,7 @@ import unittest
import json import json
import time import time
from app import app
from initdb import setup_database_for_testing, finished_database_for_testing from initdb import setup_database_for_testing, finished_database_for_testing
from data.queue import WorkQueue from data.queue import WorkQueue
@ -25,7 +26,8 @@ class QueueTestCase(unittest.TestCase):
def setUp(self): def setUp(self):
self.reporter = SaveLastCountReporter() self.reporter = SaveLastCountReporter()
self.queue = WorkQueue(QUEUE_NAME, reporter=self.reporter) self.transaction_factory = app.config['DB_TRANSACTION_FACTORY']
self.queue = WorkQueue(QUEUE_NAME, self.transaction_factory, reporter=self.reporter)
setup_database_for_testing(self) setup_database_for_testing(self)
def tearDown(self): def tearDown(self):
@ -118,7 +120,7 @@ class TestQueue(QueueTestCase):
self.queue.put(['abc', 'def'], self.TEST_MESSAGE_1) self.queue.put(['abc', 'def'], self.TEST_MESSAGE_1)
self.queue.put(['def', 'def'], self.TEST_MESSAGE_2) self.queue.put(['def', 'def'], self.TEST_MESSAGE_2)
my_queue = WorkQueue(QUEUE_NAME, ['def']) my_queue = WorkQueue(QUEUE_NAME, self.transaction_factory, ['def'])
two = my_queue.get() two = my_queue.get()
self.assertNotEqual(None, two) self.assertNotEqual(None, two)

3
test/testconfig.py
View file

@ -29,3 +29,6 @@ class TestConfig(DefaultConfig):
'deadbeef-dead-beef-dead-beefdeadbeef'] 'deadbeef-dead-beef-dead-beefdeadbeef']
USERFILES_TYPE = 'FakeUserfiles' USERFILES_TYPE = 'FakeUserfiles'
FEATURE_SUPER_USERS = True
SUPER_USERS = ['devtable']

37
util/validation.py
View file

@ -1,7 +1,16 @@
import re import re
import string
from unidecode import unidecode
INVALID_PASSWORD_MESSAGE = 'Invalid password, password must be at least ' + \ INVALID_PASSWORD_MESSAGE = 'Invalid password, password must be at least ' + \
'8 characters and contain no whitespace.' '8 characters and contain no whitespace.'
INVALID_USERNAME_CHARACTERS = r'[^a-z0-9_]'
VALID_CHARACTERS = '_' + string.digits + string.lowercase
MIN_LENGTH = 4
MAX_LENGTH = 30
def validate_email(email_address): def validate_email(email_address):
if re.match(r'[^@]+@[^@]+\.[^@]+', email_address): if re.match(r'[^@]+@[^@]+\.[^@]+', email_address):
@ -11,13 +20,14 @@ def validate_email(email_address):
def validate_username(username): def validate_username(username):
# Based off the restrictions defined in the Docker Registry API spec # Based off the restrictions defined in the Docker Registry API spec
regex_match = (re.search(r'[^a-z0-9_]', username) is None) regex_match = (re.search(INVALID_USERNAME_CHARACTERS, username) is None)
if not regex_match: if not regex_match:
return (False, 'Username must match expression [a-z0-9_]+') return (False, 'Username must match expression [a-z0-9_]+')
length_match = (len(username) >= 4 and len(username) <= 30) length_match = (len(username) >= MIN_LENGTH and len(username) <= MAX_LENGTH)
if not length_match: if not length_match:
return (False, 'Username must be between 4 and 30 characters in length') return (False, 'Username must be between %s and %s characters in length' %
(MIN_LENGTH, MAX_LENGTH))
return (True, '') return (True, '')
@ -27,3 +37,24 @@ def validate_password(password):
if re.search(r'\s', password): if re.search(r'\s', password):
return False return False
return len(password) > 7 return len(password) > 7
def _gen_filler_chars(num_filler_chars):
if num_filler_chars == 0:
yield ''
else:
for char in VALID_CHARACTERS:
for suffix in _gen_filler_chars(num_filler_chars - 1):
yield char + suffix
def generate_valid_usernames(input_username):
normalized = unidecode(input_username).strip().lower()
prefix = re.sub(INVALID_USERNAME_CHARACTERS, '_', normalized)[:30]
num_filler_chars = max(0, MIN_LENGTH - len(prefix))
while num_filler_chars + len(prefix) <= MAX_LENGTH:
for suffix in _gen_filler_chars(num_filler_chars):
yield prefix + suffix
num_filler_chars += 1