diff --git a/README.md b/README.md index 7018c76a1..ebeee06ad 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,22 @@ virtualenv --distribute venv source venv/bin/activate pip install -r requirements.txt sudo gdebi --n binary_dependencies/*.deb +sudo cp conf/logrotate/* /etc/logrotate.d/ ``` running: ``` -sudo mkdir -p /mnt/nginx/ && sudo /usr/local/nginx/sbin/nginx -c `pwd`/nginx.conf -STACK=prod gunicorn -c gunicorn_config.py application:application +sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && sudo /usr/local/nginx/sbin/nginx -c `pwd`/conf/nginx.conf +sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && STACK=prod gunicorn -c gunicorn_config.py application:application +``` + +start the log shipper: + +``` +curl -s https://get.docker.io/ubuntu/ | sudo sh +sudo docker pull quay.io/quay/logstash +sudo docker run -d -e REDIS_PORT_6379_TCP_ADDR=logs.quay.io -v /mnt/logs:/mnt/logs quay.io/quay/logstash quay.conf ``` start the workers: @@ -34,8 +43,8 @@ STACK=prod python -m workers.webhookworker -D bouncing the servers: ``` -sudo kill -HUP -kill -HUP +sudo kill -HUP `cat /mnt/logs/nginx.pid` +kill -HUP `cat /mnt/logs/gunicorn.pid` kill restart daemons diff --git a/application.py b/application.py index 26101f50b..80411d50f 100644 --- a/application.py +++ b/application.py @@ -1,10 +1,15 @@ import logging +import os from app import app as application +from data.model import db as model_db -logging.basicConfig(**application.config['LOGGING_CONFIG']) +# Initialize logging +application.config['LOGGING_CONFIG']() +# Turn off debug logging for boto +logging.getLogger('boto').setLevel(logging.CRITICAL) from endpoints.api import api from endpoints.index import index @@ -27,6 +32,16 @@ application.register_blueprint(registry, url_prefix='/v1') application.register_blueprint(api, url_prefix='/api') application.register_blueprint(webhooks, url_prefix='/webhooks') + +def close_db(exc): + db = model_db + if not db.is_closed(): + logger.debug('Disconnecting from database.') + db.close() + +application.teardown_request(close_db) + + # Remove this for prod config application.debug = True diff --git a/certs/digital_ocean b/conf/certs/digital_ocean similarity index 100% rename from certs/digital_ocean rename to conf/certs/digital_ocean diff --git a/certs/digital_ocean.pub b/conf/certs/digital_ocean.pub similarity index 100% rename from certs/digital_ocean.pub rename to conf/certs/digital_ocean.pub diff --git a/certs/quay-enc.key b/conf/certs/quay-enc.key similarity index 100% rename from certs/quay-enc.key rename to conf/certs/quay-enc.key diff --git a/certs/quay-staging-enc.key b/conf/certs/quay-staging-enc.key similarity index 100% rename from certs/quay-staging-enc.key rename to conf/certs/quay-staging-enc.key diff --git a/certs/quay-staging-unified.cert b/conf/certs/quay-staging-unified.cert similarity index 100% rename from certs/quay-staging-unified.cert rename to conf/certs/quay-staging-unified.cert diff --git a/certs/quay-staging.cert b/conf/certs/quay-staging.cert similarity index 100% rename from certs/quay-staging.cert rename to conf/certs/quay-staging.cert diff --git a/certs/quay-staging.key b/conf/certs/quay-staging.key similarity index 100% rename from certs/quay-staging.key rename to conf/certs/quay-staging.key diff --git a/certs/quay-unified.cert b/conf/certs/quay-unified.cert similarity index 100% rename from certs/quay-unified.cert rename to conf/certs/quay-unified.cert diff --git a/certs/quay.cert b/conf/certs/quay.cert similarity index 100% rename from certs/quay.cert rename to conf/certs/quay.cert diff --git a/certs/quay.key b/conf/certs/quay.key similarity index 100% rename from certs/quay.key rename to conf/certs/quay.key diff --git a/conf/hosted-http-base.conf b/conf/hosted-http-base.conf new file mode 100644 index 000000000..c3e910e8f --- /dev/null +++ b/conf/hosted-http-base.conf @@ -0,0 +1,5 @@ +server { + listen 80 default_server; + server_name _; + rewrite ^ https://$host$request_uri? permanent; +} diff --git a/conf/http-base.conf b/conf/http-base.conf new file mode 100644 index 000000000..32e8b3730 --- /dev/null +++ b/conf/http-base.conf @@ -0,0 +1,33 @@ +log_format logstash_json '{ "@timestamp": "$time_iso8601", ' + '"@fields": { ' + '"remote_addr": "$remote_addr", ' + '"remote_user": "$remote_user", ' + '"body_bytes_sent": "$body_bytes_sent", ' + '"request_time": "$request_time", ' + '"status": "$status", ' + '"request": "$request", ' + '"request_method": "$request_method", ' + '"http_referrer": "$http_referer", ' + '"http_user_agent": "$http_user_agent" } }'; + +types_hash_max_size 2048; +include /usr/local/nginx/conf/mime.types.default; + +default_type application/octet-stream; +access_log /mnt/logs/nginx.access.log logstash_json; +sendfile on; + +gzip on; +gzip_http_version 1.0; +gzip_proxied any; +gzip_min_length 500; +gzip_disable "MSIE [1-6]\."; +gzip_types text/plain text/xml text/css + text/javascript application/x-javascript + application/octet-stream; + +upstream app_server { + server unix:/tmp/gunicorn.sock fail_timeout=0; + # For a TCP configuration: + # server 192.168.0.7:8000 fail_timeout=0; +} diff --git a/conf/logrotate/quay-logrotate b/conf/logrotate/quay-logrotate new file mode 100644 index 000000000..79fdc377d --- /dev/null +++ b/conf/logrotate/quay-logrotate @@ -0,0 +1,41 @@ +/mnt/logs/nginx.access.log { + daily + rotate 7 + compress + delaycompress + missingok + notifempty + create 644 root root + + postrotate + [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /mnt/logs/nginx.pid` + endscript +} + +/mnt/logs/nginx.error.log { + daily + rotate 7 + compress + delaycompress + missingok + notifempty + create 644 root root + + postrotate + [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /mnt/logs/nginx.pid` + endscript +} + +/mnt/logs/application.log { + daily + rotate 7 + compress + delaycompress + missingok + notifempty + create 644 ubuntu ubuntu + + postrotate + [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /mnt/logs/gunicorn.pid` + endscript +} \ No newline at end of file diff --git a/conf/nginx-local.conf b/conf/nginx-local.conf new file mode 100644 index 000000000..0545399a0 --- /dev/null +++ b/conf/nginx-local.conf @@ -0,0 +1,18 @@ +include root-base.conf; + +worker_processes 2; + +http { + include http-base.conf; + + server { + include server-base.conf; + + listen 5000 default; + + location /static/ { + # checks for static file, if not found proxy to app + alias /home/jake/Projects/docker/quay/static/; + } + } +} diff --git a/conf/nginx-staging.conf b/conf/nginx-staging.conf new file mode 100644 index 000000000..f8fb03784 --- /dev/null +++ b/conf/nginx-staging.conf @@ -0,0 +1,30 @@ +include root-base.conf; + +worker_processes 2; + +user root nogroup; + +http { + include http-base.conf; + + include hosted-http-base.conf; + + server { + include server-base.conf; + + listen 443 default; + + ssl on; + ssl_certificate ./certs/quay-staging-unified.cert; + ssl_certificate_key ./certs/quay-staging.key; + ssl_session_timeout 5m; + ssl_protocols SSLv3 TLSv1; + ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; + ssl_prefer_server_ciphers on; + + location /static/ { + # checks for static file, if not found proxy to app + alias /root/quay/static/; + } + } +} diff --git a/conf/nginx.conf b/conf/nginx.conf new file mode 100644 index 000000000..896b151de --- /dev/null +++ b/conf/nginx.conf @@ -0,0 +1,30 @@ +include root-base.conf; + +worker_processes 8; + +user nobody nogroup; + +http { + include http-base.conf; + + include hosted-http-base.conf; + + server { + include server-base.conf; + + listen 443 default; + + ssl on; + ssl_certificate ./certs/quay-unified.cert; + ssl_certificate_key ./certs/quay.key; + ssl_session_timeout 5m; + ssl_protocols SSLv3 TLSv1; + ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; + ssl_prefer_server_ciphers on; + + location /static/ { + # checks for static file, if not found proxy to app + alias /home/ubuntu/quay/static/; + } + } +} diff --git a/conf/root-base.conf b/conf/root-base.conf new file mode 100644 index 000000000..16a63fda0 --- /dev/null +++ b/conf/root-base.conf @@ -0,0 +1,7 @@ +pid /mnt/logs/nginx.pid; +error_log /mnt/logs/nginx.error.log; + +events { + worker_connections 1024; + accept_mutex off; +} \ No newline at end of file diff --git a/conf/server-base.conf b/conf/server-base.conf new file mode 100644 index 000000000..f37d83bba --- /dev/null +++ b/conf/server-base.conf @@ -0,0 +1,24 @@ +client_max_body_size 8G; +client_body_temp_path /mnt/logs/client_body 1 2; +server_name _; + +keepalive_timeout 5; + +if ($args ~ "_escaped_fragment_") { + rewrite ^ /snapshot$uri; +} + +location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_buffering off; + + proxy_request_buffering off; + proxy_set_header Transfer-Encoding $http_transfer_encoding; + + proxy_pass http://app_server; + proxy_read_timeout 2000; + proxy_temp_path /mnt/logs/proxy_temp 1 2; +} \ No newline at end of file diff --git a/config.py b/config.py index cf56eb1c8..55492bab1 100644 --- a/config.py +++ b/config.py @@ -1,5 +1,6 @@ import logging -import sys +import os +import logstash_formatter from peewee import MySQLDatabase, SqliteDatabase from storage.s3 import S3Storage @@ -11,14 +12,14 @@ from test.teststorage import FakeStorage, FakeUserfiles from test import analytics as fake_analytics -LOG_FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - ' + \ - '%(funcName)s - %(message)s' - - class FlaskConfig(object): SECRET_KEY = '1cb18882-6d12-440d-a4cc-b7430fb5f884' +class FlaskProdConfig(FlaskConfig): + SESSION_COOKIE_SECURE = True + + class MailConfig(object): MAIL_SERVER = 'email-smtp.us-east-1.amazonaws.com' MAIL_USE_TLS = True @@ -134,12 +135,26 @@ class BuildNodeConfig(object): BUILD_NODE_PULL_TOKEN = 'F02O2E86CQLKZUQ0O81J8XDHQ6F0N1V36L9JTOEEK6GKKMT1GI8PTJQT4OU88Y6G' +def logs_init_builder(level=logging.DEBUG, logfile=None): + @staticmethod + def init_logs(): + if logfile: + handler = logging.FileHandler(logfile) + else: + handler = logging.StreamHandler() + + root_logger = logging.getLogger('') + root_logger.setLevel(level) + formatter = logstash_formatter.LogstashFormatter() + handler.setFormatter(formatter) + root_logger.addHandler(handler) + + return init_logs + + class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles, FakeAnalytics, StripeTestConfig): - LOGGING_CONFIG = { - 'level': logging.WARN, - 'format': LOG_FORMAT - } + LOGGING_CONFIG = logs_init_builder(logging.WARN) POPULATE_DB_TEST_DATA = True TESTING = True INCLUDE_TEST_ENDPOINTS = True @@ -148,10 +163,7 @@ class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles, class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB, StripeTestConfig, MixpanelTestConfig, GitHubTestConfig, DigitalOceanConfig, BuildNodeConfig, S3Userfiles): - LOGGING_CONFIG = { - 'level': logging.DEBUG, - 'format': LOG_FORMAT - } + LOGGING_CONFIG = logs_init_builder() SEND_FILE_MAX_AGE_DEFAULT = 0 POPULATE_DB_TEST_DATA = True INCLUDE_TEST_ENDPOINTS = True @@ -161,21 +173,14 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelTestConfig, GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig, S3Userfiles): - LOGGING_CONFIG = { - 'level': logging.DEBUG, - 'format': LOG_FORMAT - } + LOGGING_CONFIG = logs_init_builder() SEND_FILE_MAX_AGE_DEFAULT = 0 -class ProductionConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, +class ProductionConfig(FlaskProdConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelProdConfig, GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig, S3Userfiles): - LOGGING_CONFIG = { - 'stream': sys.stderr, - 'level': logging.DEBUG, - 'format': LOG_FORMAT, - 'filename': 'application.log', - } + + LOGGING_CONFIG = logs_init_builder(logfile='/mnt/logs/application.log') SEND_FILE_MAX_AGE_DEFAULT = 0 diff --git a/data/database.py b/data/database.py index 2afd3f79c..d2319773b 100644 --- a/data/database.py +++ b/data/database.py @@ -12,16 +12,6 @@ logger = logging.getLogger(__name__) db = app.config['DB_DRIVER'](app.config['DB_NAME'], **app.config['DB_CONNECTION_ARGS']) - -def close_db(exc): - if not db.is_closed(): - logger.debug('Disconnecting from database.') - db.close() - - -app.teardown_request(close_db) - - def random_string_generator(length=16): def random_string(): random = SystemRandom() diff --git a/endpoints/api.py b/endpoints/api.py index 21054b38b..c3f9cf0ad 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -37,6 +37,7 @@ route_data = None api = Blueprint('api', __name__) + @api.before_request def csrf_protect(): if request.method != "GET" and request.method != "HEAD": @@ -45,7 +46,19 @@ def csrf_protect(): # TODO: add if not token here, once we are sure all sessions have a token. if token != found_token: - abort(403) + msg = 'CSRF Failure. Session token was %s and request token was %s' + logger.error(msg, token, found_token) + + if not token: + logger.warning('No CSRF token in session.') + + +def request_error(exception=None, **kwargs): + data = kwargs.copy() + if exception: + data['message'] = exception.message + + return make_response(jsonify(data), 400) def get_route_data(): @@ -132,7 +145,7 @@ def discovery(): @api.route('/') @internal_api_call def welcome(): - return make_response('welcome', 200) + return jsonify({'version': '0.5'}) @api.route('/plans/') @@ -222,20 +235,14 @@ def convert_user_to_organization(): # Ensure that the new admin user is the not user being converted. admin_username = convert_data['adminUser'] if admin_username == user.username: - error_resp = jsonify({ - 'reason': 'invaliduser' - }) - error_resp.status_code = 400 - return error_resp + return request_error(reason='invaliduser', + message='The admin user is not valid') # Ensure that the sign in credentials work. admin_password = convert_data['adminPassword'] if not model.verify_user(admin_username, admin_password): - error_resp = jsonify({ - 'reason': 'invaliduser' - }) - error_resp.status_code = 400 - return error_resp + return request_error(reason='invaliduser', + message='The admin user credentials are not valid') # Subscribe the organization to the new plan. plan = convert_data['plan'] @@ -271,22 +278,15 @@ def change_user_details(): new_email = user_data['email'] if model.find_user_by_email(new_email): # Email already used. - error_resp = jsonify({ - 'message': 'E-mail address already used' - }) - error_resp.status_code = 400 - return error_resp + return request_error(message='E-mail address already used') - logger.debug('Sending email to change email address for user: %s', user.username) + logger.debug('Sending email to change email address for user: %s', + user.username) code = model.create_confirm_email_code(user, new_email=new_email) send_change_email(user.username, user_data['email'], code.code) except model.InvalidPasswordException, ex: - error_resp = jsonify({ - 'message': ex.message, - }) - error_resp.status_code = 400 - return error_resp + return request_error(exception=ex) return jsonify(user_view(user)) @@ -298,11 +298,7 @@ def create_new_user(): existing_user = model.get_user(user_data['username']) if existing_user: - error_resp = jsonify({ - 'message': 'The username already exists' - }) - error_resp.status_code = 400 - return error_resp + return request_error(message='The username already exists') try: new_user = model.create_user(user_data['username'], user_data['password'], @@ -311,11 +307,7 @@ def create_new_user(): send_confirmation_email(new_user.username, new_user.email, code.code) return make_response('Created', 201) except model.DataModelException as ex: - error_resp = jsonify({ - 'message': ex.message, - }) - error_resp.status_code = 400 - return error_resp + return request_error(exception=ex) @api.route('/signin', methods=['POST']) @@ -336,7 +328,7 @@ def conduct_signin(username_or_email, password): verified = model.verify_user(username_or_email, password) if verified: if common_login(verified): - return make_response('Success', 200) + return jsonify({'success': True}) else: needs_email_verification = True @@ -357,7 +349,7 @@ def conduct_signin(username_or_email, password): def logout(): logout_user() identity_changed.send(app, identity=AnonymousIdentity()) - return make_response('Success', 200) + return jsonify({'success': True}) @api.route("/recovery", methods=['POST']) @@ -459,22 +451,15 @@ def create_organization(): pass if existing: - error_resp = jsonify({ - 'message': 'A user or organization with this name already exists' - }) - error_resp.status_code = 400 - return error_resp + msg = 'A user or organization with this name already exists' + return request_error(message=msg) try: model.create_organization(org_data['name'], org_data['email'], current_user.db_user()) return make_response('Created', 201) except model.DataModelException as ex: - error_resp = jsonify({ - 'message': ex.message, - }) - error_resp.status_code = 400 - return error_resp + return request_error(exception=ex) def org_view(o, teams): @@ -529,12 +514,7 @@ def change_organization_details(orgname): if 'email' in org_data and org_data['email'] != org.email: new_email = org_data['email'] if model.find_user_by_email(new_email): - # Email already used. - error_resp = jsonify({ - 'message': 'E-mail address already used' - }) - error_resp.status_code = 400 - return error_resp + return request_error(message='E-mail address already used') logger.debug('Changing email address for organization: %s', org.username) model.update_email(org, new_email) @@ -568,7 +548,7 @@ def prototype_view(proto, org_members): 'id': proto.uuid, } -@api.route('/api/organization//prototypes', methods=['GET']) +@api.route('/organization//prototypes', methods=['GET']) @api_login_required def get_organization_prototype_permissions(orgname): permission = AdministerOrganizationPermission(orgname) @@ -606,7 +586,7 @@ def log_prototype_action(action_kind, orgname, prototype, **kwargs): log_action(action_kind, orgname, log_params) -@api.route('/api/organization//prototypes', methods=['POST']) +@api.route('/organization//prototypes', methods=['POST']) @api_login_required def create_organization_prototype_permission(orgname): permission = AdministerOrganizationPermission(orgname) @@ -619,7 +599,8 @@ def create_organization_prototype_permission(orgname): details = request.get_json() activating_username = None - if 'activating_user' in details and details['activating_user'] and 'name' in details['activating_user']: + if ('activating_user' in details and details['activating_user'] and + 'name' in details['activating_user']): activating_username = details['activating_user']['name'] delegate = details['delegate'] @@ -637,10 +618,10 @@ def create_organization_prototype_permission(orgname): if delegate_teamname else None) if activating_username and not activating_user: - abort(404) + return request_error(message='Unknown activating user') if not delegate_user and not delegate_team: - abort(400) + return request_error(message='Missing delagate user or team') role_name = details['role'] @@ -653,7 +634,7 @@ def create_organization_prototype_permission(orgname): abort(403) -@api.route('/api/organization//prototypes/', +@api.route('/organization//prototypes/', methods=['DELETE']) @api_login_required def delete_organization_prototype_permission(orgname, prototypeid): @@ -675,7 +656,7 @@ def delete_organization_prototype_permission(orgname, prototypeid): abort(403) -@api.route('/api/organization//prototypes/', +@api.route('/organization//prototypes/', methods=['PUT']) @api_login_required def update_organization_prototype_permission(orgname, prototypeid): @@ -898,7 +879,7 @@ def update_organization_team_member(orgname, teamname, membername): # Find the user. user = model.get_user(membername) if not user: - abort(400) + return request_error(message='Unknown user') # Add the user to the team. model.add_user_to_team(user, team) @@ -939,7 +920,7 @@ def create_repo(): existing = model.get_repository(namespace_name, repository_name) if existing: - return make_response('Repository already exists', 400) + return request_error(message='Repository already exists') visibility = req['visibility'] @@ -1012,7 +993,7 @@ def list_repos(): if page: try: page = int(page) - except: + except Exception: page = None username = None @@ -1161,12 +1142,13 @@ def get_repo(namespace, repository): @api.route('/repository//build/', methods=['GET']) -@api_login_required @parse_repository_name def get_repo_builds(namespace, repository): - permission = ModifyRepositoryPermission(namespace, repository) - if permission.can(): + permission = ReadRepositoryPermission(namespace, repository) + is_public = model.repository_is_public(namespace, repository) + if permission.can() or is_public: def build_view(build_obj): + # TODO(jake): Filter these logs if the current user can only *read* the repo. if build_obj.status_url: # Delegate the status to the build node node_status = requests.get(build_obj.status_url).json() @@ -1190,7 +1172,7 @@ def get_repo_builds(namespace, repository): 'builds': [build_view(build) for build in builds] }) - abort(403) # Permissions denied + abort(403) # Permission denied @api.route('/repository//build/', methods=['POST']) @@ -1241,7 +1223,8 @@ def create_webhook(namespace, repository): webhook = model.create_webhook(repo, request.get_json()) resp = jsonify(webhook_view(webhook)) repo_string = '%s/%s' % (namespace, repository) - resp.headers['Location'] = url_for('get_webhook', repository=repo_string, + resp.headers['Location'] = url_for('api.get_webhook', + repository=repo_string, public_id=webhook.public_id) log_action('add_repo_webhook', namespace, {'repo': repository, 'webhook_id': webhook.public_id}, @@ -1378,7 +1361,7 @@ def get_image_changes(namespace, repository, image_id): abort(403) -@api.route('/api/repository//tag/', +@api.route('/repository//tag/', methods=['DELETE']) @parse_repository_name def delete_full_tag(namespace, repository, tag): @@ -1542,11 +1525,7 @@ def change_user_permissions(namespace, repository, username): # This repository is not part of an organization pass except model.DataModelException as ex: - error_resp = jsonify({ - 'message': ex.message, - }) - error_resp.status_code = 400 - return error_resp + return request_error(exception=ex) log_action('change_repo_permission', namespace, {'username': username, 'repo': repository, @@ -1599,11 +1578,7 @@ def delete_user_permissions(namespace, repository, username): try: model.delete_user_permission(username, namespace, repository) except model.DataModelException as ex: - error_resp = jsonify({ - 'message': ex.message, - }) - error_resp.status_code = 400 - return error_resp + return request_error(exception=ex) log_action('delete_repo_permission', namespace, {'username': username, 'repo': repository}, @@ -1859,7 +1834,7 @@ def subscribe(user, plan, token, require_business_plan): plan_found['price'] == 0): logger.warning('Business attempting to subscribe to personal plan: %s', user.username) - abort(400) + return request_error(message='No matching plan found') private_repos = model.get_private_repo_count(user.username) @@ -2089,7 +2064,7 @@ def delete_user_robot(robot_shortname): parent = current_user.db_user() model.delete_robot(format_robot_username(parent.username, robot_shortname)) log_action('delete_robot', parent.username, {'robot': robot_shortname}) - return make_response('No Content', 204) + return make_response('Deleted', 204) @api.route('/organization//robots/', @@ -2101,7 +2076,7 @@ def delete_org_robot(orgname, robot_shortname): if permission.can(): model.delete_robot(format_robot_username(orgname, robot_shortname)) log_action('delete_robot', orgname, {'robot': robot_shortname}) - return make_response('No Content', 204) + return make_response('Deleted', 204) abort(403) diff --git a/endpoints/common.py b/endpoints/common.py index 2648f5e9c..ec4727edb 100644 --- a/endpoints/common.py +++ b/endpoints/common.py @@ -2,7 +2,7 @@ import logging import os import base64 -from flask import request, abort, session +from flask import request, abort, session, make_response from flask.ext.login import login_user, UserMixin from flask.ext.principal import identity_changed diff --git a/endpoints/index.py b/endpoints/index.py index b3896bf91..998a0f94d 100644 --- a/endpoints/index.py +++ b/endpoints/index.py @@ -2,12 +2,12 @@ import json import logging import urlparse -from flask import request, make_response, jsonify, abort, session, Blueprint +from flask import request, make_response, jsonify, session, Blueprint from functools import wraps from data import model from data.queue import webhook_queue -from app import app, mixpanel +from app import mixpanel from auth.auth import (process_auth, get_authenticated_user, get_validated_token) from util.names import parse_repository_name @@ -16,6 +16,9 @@ from auth.permissions import (ModifyRepositoryPermission, UserPermission, ReadRepositoryPermission, CreateRepositoryPermission) +from util.http import abort + + logger = logging.getLogger(__name__) index = Blueprint('index', __name__) @@ -64,14 +67,14 @@ def create_user(): model.load_token_data(password) return make_response('Verified', 201) except model.InvalidTokenException: - return make_response('Invalid access token.', 400) + abort(400, 'Invalid access token.', issue='invalid-access-token') elif '+' in username: try: model.verify_robot(username, password) return make_response('Verified', 201) except model.InvalidRobotException: - return make_response('Invalid robot account or password.', 400) + abort(400, 'Invalid robot account or password.', issue='robot-login-failure') existing_user = model.get_user(username) if existing_user: @@ -79,7 +82,8 @@ def create_user(): if verified: return make_response('Verified', 201) else: - return make_response('Invalid password.', 400) + abort(400, 'Invalid password.', issue='login-failure') + else: # New user case new_user = model.create_user(username, password, user_data['email']) @@ -131,23 +135,30 @@ def update_user(username): @generate_headers(role='write') def create_repository(namespace, repository): image_descriptions = json.loads(request.data) - repo = model.get_repository(namespace, repository) if not repo and get_authenticated_user() is None: logger.debug('Attempt to create new repository without user auth.') - abort(401) + abort(401, + message='Cannot create a repository as a guest. Please login via "docker login" first.', + issue='no-login') elif repo: permission = ModifyRepositoryPermission(namespace, repository) if not permission.can(): - abort(403) + abort(403, + message='You do not have permission to modify repository %(namespace)s/%(repository)s', + issue='no-repo-write-permission', + namespace=namespace, repository=repository) else: permission = CreateRepositoryPermission(namespace) if not permission.can(): logger.info('Attempt to create a new repo with insufficient perms.') - abort(403) + abort(403, + message='You do not have permission to create repositories in namespace "%(namespace)s"', + issue='no-create-permission', + namespace=namespace) logger.debug('Creaing repository with owner: %s' % get_authenticated_user().username) @@ -200,7 +211,7 @@ def update_images(namespace, repository): repo = model.get_repository(namespace, repository) if not repo: # Make sure the repo actually exists. - abort(404) + abort(404, message='Unknown repository', issue='unknown-repo') image_with_checksums = json.loads(request.data) @@ -248,7 +259,7 @@ def get_repository_images(namespace, repository): # We can't rely on permissions to tell us if a repo exists anymore repo = model.get_repository(namespace, repository) if not repo: - abort(404) + abort(404, message='Unknown repository', issue='unknown-repo') all_images = [] for image in model.get_repository_images(namespace, repository): @@ -296,18 +307,18 @@ def get_repository_images(namespace, repository): @parse_repository_name @generate_headers(role='write') def delete_repository_images(namespace, repository): - return make_response('Not Implemented', 501) + abort(501, 'Not Implemented', issue='not-implemented') @index.route('/repositories//auth', methods=['PUT']) @parse_repository_name def put_repository_auth(namespace, repository): - return make_response('Not Implemented', 501) + abort(501, 'Not Implemented', issue='not-implemented') @index.route('/search', methods=['GET']) def get_search(): - return make_response('Not Implemented', 501) + abort(501, 'Not Implemented', issue='not-implemented') @index.route('/_ping') diff --git a/endpoints/registry.py b/endpoints/registry.py index 57895aebe..1446238ca 100644 --- a/endpoints/registry.py +++ b/endpoints/registry.py @@ -1,8 +1,8 @@ import logging import json -from flask import (make_response, request, session, Response, abort, - redirect, Blueprint) +from flask import (make_response, request, session, Response, redirect, + Blueprint, abort as flask_abort) from functools import wraps from datetime import datetime from time import time @@ -12,6 +12,7 @@ from data.queue import image_diff_queue from app import app from auth.auth import process_auth, extract_namespace_repo_from_session from util import checksums, changes +from util.http import abort from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission) from data import model @@ -45,8 +46,9 @@ def require_completion(f): def wrapper(namespace, repository, *args, **kwargs): if store.exists(store.image_mark_path(namespace, repository, kwargs['image_id'])): - logger.warning('Image is already being uploaded: %s', kwargs['image_id']) - abort(400) # 'Image is being uploaded, retry later') + abort(400, 'Image %(image_id)s is being uploaded, retry later', + issue='upload-in-progress', image_id=kwargs['image_id']) + return f(namespace, repository, *args, **kwargs) return wrapper @@ -90,9 +92,8 @@ def get_image_layer(namespace, repository, image_id, headers): try: return Response(store.stream_read(path), headers=headers) except IOError: - logger.warning('Image not found: %s', image_id) - abort(404) # 'Image not found', 404) - + abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id) + abort(403) @@ -108,16 +109,20 @@ def put_image_layer(namespace, repository, image_id): json_data = store.get_content(store.image_json_path(namespace, repository, image_id)) except IOError: - abort(404) # 'Image not found', 404) + abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id) + layer_path = store.image_layer_path(namespace, repository, image_id) mark_path = store.image_mark_path(namespace, repository, image_id) + if store.exists(layer_path) and not store.exists(mark_path): - abort(409) # 'Image already exists', 409) + abort(409, 'Image already exists', issue='image-exists', image_id=image_id) + input_stream = request.stream if request.headers.get('transfer-encoding') == 'chunked': # Careful, might work only with WSGI servers supporting chunked # encoding (Gunicorn) input_stream = request.environ['wsgi.input'] + # compute checksums csums = [] sr = SocketReader(input_stream) @@ -127,6 +132,7 @@ def put_image_layer(namespace, repository, image_id): sr.add_handler(sum_hndlr) store.stream_write(layer_path, sr) csums.append('sha256:{0}'.format(h.hexdigest())) + try: image_size = tmp.tell() @@ -139,6 +145,7 @@ def put_image_layer(namespace, repository, image_id): except (IOError, checksums.TarError) as e: logger.debug('put_image_layer: Error when computing tarsum ' '{0}'.format(e)) + try: checksum = store.get_content(store.image_checksum_path(namespace, repository, @@ -148,10 +155,13 @@ def put_image_layer(namespace, repository, image_id): # Not removing the mark though, image is not downloadable yet. session['checksum'] = csums return make_response('true', 200) + # We check if the checksums provided matches one the one we computed if checksum not in csums: logger.warning('put_image_layer: Wrong checksum') - abort(400) # 'Checksum mismatch, ignoring the layer') + abort(400, 'Checksum mismatch; ignoring the layer for image %(image_id)s', + issue='checksum-mismatch', image_id=image_id) + # Checksum is ok, we remove the marker store.remove(mark_path) @@ -177,24 +187,31 @@ def put_image_checksum(namespace, repository, image_id): checksum = request.headers.get('X-Docker-Checksum') if not checksum: - logger.warning('Missing Image\'s checksum: %s', image_id) - abort(400) # 'Missing Image\'s checksum') + abort(400, "Missing checksum for image %(image_id)s", issue='missing-checksum', image_id=image_id) + if not session.get('checksum'): - logger.warning('Checksum not found in Cookie for image: %s', image_id) - abort(400) # 'Checksum not found in Cookie') + abort(400, 'Checksum not found in Cookie for image %(imaage_id)s', + issue='missing-checksum-cookie', image_id=image_id) + if not store.exists(store.image_json_path(namespace, repository, image_id)): - abort(404) # 'Image not found', 404) + abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id) + mark_path = store.image_mark_path(namespace, repository, image_id) if not store.exists(mark_path): - abort(409) # 'Cannot set this image checksum', 409) + abort(409, 'Cannot set checksum for image %(image_id)s', + issue='image-write-error', image_id=image_id) + err = store_checksum(namespace, repository, image_id, checksum) if err: - abort(err) + abort(400, err) + if checksum not in session.get('checksum', []): logger.debug('session checksums: %s' % session.get('checksum', [])) logger.debug('client supplied checksum: %s' % checksum) logger.debug('put_image_layer: Wrong checksum') - abort(400) # 'Checksum mismatch') + abort(400, 'Checksum mismatch for image: %(image_id)s', + issue='checksum-mismatch', image_id=image_id) + # Checksum is ok, we remove the marker store.remove(mark_path) @@ -225,16 +242,19 @@ def get_image_json(namespace, repository, image_id, headers): data = store.get_content(store.image_json_path(namespace, repository, image_id)) except IOError: - abort(404) # 'Image not found', 404) + flask_abort(404) + try: size = store.get_size(store.image_layer_path(namespace, repository, image_id)) headers['X-Docker-Size'] = str(size) except OSError: pass + checksum_path = store.image_checksum_path(namespace, repository, image_id) if store.exists(checksum_path): headers['X-Docker-Checksum'] = store.get_content(checksum_path) + response = make_response(data, 200) response.headers.extend(headers) return response @@ -255,7 +275,8 @@ def get_image_ancestry(namespace, repository, image_id, headers): data = store.get_content(store.image_ancestry_path(namespace, repository, image_id)) except IOError: - abort(404) # 'Image not found', 404) + abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id) + response = make_response(json.dumps(json.loads(data)), 200) response.headers.extend(headers) return response @@ -280,6 +301,7 @@ def store_checksum(namespace, repository, image_id, checksum): checksum_parts = checksum.split(':') if len(checksum_parts) != 2: return 'Invalid checksum format' + # We store the checksum checksum_path = store.image_checksum_path(namespace, repository, image_id) store.put_content(checksum_path, checksum) @@ -298,36 +320,39 @@ def put_image_json(namespace, repository, image_id): except json.JSONDecodeError: pass if not data or not isinstance(data, dict): - logger.warning('Invalid JSON for image: %s json: %s', image_id, - request.data) - abort(400) # 'Invalid JSON') + abort(400, 'Invalid JSON for image: %(image_id)s\nJSON: %(json)s', + issue='invalid-request', image_id=image_id, json=request.data) + if 'id' not in data: - logger.warning('Missing key `id\' in JSON for image: %s', image_id) - abort(400) # 'Missing key `id\' in JSON') + abort(400, 'Missing key `id` in JSON for image: %(image_id)s', + issue='invalid-request', image_id=image_id) + # Read the checksum checksum = request.headers.get('X-Docker-Checksum') if checksum: # Storing the checksum is optional at this stage err = store_checksum(namespace, repository, image_id, checksum) if err: - abort(err) + abort(400, err, issue='write-error') + else: # We cleanup any old checksum in case it's a retry after a fail store.remove(store.image_checksum_path(namespace, repository, image_id)) if image_id != data['id']: - logger.warning('JSON data contains invalid id for image: %s', image_id) - abort(400) # 'JSON data contains invalid id') + abort(400, 'JSON data contains invalid id for image: %(image_id)s', + issue='invalid-request', image_id=image_id) + parent_id = data.get('parent') - if parent_id and not store.exists(store.image_json_path(namespace, - repository, - data['parent'])): - logger.warning('Image depends on a non existing parent image: %s', - image_id) - abort(400) # 'Image depends on a non existing parent') + if (parent_id and not + store.exists(store.image_json_path(namespace, repository, parent_id))): + abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s', + issue='invalid-request', image_id=image_id, parent_id=parent_id) + json_path = store.image_json_path(namespace, repository, image_id) mark_path = store.image_mark_path(namespace, repository, image_id) if store.exists(json_path) and not store.exists(mark_path): - abort(409) # 'Image already exists', 409) + abort(409, 'Image already exists', issue='image-exists', image_id=image_id) + # If we reach that point, it means that this is a new image or a retry # on a failed push # save the metadata diff --git a/endpoints/web.py b/endpoints/web.py index 57d15164a..9550266c9 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -242,7 +242,7 @@ def github_oauth_callback(): return render_page_template('githuberror.html', error_message=ex.message) if common_login(to_login): - return redirect(url_for('index')) + return redirect(url_for('web.index')) return render_page_template('githuberror.html') @@ -255,7 +255,7 @@ def github_oauth_attach(): github_id = user_data['id'] user_obj = current_user.db_user() model.attach_federated_login(user_obj, 'github', github_id) - return redirect(url_for('user')) + return redirect(url_for('web.user')) @web.route('/confirm', methods=['GET']) @@ -271,7 +271,8 @@ def confirm_email(): common_login(user) - return redirect(url_for('user', tab='email') if new_email else url_for('index')) + return redirect(url_for('web.user', tab='email') + if new_email else url_for('web.index')) @web.route('/recovery', methods=['GET']) @@ -281,6 +282,6 @@ def confirm_recovery(): if user: common_login(user) - return redirect(url_for('user')) + return redirect(url_for('web.user')) else: abort(403) diff --git a/gunicorn_config.py b/gunicorn_config.py index 59141b06d..88055c1fa 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -3,3 +3,4 @@ workers = 8 worker_class = 'gevent' timeout = 2000 daemon = True +pidfile = '/mnt/logs/gunicorn.pid' \ No newline at end of file diff --git a/initdb.py b/initdb.py index a3ee26362..0cdb5f91a 100644 --- a/initdb.py +++ b/initdb.py @@ -4,7 +4,8 @@ import hashlib import random from datetime import datetime, timedelta -from peewee import SqliteDatabase, create_model_tables, drop_model_tables +from peewee import (SqliteDatabase, create_model_tables, drop_model_tables, + savepoint_sqlite) from data.database import * from data import model @@ -29,7 +30,6 @@ SAMPLE_CMDS = [["/bin/bash"], REFERENCE_DATE = datetime(2013, 6, 23) TEST_STRIPE_ID = 'cus_2tmnh3PkXQS8NG' - def __gen_checksum(image_id): h = hashlib.md5(image_id) return 'tarsum+sha256:' + h.hexdigest() + h.hexdigest() @@ -113,6 +113,44 @@ def __generate_repository(user, name, description, is_public, permissions, return repo +db_initialized_for_testing = False +testcases = {} + +def finished_database_for_testing(testcase): + """ Called when a testcase has finished using the database, indicating that + any changes should be discarded. + """ + global testcases + testcases[testcase]['savepoint'].__exit__(True, None, None) + +def setup_database_for_testing(testcase): + """ Called when a testcase has started using the database, indicating that + the database should be setup (if not already) and a savepoint created. + """ + + # Sanity check to make sure we're not killing our prod db + db = model.db + if (not isinstance(model.db, SqliteDatabase) or + app.config['DB_DRIVER'] is not SqliteDatabase): + raise RuntimeError('Attempted to wipe production database!') + + global db_initialized_for_testing + if not db_initialized_for_testing: + logger.debug('Setting up DB for testing.') + + # Setup the database. + wipe_database() + initialize_database() + populate_database() + + db_initialized_for_testing = True + + # Create a savepoint for the testcase. + global testcases + testcases[testcase] = {} + testcases[testcase]['savepoint'] = savepoint_sqlite(db) + testcases[testcase]['savepoint'].__enter__() + def initialize_database(): create_model_tables(all_models) @@ -350,7 +388,7 @@ def populate_database(): metadata={'token_code': 'somecode', 'repo': 'orgrepo'}) if __name__ == '__main__': - logging.basicConfig(**app.config['LOGGING_CONFIG']) + app.config['LOGGING_CONFIG']() initialize_database() if app.config.get('POPULATE_DB_TEST_DATA', False): diff --git a/nginx-staging.conf b/nginx-staging.conf deleted file mode 100644 index b29bda2c5..000000000 --- a/nginx-staging.conf +++ /dev/null @@ -1,83 +0,0 @@ -worker_processes 2; - -user root nogroup; -pid /mnt/nginx/nginx.pid; -error_log /mnt/nginx/nginx.error.log; - -events { - worker_connections 1024; - accept_mutex off; -} - -http { - types_hash_max_size 2048; - include /usr/local/nginx/conf/mime.types.default; - - default_type application/octet-stream; - access_log /mnt/nginx/nginx.access.log combined; - sendfile on; - - root /root/quay/; - - gzip on; - gzip_http_version 1.0; - gzip_proxied any; - gzip_min_length 500; - gzip_disable "MSIE [1-6]\."; - gzip_types text/plain text/xml text/css - text/javascript application/x-javascript - application/octet-stream; - - upstream app_server { - server unix:/tmp/gunicorn.sock fail_timeout=0; - # For a TCP configuration: - # server 192.168.0.7:8000 fail_timeout=0; - } - - server { - listen 80 default_server; - server_name _; - rewrite ^ https://$host$request_uri? permanent; - } - - server { - listen 443 default; - client_max_body_size 8G; - client_body_temp_path /mnt/nginx/client_body 1 2; - server_name _; - - keepalive_timeout 5; - - ssl on; - ssl_certificate ./certs/quay-staging-unified.cert; - ssl_certificate_key ./certs/quay-staging.key; - ssl_session_timeout 5m; - ssl_protocols SSLv3 TLSv1; - ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; - ssl_prefer_server_ciphers on; - - if ($args ~ "_escaped_fragment_") { - rewrite ^ /snapshot$uri; - } - - location /static/ { - # checks for static file, if not found proxy to app - alias /root/quay/static/; - } - - location / { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; - proxy_redirect off; - proxy_buffering off; - - proxy_request_buffering off; - proxy_set_header Transfer-Encoding $http_transfer_encoding; - - proxy_pass http://app_server; - proxy_read_timeout 2000; - proxy_temp_path /mnt/nginx/proxy_temp 1 2; - } - } -} diff --git a/nginx.conf b/nginx.conf deleted file mode 100644 index 53cd3c9f8..000000000 --- a/nginx.conf +++ /dev/null @@ -1,81 +0,0 @@ -worker_processes 8; - -user nobody nogroup; -pid /mnt/nginx/nginx.pid; -error_log /mnt/nginx/nginx.error.log; - -events { - worker_connections 1024; - accept_mutex off; -} - -http { - types_hash_max_size 2048; - include /usr/local/nginx/conf/mime.types.default; - - default_type application/octet-stream; - access_log /mnt/nginx/nginx.access.log combined; - sendfile on; - - gzip on; - gzip_http_version 1.0; - gzip_proxied any; - gzip_min_length 500; - gzip_disable "MSIE [1-6]\."; - gzip_types text/plain text/xml text/css - text/javascript application/x-javascript - application/octet-stream; - - upstream app_server { - server unix:/tmp/gunicorn.sock fail_timeout=0; - # For a TCP configuration: - # server 192.168.0.7:8000 fail_timeout=0; - } - - server { - listen 80 default_server; - server_name _; - rewrite ^ https://$host$request_uri? permanent; - } - - server { - listen 443 default; - client_max_body_size 8G; - client_body_temp_path /mnt/nginx/client_body 1 2; - server_name _; - - keepalive_timeout 5; - - ssl on; - ssl_certificate ./certs/quay-unified.cert; - ssl_certificate_key ./certs/quay.key; - ssl_session_timeout 5m; - ssl_protocols SSLv3 TLSv1; - ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; - ssl_prefer_server_ciphers on; - - if ($args ~ "_escaped_fragment_") { - rewrite ^ /snapshot$uri; - } - - location /static/ { - # checks for static file, if not found proxy to app - alias /home/ubuntu/quay/static/; - } - - location / { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; - proxy_redirect off; - proxy_buffering off; - - proxy_request_buffering off; - proxy_set_header Transfer-Encoding $http_transfer_encoding; - - proxy_pass http://app_server; - proxy_read_timeout 2000; - proxy_temp_path /mnt/nginx/proxy_temp 1 2; - } - } -} \ No newline at end of file diff --git a/requirements-nover.txt b/requirements-nover.txt index f42a8a90d..c430edf5a 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -17,4 +17,5 @@ apscheduler python-daemon paramiko python-digitalocean -xhtml2pdf \ No newline at end of file +xhtml2pdf +logstash_formatter diff --git a/requirements.txt b/requirements.txt index ce9a22c54..7438e6dce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -APScheduler==2.1.1 +APScheduler==2.1.2 Flask==0.10.1 Flask-Login==0.2.9 Flask-Mail==0.9.0 Flask-Principal==0.4.0 -Jinja2==2.7.1 +Jinja2==2.7.2 MarkupSafe==0.18 Pillow==2.3.0 PyMySQL==0.6.1 @@ -11,19 +11,20 @@ Werkzeug==0.9.4 argparse==1.2.1 beautifulsoup4==4.3.2 blinker==1.3 -boto==2.21.2 +boto==2.24.0 distribute==0.6.34 ecdsa==0.10 gevent==1.0 -greenlet==0.4.1 +greenlet==0.4.2 gunicorn==18.0 html5lib==1.0b3 itsdangerous==0.23 lockfile==0.9.1 +logstash-formatter==0.5.8 marisa-trie==0.5.1 -mixpanel-py==3.0.0 -paramiko==1.12.0 -peewee==2.1.7 +mixpanel-py==3.1.1 +paramiko==1.12.1 +peewee==2.2.0 py-bcrypt==0.4 pyPdf==1.13 pycrypto==2.6.1 @@ -31,8 +32,8 @@ python-daemon==1.6 python-dateutil==2.2 python-digitalocean==0.6 reportlab==2.7 -requests==2.1.0 -six==1.4.1 -stripe==1.11.0 +requests==2.2.1 +six==1.5.2 +stripe==1.12.0 wsgiref==0.1.2 xhtml2pdf==0.0.5 diff --git a/static/css/quay.css b/static/css/quay.css index 1fe07fddc..7ee169fdf 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -2474,6 +2474,28 @@ p.editable:hover i { font-size: 16px; } +.repo-breadcrumb-element .crumb { + cursor: pointer; +} + +.repo-breadcrumb-element .crumb:nth-last-of-type(3), .repo-breadcrumb-element .crumb:nth-last-of-type(3) a { + color: #aaa !important; +} + +.repo-breadcrumb-element .crumb:nth-last-of-type(2), .repo-breadcrumb-element .crumb:nth-last-of-type(2) a { + color: #888 !important; +} + +.repo-breadcrumb-element .crumb:after { + content: "/"; + color: #ccc; + margin-left: 4px; +} + +.repo-breadcrumb-element .crumb:hover, .repo-breadcrumb-element .crumb:hover a { + color: #2a6496 !important; + text-decoration: none; +} /* Overrides for typeahead to work with bootstrap 3. */ diff --git a/static/directives/header-bar.html b/static/directives/header-bar.html index 32ee88d23..78159b7ae 100644 --- a/static/directives/header-bar.html +++ b/static/directives/header-bar.html @@ -15,7 +15,7 @@ diff --git a/static/partials/user-admin.html b/static/partials/user-admin.html index 275e0ee74..347b0f21c 100644 --- a/static/partials/user-admin.html +++ b/static/partials/user-admin.html @@ -185,7 +185,11 @@ ng-model="org.adminUser" required autofocus> - The username and password for the account that will become administrator of the organization + + The username and password for the account that will become an administrator of the organization. + Note that this account must be a separate Quay.io account from the account that you are + trying to convert, and must already exist. + diff --git a/static/partials/view-repo.html b/static/partials/view-repo.html index 89091258e..969b59275 100644 --- a/static/partials/view-repo.html +++ b/static/partials/view-repo.html @@ -10,7 +10,7 @@

- {{repo.namespace}} / {{repo.name}} + diff --git a/static/sitemap.xml b/static/sitemap.xml index 5a331996c..db9440376 100644 --- a/static/sitemap.xml +++ b/static/sitemap.xml @@ -17,10 +17,6 @@ https://quay.io/repository/ always - - https://quay.io/guide/ - weekly - https://quay.io/tos monthly diff --git a/test/specs.py b/test/specs.py index 10e3eee13..e0a037655 100644 --- a/test/specs.py +++ b/test/specs.py @@ -103,320 +103,328 @@ class TestSpec(object): def build_specs(): return [ - TestSpec(url_for('welcome'), 200, 200, 200, 200), + TestSpec(url_for('api.welcome'), 200, 200, 200, 200), - TestSpec(url_for('list_plans'), 200, 200, 200, 200), + TestSpec(url_for('api.list_plans'), 200, 200, 200, 200), - TestSpec(url_for('get_logged_in_user'), 200, 200, 200, 200), + TestSpec(url_for('api.get_logged_in_user'), 200, 200, 200, 200), - TestSpec(url_for('change_user_details'), + TestSpec(url_for('api.change_user_details'), 401, 200, 200, 200).set_method('PUT'), - TestSpec(url_for('create_new_user'), 201, 201, 201, + TestSpec(url_for('api.create_new_user'), 201, 201, 201, 201).set_method('POST').set_data_from_obj(NEW_USER_DETAILS), - TestSpec(url_for('signin_user'), 200, 200, 200, + TestSpec(url_for('api.signin_user'), 200, 200, 200, 200).set_method('POST').set_data_from_obj(SIGNIN_DETAILS), - TestSpec(url_for('request_recovery_email'), 201, 201, 201, + TestSpec(url_for('api.request_recovery_email'), 201, 201, 201, 201).set_method('POST').set_data_from_obj(SEND_RECOVERY_DETAILS), - TestSpec(url_for('get_matching_users', prefix='dev'), 401, 200, 200, 200), + TestSpec(url_for('api.get_matching_users', prefix='dev'), + 401, 200, 200, 200), - TestSpec(url_for('get_matching_entities', prefix='dev'), 401, 200, 200, + TestSpec(url_for('api.get_matching_entities', prefix='dev'), 401, 200, 200, 200), - TestSpec(url_for('get_organization', orgname=ORG), 401, 403, 200, 200), + TestSpec(url_for('api.get_organization', orgname=ORG), 401, 403, 200, 200), - TestSpec(url_for('get_organization_private_allowed', orgname=ORG)), + TestSpec(url_for('api.get_organization_private_allowed', orgname=ORG)), - TestSpec(url_for('update_organization_team', orgname=ORG, + TestSpec(url_for('api.update_organization_team', orgname=ORG, teamname=ORG_OWNERS)).set_method('PUT'), - TestSpec(url_for('update_organization_team', orgname=ORG, + TestSpec(url_for('api.update_organization_team', orgname=ORG, teamname=ORG_READERS)).set_method('PUT'), - TestSpec(url_for('delete_organization_team', orgname=ORG, + TestSpec(url_for('api.delete_organization_team', orgname=ORG, teamname=ORG_OWNERS), admin_code=400).set_method('DELETE'), - TestSpec(url_for('delete_organization_team', orgname=ORG, + TestSpec(url_for('api.delete_organization_team', orgname=ORG, teamname=ORG_READERS), admin_code=204).set_method('DELETE'), - TestSpec(url_for('get_organization_team_members', orgname=ORG, + TestSpec(url_for('api.get_organization_team_members', orgname=ORG, teamname=ORG_OWNERS)), - TestSpec(url_for('get_organization_team_members', orgname=ORG, + TestSpec(url_for('api.get_organization_team_members', orgname=ORG, teamname=ORG_READERS), read_code=200), - TestSpec(url_for('update_organization_team_member', orgname=ORG, + TestSpec(url_for('api.update_organization_team_member', orgname=ORG, teamname=ORG_OWNERS, membername=ORG_OWNER), admin_code=400).set_method('PUT'), - TestSpec(url_for('update_organization_team_member', orgname=ORG, + TestSpec(url_for('api.update_organization_team_member', orgname=ORG, teamname=ORG_READERS, membername=ORG_OWNER)).set_method('PUT'), - TestSpec(url_for('delete_organization_team_member', orgname=ORG, + TestSpec(url_for('api.delete_organization_team_member', orgname=ORG, teamname=ORG_OWNERS, membername=ORG_OWNER), admin_code=400).set_method('DELETE'), - TestSpec(url_for('delete_organization_team_member', orgname=ORG, + TestSpec(url_for('api.delete_organization_team_member', orgname=ORG, teamname=ORG_READERS, membername=ORG_OWNER), admin_code=400).set_method('DELETE'), - (TestSpec(url_for('create_repo')) + (TestSpec(url_for('api.create_repo')) .set_method('POST') .set_data_from_obj(NEW_ORG_REPO_DETAILS)), - TestSpec(url_for('find_repos'), 200, 200, 200, 200), + TestSpec(url_for('api.find_repos'), 200, 200, 200, 200), - TestSpec(url_for('list_repos'), 200, 200, 200, 200), + TestSpec(url_for('api.list_repos'), 200, 200, 200, 200), - TestSpec(url_for('update_repo', repository=PUBLIC_REPO), + TestSpec(url_for('api.update_repo', repository=PUBLIC_REPO), admin_code=403).set_method('PUT'), - (TestSpec(url_for('update_repo', repository=ORG_REPO)) + (TestSpec(url_for('api.update_repo', repository=ORG_REPO)) .set_method('PUT') .set_data_from_obj(UPDATE_REPO_DETAILS)), - (TestSpec(url_for('update_repo', repository=PRIVATE_REPO)) + (TestSpec(url_for('api.update_repo', repository=PRIVATE_REPO)) .set_method('PUT') .set_data_from_obj(UPDATE_REPO_DETAILS)), - (TestSpec(url_for('change_repo_visibility', repository=PUBLIC_REPO), + (TestSpec(url_for('api.change_repo_visibility', repository=PUBLIC_REPO), admin_code=403).set_method('POST') .set_data_from_obj(CHANGE_VISIBILITY_DETAILS)), - (TestSpec(url_for('change_repo_visibility', repository=ORG_REPO)) + (TestSpec(url_for('api.change_repo_visibility', repository=ORG_REPO)) .set_method('POST').set_data_from_obj(CHANGE_VISIBILITY_DETAILS)), - (TestSpec(url_for('change_repo_visibility', repository=PRIVATE_REPO)) + (TestSpec(url_for('api.change_repo_visibility', repository=PRIVATE_REPO)) .set_method('POST').set_data_from_obj(CHANGE_VISIBILITY_DETAILS)), - TestSpec(url_for('delete_repository', repository=PUBLIC_REPO), + TestSpec(url_for('api.delete_repository', repository=PUBLIC_REPO), admin_code=403).set_method('DELETE'), - TestSpec(url_for('delete_repository', repository=ORG_REPO), + TestSpec(url_for('api.delete_repository', repository=ORG_REPO), admin_code=204).set_method('DELETE'), - TestSpec(url_for('delete_repository', repository=PRIVATE_REPO), + TestSpec(url_for('api.delete_repository', repository=PRIVATE_REPO), admin_code=204).set_method('DELETE'), - TestSpec(url_for('get_repo', repository=PUBLIC_REPO), + TestSpec(url_for('api.get_repo', repository=PUBLIC_REPO), 200, 200, 200,200), - TestSpec(url_for('get_repo', repository=ORG_REPO), + TestSpec(url_for('api.get_repo', repository=ORG_REPO), 403, 403, 200, 200), - TestSpec(url_for('get_repo', repository=PRIVATE_REPO), + TestSpec(url_for('api.get_repo', repository=PRIVATE_REPO), 403, 403, 200, 200), - TestSpec(url_for('get_repo_builds', repository=PUBLIC_REPO), - admin_code=403), - TestSpec(url_for('get_repo_builds', repository=ORG_REPO)), - TestSpec(url_for('get_repo_builds', repository=PRIVATE_REPO)), + TestSpec(url_for('api.get_repo_builds', repository=PUBLIC_REPO), + 200, 200, 200, 200), + TestSpec(url_for('api.get_repo_builds', repository=ORG_REPO), + 403, 403, 200, 200), + TestSpec(url_for('api.get_repo_builds', repository=PRIVATE_REPO), + 403, 403, 200, 200), - TestSpec(url_for('get_filedrop_url'), 401, 200, 200, + TestSpec(url_for('api.get_filedrop_url'), 401, 200, 200, 200).set_method('POST').set_data_from_obj(FILE_DROP_DETAILS), - (TestSpec(url_for('request_repo_build', repository=PUBLIC_REPO), + (TestSpec(url_for('api.request_repo_build', repository=PUBLIC_REPO), admin_code=403).set_method('POST') .set_data_from_obj(CREATE_BUILD_DETAILS)), - (TestSpec(url_for('request_repo_build', repository=ORG_REPO), + (TestSpec(url_for('api.request_repo_build', repository=ORG_REPO), admin_code=201).set_method('POST') .set_data_from_obj(CREATE_BUILD_DETAILS)), - (TestSpec(url_for('request_repo_build', repository=PRIVATE_REPO), + (TestSpec(url_for('api.request_repo_build', repository=PRIVATE_REPO), admin_code=201).set_method('POST') .set_data_from_obj(CREATE_BUILD_DETAILS)), - TestSpec(url_for('create_webhook', repository=PUBLIC_REPO), + TestSpec(url_for('api.create_webhook', repository=PUBLIC_REPO), admin_code=403).set_method('POST'), - TestSpec(url_for('create_webhook', + TestSpec(url_for('api.create_webhook', repository=ORG_REPO)).set_method('POST'), - TestSpec(url_for('create_webhook', + TestSpec(url_for('api.create_webhook', repository=PRIVATE_REPO)).set_method('POST'), - TestSpec(url_for('get_webhook', repository=PUBLIC_REPO, + TestSpec(url_for('api.get_webhook', repository=PUBLIC_REPO, public_id=FAKE_WEBHOOK), admin_code=403), - TestSpec(url_for('get_webhook', repository=ORG_REPO, + TestSpec(url_for('api.get_webhook', repository=ORG_REPO, public_id=FAKE_WEBHOOK), admin_code=400), - TestSpec(url_for('get_webhook', repository=PRIVATE_REPO, + TestSpec(url_for('api.get_webhook', repository=PRIVATE_REPO, public_id=FAKE_WEBHOOK), admin_code=400), - TestSpec(url_for('list_webhooks', repository=PUBLIC_REPO), admin_code=403), - TestSpec(url_for('list_webhooks', repository=ORG_REPO)), - TestSpec(url_for('list_webhooks', repository=PRIVATE_REPO)), + TestSpec(url_for('api.list_webhooks', repository=PUBLIC_REPO), + admin_code=403), + TestSpec(url_for('api.list_webhooks', repository=ORG_REPO)), + TestSpec(url_for('api.list_webhooks', repository=PRIVATE_REPO)), - TestSpec(url_for('delete_webhook', repository=PUBLIC_REPO, + TestSpec(url_for('api.delete_webhook', repository=PUBLIC_REPO, public_id=FAKE_WEBHOOK), admin_code=403).set_method('DELETE'), - TestSpec(url_for('delete_webhook', repository=ORG_REPO, + TestSpec(url_for('api.delete_webhook', repository=ORG_REPO, public_id=FAKE_WEBHOOK), admin_code=400).set_method('DELETE'), - TestSpec(url_for('delete_webhook', repository=PRIVATE_REPO, + TestSpec(url_for('api.delete_webhook', repository=PRIVATE_REPO, public_id=FAKE_WEBHOOK), admin_code=400).set_method('DELETE'), - TestSpec(url_for('list_repository_images', repository=PUBLIC_REPO), + TestSpec(url_for('api.list_repository_images', repository=PUBLIC_REPO), 200, 200, 200, 200), - TestSpec(url_for('list_repository_images', repository=ORG_REPO), + TestSpec(url_for('api.list_repository_images', repository=ORG_REPO), 403, 403, 200, 200), - TestSpec(url_for('list_repository_images', repository=PRIVATE_REPO), + TestSpec(url_for('api.list_repository_images', repository=PRIVATE_REPO), 403, 403, 200, 200), - TestSpec(url_for('get_image', repository=PUBLIC_REPO, + TestSpec(url_for('api.get_image', repository=PUBLIC_REPO, image_id=FAKE_IMAGE_ID), 404, 404, 404, 404), - TestSpec(url_for('get_image', repository=ORG_REPO, + TestSpec(url_for('api.get_image', repository=ORG_REPO, image_id=FAKE_IMAGE_ID), 403, 403, 404, 404), - TestSpec(url_for('get_image', repository=PRIVATE_REPO, + TestSpec(url_for('api.get_image', repository=PRIVATE_REPO, image_id=FAKE_IMAGE_ID), 403, 403, 404, 404), - TestSpec(url_for('get_image_changes', repository=PUBLIC_REPO, + TestSpec(url_for('api.get_image_changes', repository=PUBLIC_REPO, image_id=FAKE_IMAGE_ID), 404, 404, 404, 404), - TestSpec(url_for('get_image_changes', repository=ORG_REPO, + TestSpec(url_for('api.get_image_changes', repository=ORG_REPO, image_id=FAKE_IMAGE_ID), 403, 403, 404, 404), - TestSpec(url_for('get_image_changes', repository=PRIVATE_REPO, + TestSpec(url_for('api.get_image_changes', repository=PRIVATE_REPO, image_id=FAKE_IMAGE_ID), 403, 403, 404, 404), - TestSpec(url_for('list_tag_images', repository=PUBLIC_REPO, + TestSpec(url_for('api.list_tag_images', repository=PUBLIC_REPO, tag=FAKE_TAG_NAME), 404, 404, 404, 404), - TestSpec(url_for('list_tag_images', repository=ORG_REPO, + TestSpec(url_for('api.list_tag_images', repository=ORG_REPO, tag=FAKE_TAG_NAME), 403, 403, 404, 404), - TestSpec(url_for('list_tag_images', repository=PRIVATE_REPO, + TestSpec(url_for('api.list_tag_images', repository=PRIVATE_REPO, tag=FAKE_TAG_NAME), 403, 403, 404, 404), - TestSpec(url_for('list_repo_team_permissions', repository=PUBLIC_REPO), + TestSpec(url_for('api.list_repo_team_permissions', repository=PUBLIC_REPO), admin_code=403), - TestSpec(url_for('list_repo_team_permissions', repository=ORG_REPO)), - TestSpec(url_for('list_repo_team_permissions', repository=PRIVATE_REPO)), + TestSpec(url_for('api.list_repo_team_permissions', repository=ORG_REPO)), + TestSpec(url_for('api.list_repo_team_permissions', + repository=PRIVATE_REPO)), - TestSpec(url_for('list_repo_user_permissions', repository=PUBLIC_REPO), + TestSpec(url_for('api.list_repo_user_permissions', repository=PUBLIC_REPO), admin_code=403), - TestSpec(url_for('list_repo_user_permissions', repository=ORG_REPO)), - TestSpec(url_for('list_repo_user_permissions', repository=PRIVATE_REPO)), + TestSpec(url_for('api.list_repo_user_permissions', repository=ORG_REPO)), + TestSpec(url_for('api.list_repo_user_permissions', + repository=PRIVATE_REPO)), - TestSpec(url_for('get_user_permissions', repository=PUBLIC_REPO, + TestSpec(url_for('api.get_user_permissions', repository=PUBLIC_REPO, username=FAKE_USERNAME), admin_code=403), - TestSpec(url_for('get_user_permissions', repository=ORG_REPO, + TestSpec(url_for('api.get_user_permissions', repository=ORG_REPO, username=FAKE_USERNAME), admin_code=400), - TestSpec(url_for('get_user_permissions', repository=PRIVATE_REPO, + TestSpec(url_for('api.get_user_permissions', repository=PRIVATE_REPO, username=FAKE_USERNAME), admin_code=400), - TestSpec(url_for('get_team_permissions', repository=PUBLIC_REPO, + TestSpec(url_for('api.get_team_permissions', repository=PUBLIC_REPO, teamname=ORG_OWNERS), admin_code=403), - TestSpec(url_for('get_team_permissions', repository=PUBLIC_REPO, + TestSpec(url_for('api.get_team_permissions', repository=PUBLIC_REPO, teamname=ORG_READERS), admin_code=403), - TestSpec(url_for('get_team_permissions', repository=ORG_REPO, + TestSpec(url_for('api.get_team_permissions', repository=ORG_REPO, teamname=ORG_OWNERS), admin_code=400), - TestSpec(url_for('get_team_permissions', repository=ORG_REPO, + TestSpec(url_for('api.get_team_permissions', repository=ORG_REPO, teamname=ORG_READERS)), - TestSpec(url_for('get_team_permissions', repository=PRIVATE_REPO, + TestSpec(url_for('api.get_team_permissions', repository=PRIVATE_REPO, teamname=ORG_OWNERS), admin_code=400), - TestSpec(url_for('get_team_permissions', repository=PRIVATE_REPO, + TestSpec(url_for('api.get_team_permissions', repository=PRIVATE_REPO, teamname=ORG_READERS), admin_code=400), - TestSpec(url_for('change_user_permissions', repository=PUBLIC_REPO, + TestSpec(url_for('api.change_user_permissions', repository=PUBLIC_REPO, username=FAKE_USERNAME), admin_code=403).set_method('PUT'), - TestSpec(url_for('change_user_permissions', repository=ORG_REPO, + TestSpec(url_for('api.change_user_permissions', repository=ORG_REPO, username=FAKE_USERNAME), admin_code=400).set_method('PUT'), - TestSpec(url_for('change_user_permissions', repository=PRIVATE_REPO, + TestSpec(url_for('api.change_user_permissions', repository=PRIVATE_REPO, username=FAKE_USERNAME), admin_code=400).set_method('PUT'), - (TestSpec(url_for('change_team_permissions', repository=PUBLIC_REPO, + (TestSpec(url_for('api.change_team_permissions', repository=PUBLIC_REPO, teamname=ORG_OWNERS), admin_code=403) .set_method('PUT') .set_data_from_obj(CHANGE_PERMISSION_DETAILS)), - (TestSpec(url_for('change_team_permissions', repository=PUBLIC_REPO, + (TestSpec(url_for('api.change_team_permissions', repository=PUBLIC_REPO, teamname=ORG_READERS), admin_code=403) .set_method('PUT') .set_data_from_obj(CHANGE_PERMISSION_DETAILS)), - (TestSpec(url_for('change_team_permissions', repository=ORG_REPO, + (TestSpec(url_for('api.change_team_permissions', repository=ORG_REPO, teamname=ORG_OWNERS)) .set_method('PUT') .set_data_from_obj(CHANGE_PERMISSION_DETAILS)), - (TestSpec(url_for('change_team_permissions', repository=ORG_REPO, + (TestSpec(url_for('api.change_team_permissions', repository=ORG_REPO, teamname=ORG_READERS)) .set_method('PUT') .set_data_from_obj(CHANGE_PERMISSION_DETAILS)), - (TestSpec(url_for('change_team_permissions', repository=PRIVATE_REPO, + (TestSpec(url_for('api.change_team_permissions', repository=PRIVATE_REPO, teamname=ORG_OWNERS), admin_code=400) .set_method('PUT') .set_data_from_obj(CHANGE_PERMISSION_DETAILS)), - (TestSpec(url_for('change_team_permissions', repository=PRIVATE_REPO, + (TestSpec(url_for('api.change_team_permissions', repository=PRIVATE_REPO, teamname=ORG_READERS), admin_code=400) .set_method('PUT') .set_data_from_obj(CHANGE_PERMISSION_DETAILS)), - TestSpec(url_for('delete_user_permissions', repository=PUBLIC_REPO, + TestSpec(url_for('api.delete_user_permissions', repository=PUBLIC_REPO, username=FAKE_USERNAME), admin_code=403).set_method('DELETE'), - TestSpec(url_for('delete_user_permissions', repository=ORG_REPO, + TestSpec(url_for('api.delete_user_permissions', repository=ORG_REPO, username=FAKE_USERNAME), admin_code=400).set_method('DELETE'), - TestSpec(url_for('delete_user_permissions', repository=PRIVATE_REPO, + TestSpec(url_for('api.delete_user_permissions', repository=PRIVATE_REPO, username=FAKE_USERNAME), admin_code=400).set_method('DELETE'), - TestSpec(url_for('delete_team_permissions', repository=PUBLIC_REPO, + TestSpec(url_for('api.delete_team_permissions', repository=PUBLIC_REPO, teamname=ORG_OWNERS), admin_code=403).set_method('DELETE'), - TestSpec(url_for('delete_team_permissions', repository=PUBLIC_REPO, + TestSpec(url_for('api.delete_team_permissions', repository=PUBLIC_REPO, teamname=ORG_READERS), admin_code=403).set_method('DELETE'), - TestSpec(url_for('delete_team_permissions', repository=ORG_REPO, + TestSpec(url_for('api.delete_team_permissions', repository=ORG_REPO, teamname=ORG_OWNERS), admin_code=400).set_method('DELETE'), - TestSpec(url_for('delete_team_permissions', repository=ORG_REPO, + TestSpec(url_for('api.delete_team_permissions', repository=ORG_REPO, teamname=ORG_READERS), admin_code=204).set_method('DELETE'), - TestSpec(url_for('delete_team_permissions', repository=PRIVATE_REPO, + TestSpec(url_for('api.delete_team_permissions', repository=PRIVATE_REPO, teamname=ORG_OWNERS), admin_code=400).set_method('DELETE'), - TestSpec(url_for('delete_team_permissions', repository=PRIVATE_REPO, + TestSpec(url_for('api.delete_team_permissions', repository=PRIVATE_REPO, teamname=ORG_READERS), admin_code=400).set_method('DELETE'), - TestSpec(url_for('list_repo_tokens', repository=PUBLIC_REPO), + TestSpec(url_for('api.list_repo_tokens', repository=PUBLIC_REPO), admin_code=403), - TestSpec(url_for('list_repo_tokens', repository=ORG_REPO)), - TestSpec(url_for('list_repo_tokens', repository=PRIVATE_REPO)), + TestSpec(url_for('api.list_repo_tokens', repository=ORG_REPO)), + TestSpec(url_for('api.list_repo_tokens', repository=PRIVATE_REPO)), - TestSpec(url_for('get_tokens', repository=PUBLIC_REPO, code=FAKE_TOKEN), - admin_code=403), - TestSpec(url_for('get_tokens', repository=ORG_REPO, code=FAKE_TOKEN), - admin_code=400), - TestSpec(url_for('get_tokens', repository=PRIVATE_REPO, code=FAKE_TOKEN), + TestSpec(url_for('api.get_tokens', repository=PUBLIC_REPO, + code=FAKE_TOKEN), admin_code=403), + TestSpec(url_for('api.get_tokens', repository=ORG_REPO, code=FAKE_TOKEN), admin_code=400), + TestSpec(url_for('api.get_tokens', repository=PRIVATE_REPO, + code=FAKE_TOKEN), admin_code=400), - TestSpec(url_for('create_token', repository=PUBLIC_REPO), + TestSpec(url_for('api.create_token', repository=PUBLIC_REPO), admin_code=403).set_method('POST'), - (TestSpec(url_for('create_token', repository=ORG_REPO), + (TestSpec(url_for('api.create_token', repository=ORG_REPO), admin_code=201).set_method('POST') .set_data_from_obj(CREATE_TOKEN_DETAILS)), - (TestSpec(url_for('create_token', repository=PRIVATE_REPO), + (TestSpec(url_for('api.create_token', repository=PRIVATE_REPO), admin_code=201).set_method('POST') .set_data_from_obj(CREATE_TOKEN_DETAILS)), - TestSpec(url_for('change_token', repository=PUBLIC_REPO, code=FAKE_TOKEN), - admin_code=403).set_method('PUT'), - TestSpec(url_for('change_token', repository=ORG_REPO, code=FAKE_TOKEN), + TestSpec(url_for('api.change_token', repository=PUBLIC_REPO, + code=FAKE_TOKEN), admin_code=403).set_method('PUT'), + TestSpec(url_for('api.change_token', repository=ORG_REPO, code=FAKE_TOKEN), admin_code=400).set_method('PUT'), - TestSpec(url_for('change_token', repository=PRIVATE_REPO, + TestSpec(url_for('api.change_token', repository=PRIVATE_REPO, code=FAKE_TOKEN), admin_code=400).set_method('PUT'), - TestSpec(url_for('delete_token', repository=PUBLIC_REPO, code=FAKE_TOKEN), - admin_code=403).set_method('DELETE'), - TestSpec(url_for('delete_token', repository=ORG_REPO, code=FAKE_TOKEN), + TestSpec(url_for('api.delete_token', repository=PUBLIC_REPO, + code=FAKE_TOKEN), admin_code=403).set_method('DELETE'), + TestSpec(url_for('api.delete_token', repository=ORG_REPO, code=FAKE_TOKEN), admin_code=400).set_method('DELETE'), - TestSpec(url_for('delete_token', repository=PRIVATE_REPO, + TestSpec(url_for('api.delete_token', repository=PRIVATE_REPO, code=FAKE_TOKEN), admin_code=400).set_method('DELETE'), - TestSpec(url_for('update_user_subscription'), 401, 400, 400, 400).set_method('PUT'), + TestSpec(url_for('api.update_user_subscription'), + 401, 400, 400, 400).set_method('PUT'), - TestSpec(url_for('update_org_subscription', orgname=ORG), + TestSpec(url_for('api.update_org_subscription', orgname=ORG), 401, 403, 403, 400).set_method('PUT'), - TestSpec(url_for('get_user_subscription'), 401, 200, 200, 200), + TestSpec(url_for('api.get_user_subscription'), 401, 200, 200, 200), - TestSpec(url_for('get_org_subscription', orgname=ORG)), + TestSpec(url_for('api.get_org_subscription', orgname=ORG)), - TestSpec(url_for('list_repo_logs', repository=PUBLIC_REPO), admin_code=403), - TestSpec(url_for('list_repo_logs', repository=ORG_REPO)), - TestSpec(url_for('list_repo_logs', repository=PRIVATE_REPO)), + TestSpec(url_for('api.list_repo_logs', repository=PUBLIC_REPO), + admin_code=403), + TestSpec(url_for('api.list_repo_logs', repository=ORG_REPO)), + TestSpec(url_for('api.list_repo_logs', repository=PRIVATE_REPO)), - TestSpec(url_for('list_org_logs', orgname=ORG)), + TestSpec(url_for('api.list_org_logs', orgname=ORG)), ] @@ -460,120 +468,132 @@ class IndexTestSpec(object): def build_index_specs(): return [ - IndexTestSpec(url_for('get_image_layer', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.get_image_layer', image_id=FAKE_IMAGE_ID), PUBLIC_REPO, 200, 200, 200, 200), - IndexTestSpec(url_for('get_image_layer', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.get_image_layer', image_id=FAKE_IMAGE_ID), PRIVATE_REPO), - IndexTestSpec(url_for('get_image_layer', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.get_image_layer', image_id=FAKE_IMAGE_ID), ORG_REPO), - IndexTestSpec(url_for('put_image_layer', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.put_image_layer', image_id=FAKE_IMAGE_ID), PUBLIC_REPO, 403, 403, 403, 403).set_method('PUT'), - IndexTestSpec(url_for('put_image_layer', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.put_image_layer', image_id=FAKE_IMAGE_ID), PRIVATE_REPO, 403, 403, 403, 404).set_method('PUT'), - IndexTestSpec(url_for('put_image_layer', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.put_image_layer', image_id=FAKE_IMAGE_ID), ORG_REPO, 403, 403, 403, 404).set_method('PUT'), - IndexTestSpec(url_for('put_image_checksum', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.put_image_checksum', + image_id=FAKE_IMAGE_ID), PUBLIC_REPO, 403, 403, 403, 403).set_method('PUT'), - IndexTestSpec(url_for('put_image_checksum', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.put_image_checksum', + image_id=FAKE_IMAGE_ID), PRIVATE_REPO, 403, 403, 403, 400).set_method('PUT'), - IndexTestSpec(url_for('put_image_checksum', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.put_image_checksum', + image_id=FAKE_IMAGE_ID), ORG_REPO, 403, 403, 403, 400).set_method('PUT'), - IndexTestSpec(url_for('get_image_json', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.get_image_json', image_id=FAKE_IMAGE_ID), PUBLIC_REPO, 404, 404, 404, 404), - IndexTestSpec(url_for('get_image_json', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.get_image_json', image_id=FAKE_IMAGE_ID), PRIVATE_REPO, 403, 403, 404, 404), - IndexTestSpec(url_for('get_image_json', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.get_image_json', image_id=FAKE_IMAGE_ID), ORG_REPO, 403, 403, 404, 404), - IndexTestSpec(url_for('get_image_ancestry', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.get_image_ancestry', + image_id=FAKE_IMAGE_ID), PUBLIC_REPO, 404, 404, 404, 404), - IndexTestSpec(url_for('get_image_ancestry', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.get_image_ancestry', + image_id=FAKE_IMAGE_ID), PRIVATE_REPO, 403, 403, 404, 404), - IndexTestSpec(url_for('get_image_ancestry', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.get_image_ancestry', + image_id=FAKE_IMAGE_ID), ORG_REPO, 403, 403, 404, 404), - IndexTestSpec(url_for('put_image_json', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.put_image_json', image_id=FAKE_IMAGE_ID), PUBLIC_REPO, 403, 403, 403, 403).set_method('PUT'), - IndexTestSpec(url_for('put_image_json', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.put_image_json', image_id=FAKE_IMAGE_ID), PRIVATE_REPO, 403, 403, 403, 400).set_method('PUT'), - IndexTestSpec(url_for('put_image_json', image_id=FAKE_IMAGE_ID), + IndexTestSpec(url_for('registry.put_image_json', image_id=FAKE_IMAGE_ID), ORG_REPO, 403, 403, 403, 400).set_method('PUT'), - IndexTestSpec(url_for('create_user'), NO_REPO, 201, 201, 201, + IndexTestSpec(url_for('index.create_user'), NO_REPO, 201, 201, 201, 201).set_method('POST').set_data_from_obj(NEW_USER_DETAILS), - IndexTestSpec(url_for('get_user'), NO_REPO, 404, 200, 200, 200), + IndexTestSpec(url_for('index.get_user'), NO_REPO, 404, 200, 200, 200), - IndexTestSpec(url_for('update_user', username=FAKE_USERNAME), + IndexTestSpec(url_for('index.update_user', username=FAKE_USERNAME), NO_REPO, 403, 403, 403, 403).set_method('PUT'), - IndexTestSpec(url_for('create_repository', repository=PUBLIC_REPO), + IndexTestSpec(url_for('index.create_repository', repository=PUBLIC_REPO), NO_REPO, 403, 403, 403, 403).set_method('PUT'), - IndexTestSpec(url_for('create_repository', repository=PRIVATE_REPO), + IndexTestSpec(url_for('index.create_repository', repository=PRIVATE_REPO), NO_REPO, 403, 403, 403, 201).set_method('PUT'), - IndexTestSpec(url_for('create_repository', repository=ORG_REPO), + IndexTestSpec(url_for('index.create_repository', repository=ORG_REPO), NO_REPO, 403, 403, 403, 201).set_method('PUT'), - IndexTestSpec(url_for('update_images', repository=PUBLIC_REPO), NO_REPO, - 403, 403, 403, 403).set_method('PUT'), - IndexTestSpec(url_for('update_images', repository=PRIVATE_REPO), NO_REPO, - 403, 403, 403, 204).set_method('PUT'), - IndexTestSpec(url_for('update_images', repository=ORG_REPO), NO_REPO, + IndexTestSpec(url_for('index.update_images', repository=PUBLIC_REPO), + NO_REPO, 403, 403, 403, 403).set_method('PUT'), + IndexTestSpec(url_for('index.update_images', repository=PRIVATE_REPO), + NO_REPO, 403, 403, 403, 204).set_method('PUT'), + IndexTestSpec(url_for('index.update_images', repository=ORG_REPO), NO_REPO, 403, 403, 403, 204).set_method('PUT'), - IndexTestSpec(url_for('get_repository_images', repository=PUBLIC_REPO), + IndexTestSpec(url_for('index.get_repository_images', + repository=PUBLIC_REPO), NO_REPO, 200, 200, 200, 200), - IndexTestSpec(url_for('get_repository_images', repository=PRIVATE_REPO)), - IndexTestSpec(url_for('get_repository_images', repository=ORG_REPO)), + IndexTestSpec(url_for('index.get_repository_images', + repository=PRIVATE_REPO)), + IndexTestSpec(url_for('index.get_repository_images', repository=ORG_REPO)), - IndexTestSpec(url_for('delete_repository_images', repository=PUBLIC_REPO), + IndexTestSpec(url_for('index.delete_repository_images', + repository=PUBLIC_REPO), NO_REPO, 501, 501, 501, 501).set_method('DELETE'), - IndexTestSpec(url_for('put_repository_auth', repository=PUBLIC_REPO), + IndexTestSpec(url_for('index.put_repository_auth', repository=PUBLIC_REPO), NO_REPO, 501, 501, 501, 501).set_method('PUT'), - IndexTestSpec(url_for('get_search'), NO_REPO, 501, 501, 501, 501), + IndexTestSpec(url_for('index.get_search'), NO_REPO, 501, 501, 501, 501), - IndexTestSpec(url_for('ping'), NO_REPO, 200, 200, 200, 200), + IndexTestSpec(url_for('index.ping'), NO_REPO, 200, 200, 200, 200), - IndexTestSpec(url_for('get_tags', repository=PUBLIC_REPO), NO_REPO, + IndexTestSpec(url_for('tags.get_tags', repository=PUBLIC_REPO), NO_REPO, 200, 200, 200, 200), - IndexTestSpec(url_for('get_tags', repository=PRIVATE_REPO)), - IndexTestSpec(url_for('get_tags', repository=ORG_REPO)), + IndexTestSpec(url_for('tags.get_tags', repository=PRIVATE_REPO)), + IndexTestSpec(url_for('tags.get_tags', repository=ORG_REPO)), - IndexTestSpec(url_for('get_tag', repository=PUBLIC_REPO, + IndexTestSpec(url_for('tags.get_tag', repository=PUBLIC_REPO, tag=FAKE_TAG_NAME), NO_REPO, 400, 400, 400, 400), - IndexTestSpec(url_for('get_tag', repository=PRIVATE_REPO, + IndexTestSpec(url_for('tags.get_tag', repository=PRIVATE_REPO, tag=FAKE_TAG_NAME), NO_REPO, 403, 403, 400, 400), - IndexTestSpec(url_for('get_tag', repository=ORG_REPO, + IndexTestSpec(url_for('tags.get_tag', repository=ORG_REPO, tag=FAKE_TAG_NAME), NO_REPO, 403, 403, 400, 400), - IndexTestSpec(url_for('put_tag', repository=PUBLIC_REPO, + IndexTestSpec(url_for('tags.put_tag', repository=PUBLIC_REPO, tag=FAKE_TAG_NAME), NO_REPO, 403, 403, 403, 403).set_method('PUT'), - IndexTestSpec(url_for('put_tag', repository=PRIVATE_REPO, + IndexTestSpec(url_for('tags.put_tag', repository=PRIVATE_REPO, tag=FAKE_TAG_NAME), NO_REPO, 403, 403, 403, 400).set_method('PUT'), - IndexTestSpec(url_for('put_tag', repository=ORG_REPO, tag=FAKE_TAG_NAME), + IndexTestSpec(url_for('tags.put_tag', repository=ORG_REPO, + tag=FAKE_TAG_NAME), NO_REPO, 403, 403, 403, 400).set_method('PUT'), - IndexTestSpec(url_for('delete_tag', repository=PUBLIC_REPO, + IndexTestSpec(url_for('tags.delete_tag', repository=PUBLIC_REPO, tag=FAKE_TAG_NAME), NO_REPO, 403, 403, 403, 403).set_method('DELETE'), - IndexTestSpec(url_for('delete_tag', repository=PRIVATE_REPO, + IndexTestSpec(url_for('tags.delete_tag', repository=PRIVATE_REPO, tag=FAKE_TAG_NAME), NO_REPO, 403, 403, 403, 400).set_method('DELETE'), - IndexTestSpec(url_for('delete_tag', repository=ORG_REPO, + IndexTestSpec(url_for('tags.delete_tag', repository=ORG_REPO, tag=FAKE_TAG_NAME), NO_REPO, 403, 403, 403, 400).set_method('DELETE'), - IndexTestSpec(url_for('delete_repository_tags', repository=PUBLIC_REPO), + IndexTestSpec(url_for('tags.delete_repository_tags', + repository=PUBLIC_REPO), NO_REPO, 403, 403, 403, 403).set_method('DELETE'), - IndexTestSpec(url_for('delete_repository_tags', repository=PRIVATE_REPO), + IndexTestSpec(url_for('tags.delete_repository_tags', + repository=PRIVATE_REPO), NO_REPO, 403, 403, 403, 204).set_method('DELETE'), - IndexTestSpec(url_for('delete_repository_tags', repository=ORG_REPO), + IndexTestSpec(url_for('tags.delete_repository_tags', repository=ORG_REPO), NO_REPO, 403, 403, 403, 204).set_method('DELETE'), ] diff --git a/test/test_api_security.py b/test/test_api_security.py index 49a5bc8a5..78cb7d3a7 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -1,13 +1,14 @@ import unittest -import json - -import endpoints.api +from endpoints.api import api from app import app -from initdb import wipe_database, initialize_database, populate_database +from initdb import setup_database_for_testing, finished_database_for_testing from specs import build_specs +app.register_blueprint(api, url_prefix='/api') + + NO_ACCESS_USER = 'freshuser' READ_ACCESS_USER = 'reader' ADMIN_ACCESS_USER = 'devtable' @@ -15,9 +16,10 @@ ADMIN_ACCESS_USER = 'devtable' class ApiTestCase(unittest.TestCase): def setUp(self): - wipe_database() - initialize_database() - populate_database() + setup_database_for_testing(self) + + def tearDown(self): + finished_database_for_testing(self) class _SpecTestBuilder(type): @@ -27,8 +29,10 @@ class _SpecTestBuilder(type): with app.test_client() as c: if auth_username: # Temporarily remove the teardown functions - teardown_funcs = app.teardown_request_funcs[None] - app.teardown_request_funcs[None] = [] + teardown_funcs = [] + if None in app.teardown_request_funcs: + teardown_funcs = app.teardown_request_funcs[None] + app.teardown_request_funcs[None] = [] with c.session_transaction() as sess: sess['user_id'] = auth_username diff --git a/test/test_endpoint_security.py b/test/test_endpoint_security.py index e762ec5ea..724629f3a 100644 --- a/test/test_endpoint_security.py +++ b/test/test_endpoint_security.py @@ -1,13 +1,17 @@ import unittest -import endpoints.registry -import endpoints.index -import endpoints.tags - from app import app from util.names import parse_namespace_repository -from initdb import wipe_database, initialize_database, populate_database +from initdb import setup_database_for_testing, finished_database_for_testing from specs import build_index_specs +from endpoints.registry import registry +from endpoints.index import index +from endpoints.tags import tags + + +app.register_blueprint(index, url_prefix='/v1') +app.register_blueprint(tags, url_prefix='/v1') +app.register_blueprint(registry, url_prefix='/v1') NO_ACCESS_USER = 'freshuser' @@ -16,10 +20,11 @@ ADMIN_ACCESS_USER = 'devtable' class EndpointTestCase(unittest.TestCase): - def setUp(self): - wipe_database() - initialize_database() - populate_database() + def setUp(self): + setup_database_for_testing(self) + + def tearDown(self): + finished_database_for_testing(self) class _SpecTestBuilder(type): @@ -29,8 +34,10 @@ class _SpecTestBuilder(type): with app.test_client() as c: if session_var_list: # Temporarily remove the teardown functions - teardown_funcs = app.teardown_request_funcs[None] - app.teardown_request_funcs[None] = [] + teardown_funcs = [] + if None in app.teardown_request_funcs: + teardown_funcs = app.teardown_request_funcs[None] + app.teardown_request_funcs[None] = [] with c.session_transaction() as sess: for sess_key, sess_val in session_var_list: diff --git a/util/changes.py b/util/changes.py index 814359470..eaeec9d83 100644 --- a/util/changes.py +++ b/util/changes.py @@ -12,7 +12,11 @@ ALLOWED_TYPES = {tarfile.REGTYPE, tarfile.AREGTYPE} def files_and_dirs_from_tar(source_stream, removed_prefix_collector): - tar_stream = tarfile.open(mode='r|*', fileobj=source_stream) + try: + tar_stream = tarfile.open(mode='r|*', fileobj=source_stream) + except tarfile.ReadError: + # Empty tar file + return for tar_info in tar_stream: absolute = os.path.relpath(tar_info.name.decode('utf-8'), './') diff --git a/util/http.py b/util/http.py new file mode 100644 index 000000000..6592768cc --- /dev/null +++ b/util/http.py @@ -0,0 +1,59 @@ +import logging + +from app import mixpanel +from flask import request, abort as flask_abort, jsonify +from auth.auth import get_authenticated_user, get_validated_token + +logger = logging.getLogger(__name__) + + +DEFAULT_MESSAGE = {} +DEFAULT_MESSAGE[400] = 'Invalid Request' +DEFAULT_MESSAGE[401] = 'Unauthorized' +DEFAULT_MESSAGE[403] = 'Permission Denied' +DEFAULT_MESSAGE[404] = 'Not Found' +DEFAULT_MESSAGE[409] = 'Conflict' +DEFAULT_MESSAGE[501] = 'Not Implemented' + +def abort(status_code, message=None, issue=None, **kwargs): + message = (str(message) % kwargs if message else + DEFAULT_MESSAGE.get(status_code, '')) + + params = dict(request.view_args) + params.update(kwargs) + + params['url'] = request.url + params['status_code'] = status_code + params['message'] = message + + # Add the user information. + auth_user = get_authenticated_user() + auth_token = get_validated_token() + if auth_user: + mixpanel.track(auth_user.username, 'http_error', params) + message = '%s (user: %s)' % (message, auth_user.username) + elif auth_token: + mixpanel.track(auth_token.code, 'http_error', params) + message = '%s (token: %s)' % (message, + auth_token.friendly_name or auth_token.code) + + # Log the abort. + logger.error('Error %s: %s; Arguments: %s' % (status_code, message, params)) + + # Calculate the issue URL (if the issue ID was supplied). + issue_url = None + if issue: + issue_url = 'http://docs.quay.io/issues/%s.html' % (issue) + + # Create the final response data and message. + data = {} + data['error'] = message + + if issue_url: + data['info_url'] = issue_url + + resp = jsonify(data) + resp.status_code = status_code + + # Report the abort to the user. + flask_abort(resp)