From 2597bcef3fc330457a6818362ac4b99e7dbec8cc Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 11 Aug 2014 15:47:44 -0400 Subject: [PATCH 01/57] Add support for login with Google. Note that this CL is not complete --- config.py | 12 +- endpoints/callbacks.py | 128 ++++++++++++++---- initdb.py | 2 + static/directives/external-login-button.html | 17 +++ static/directives/signin-form.html | 6 +- static/directives/signup-form.html | 6 +- static/js/app.js | 100 +++++++++----- static/js/controllers.js | 5 +- static/partials/user-admin.html | 30 +++- .../{githuberror.html => ologinerror.html} | 8 +- 10 files changed, 231 insertions(+), 83 deletions(-) create mode 100644 static/directives/external-login-button.html rename templates/{githuberror.html => ologinerror.html} (66%) diff --git a/config.py b/config.py index a903fa29a..708449714 100644 --- a/config.py +++ b/config.py @@ -19,7 +19,7 @@ def build_requests_session(): CLIENT_WHITELIST = ['SERVER_HOSTNAME', 'PREFERRED_URL_SCHEME', 'GITHUB_CLIENT_ID', 'GITHUB_LOGIN_CLIENT_ID', 'MIXPANEL_KEY', 'STRIPE_PUBLISHABLE_KEY', 'ENTERPRISE_LOGO_URL', 'SENTRY_PUBLIC_DSN', 'AUTHENTICATION_TYPE', - 'REGISTRY_TITLE', 'REGISTRY_TITLE_SHORT'] + 'REGISTRY_TITLE', 'REGISTRY_TITLE_SHORT', 'GOOGLE_LOGIN_CLIENT_ID'] def getFrontendVisibleConfig(config_dict): @@ -115,6 +115,13 @@ class DefaultConfig(object): GITHUB_LOGIN_CLIENT_ID = '' GITHUB_LOGIN_CLIENT_SECRET = '' + # Google Config. + GOOGLE_TOKEN_URL = 'https://accounts.google.com/o/oauth2/token' + GOOGLE_USER_URL = 'https://www.googleapis.com/oauth2/v1/userinfo' + + GOOGLE_LOGIN_CLIENT_ID = '' + GOOGLE_LOGIN_CLIENT_SECRET = '' + # Requests based HTTP client with a large request pool HTTPCLIENT = build_requests_session() @@ -144,6 +151,9 @@ class DefaultConfig(object): # Feature Flag: Whether GitHub login is supported. FEATURE_GITHUB_LOGIN = False + # Feature Flag: Whether Google login is supported. + FEATURE_GOOGLE_LOGIN = False + # Feature flag, whether to enable olark chat FEATURE_OLARK_CHAT = False diff --git a/endpoints/callbacks.py b/endpoints/callbacks.py index 015f3c3a7..ba53cbc5c 100644 --- a/endpoints/callbacks.py +++ b/endpoints/callbacks.py @@ -7,6 +7,7 @@ from endpoints.common import render_page_template, common_login, route_show_if from app import app, analytics from data import model from util.names import parse_repository_name +from util.validation import generate_valid_usernames from util.http import abort from auth.permissions import AdministerRepositoryPermission from auth.auth import require_session_login @@ -21,19 +22,31 @@ client = app.config['HTTPCLIENT'] callback = Blueprint('callback', __name__) -def exchange_github_code_for_token(code, for_login=True): +def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_encode=False): code = request.args.get('code') + id_config = service_name + '_LOGIN_CLIENT_ID' if for_login else service_name + '_CLIENT_ID' + secret_config = service_name + '_LOGIN_CLIENT_SECRET' if for_login else service_name + '_CLIENT_SECRET' + payload = { - 'client_id': app.config['GITHUB_LOGIN_CLIENT_ID' if for_login else 'GITHUB_CLIENT_ID'], - 'client_secret': app.config['GITHUB_LOGIN_CLIENT_SECRET' if for_login else 'GITHUB_CLIENT_SECRET'], + 'client_id': app.config[id_config], + 'client_secret': app.config[secret_config], 'code': code, + 'grant_type': 'authorization_code', + 'redirect_uri': '%s://%s/oauth2/%s/callback' % (app.config['PREFERRED_URL_SCHEME'], + app.config['SERVER_HOSTNAME'], + service_name.lower()) } + headers = { 'Accept': 'application/json' } - get_access_token = client.post(app.config['GITHUB_TOKEN_URL'], - params=payload, headers=headers) + if form_encode: + get_access_token = client.post(app.config[service_name + '_TOKEN_URL'], + data=payload, headers=headers) + else: + get_access_token = client.post(app.config[service_name + '_TOKEN_URL'], + params=payload, headers=headers) json_data = get_access_token.json() if not json_data: @@ -52,17 +65,76 @@ def get_github_user(token): return get_user.json() +def get_google_user(token): + token_param = { + 'access_token': token, + 'alt': 'json', + } + + get_user = client.get(app.config['GOOGLE_USER_URL'], params=token_param) + return get_user.json() + +def conduct_oauth_login(service_name, user_id, username, email): + to_login = model.verify_federated_login(service_name.lower(), user_id) + if not to_login: + # try to create the user + try: + valid = next(generate_valid_usernames(username)) + to_login = model.create_federated_user(valid, email, service_name.lower(), + user_id, set_password_notification=True) + + # Success, tell analytics + analytics.track(to_login.username, 'register', {'service': service_name.lower()}) + + state = request.args.get('state', None) + if state: + logger.debug('Aliasing with state: %s' % state) + analytics.alias(to_login.username, state) + + except model.DataModelException, ex: + return render_page_template('ologinerror.html', service_name=service_name, + error_message=ex.message) + + if common_login(to_login): + return redirect(url_for('web.index')) + + return render_page_template('ologinerror.html', service_name=service_name, + error_message='Unknown error') + + +@callback.route('/google/callback', methods=['GET']) +@route_show_if(features.GOOGLE_LOGIN) +def google_oauth_callback(): + error = request.args.get('error', None) + if error: + return render_page_template('ologinerror.html', service_name='Google', error_message=error) + + token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE', form_encode=True) + user_data = get_google_user(token) + if not user_data or not user_data.get('id', None) or not user_data.get('email', None): + return render_page_template('ologinerror.html', service_name = 'Google', + error_message='Could not load user data') + + username = user_data['email'] + at = username.find('@') + if at > 0: + username = username[0:at] + + return conduct_oauth_login('Google', user_data['id'], username, user_data['email']) + + @callback.route('/github/callback', methods=['GET']) @route_show_if(features.GITHUB_LOGIN) def github_oauth_callback(): error = request.args.get('error', None) if error: - return render_page_template('githuberror.html', error_message=error) + return render_page_template('ologinerror.html', service_name = 'GitHub', error_message=error) - token = exchange_github_code_for_token(request.args.get('code')) + token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB') user_data = get_github_user(token) if not user_data: - return render_page_template('githuberror.html', error_message='Could not load user data') + return render_page_template('ologinerror.html', service_name = 'GitHub', + error_message='Could not load user data') username = user_data['login'] github_id = user_data['id'] @@ -84,38 +156,34 @@ def github_oauth_callback(): if user_email['primary']: break - to_login = model.verify_federated_login('github', github_id) - if not to_login: - # try to create the user - try: - to_login = model.create_federated_user(username, found_email, 'github', - github_id, set_password_notification=True) + return conduct_oauth_login('github', github_id, username, found_email) - # Success, tell analytics - analytics.track(to_login.username, 'register', {'service': 'github'}) - state = request.args.get('state', None) - if state: - logger.debug('Aliasing with state: %s' % state) - analytics.alias(to_login.username, state) +@callback.route('/google/callback/attach', methods=['GET']) +@route_show_if(features.GOOGLE_LOGIN) +@require_session_login +def google_oauth_attach(): + token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE') + user_data = get_google_user(token) + if not user_data or not user_data.get('id', None): + return render_page_template('ologinerror.html', service_name = 'Google', + error_message='Could not load user data') - except model.DataModelException, ex: - return render_page_template('githuberror.html', error_message=ex.message) - - if common_login(to_login): - return redirect(url_for('web.index')) - - return render_page_template('githuberror.html') + google_id = user_data['id'] + user_obj = current_user.db_user() + model.attach_federated_login(user_obj, 'google', google_id) + return redirect(url_for('web.user')) @callback.route('/github/callback/attach', methods=['GET']) @route_show_if(features.GITHUB_LOGIN) @require_session_login def github_oauth_attach(): - token = exchange_github_code_for_token(request.args.get('code')) + token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB') user_data = get_github_user(token) if not user_data: - return render_page_template('githuberror.html', error_message='Could not load user data') + return render_page_template('ologinerror.html', service_name = 'GitHub', + error_message='Could not load user data') github_id = user_data['id'] user_obj = current_user.db_user() @@ -130,7 +198,7 @@ def github_oauth_attach(): def attach_github_build_trigger(namespace, repository): permission = AdministerRepositoryPermission(namespace, repository) if permission.can(): - token = exchange_github_code_for_token(request.args.get('code'), for_login=False) + token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB', for_login=False) repo = model.get_repository(namespace, repository) if not repo: msg = 'Invalid repository: %s/%s' % (namespace, repository) diff --git a/initdb.py b/initdb.py index 7e48ae3af..5d10a2039 100644 --- a/initdb.py +++ b/initdb.py @@ -179,6 +179,8 @@ def initialize_database(): TeamRole.create(name='member') Visibility.create(name='public') Visibility.create(name='private') + + LoginService.create(name='google') LoginService.create(name='github') LoginService.create(name='quayrobot') LoginService.create(name='ldap') diff --git a/static/directives/external-login-button.html b/static/directives/external-login-button.html new file mode 100644 index 000000000..cc1d39bbd --- /dev/null +++ b/static/directives/external-login-button.html @@ -0,0 +1,17 @@ + + + + + Sign In with GitHub + Attach to GitHub Account + + + + + + + Sign In with Google + Attach to Google Account + + + diff --git a/static/directives/signin-form.html b/static/directives/signin-form.html index f56b8f8db..ec57619a8 100644 --- a/static/directives/signin-form.html +++ b/static/directives/signin-form.html @@ -11,10 +11,8 @@ OR - - Sign In with GitHub - +
+
Invalid username or password.
diff --git a/static/directives/signup-form.html b/static/directives/signup-form.html index fb0ccc6fa..249bff31c 100644 --- a/static/directives/signup-form.html +++ b/static/directives/signup-form.html @@ -18,10 +18,8 @@ OR - - Sign In with GitHub - +
+

No credit card required.

diff --git a/static/js/app.js b/static/js/app.js index 8daec75d4..978012bb6 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1278,10 +1278,41 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading var keyService = {} keyService['stripePublishableKey'] = Config['STRIPE_PUBLISHABLE_KEY']; + keyService['githubClientId'] = Config['GITHUB_CLIENT_ID']; keyService['githubLoginClientId'] = Config['GITHUB_LOGIN_CLIENT_ID']; keyService['githubRedirectUri'] = Config.getUrl('/oauth2/github/callback'); + keyService['googleLoginClientId'] = Config['GOOGLE_LOGIN_CLIENT_ID']; + keyService['googleRedirectUri'] = Config.getUrl('/oauth2/google/callback'); + + keyService['googleLoginUrl'] = 'https://accounts.google.com/o/oauth2/auth?response_type=code&'; + keyService['githubLoginUrl'] = 'https://github.com/login/oauth/authorize?'; + + keyService['googleLoginScope'] = 'openid email'; + keyService['githubLoginScope'] = 'user:email'; + + keyService.getExternalLoginUrl = function(service, action) { + var state_clause = ''; + if (Config.MIXPANEL_KEY && window.mixpanel) { + if (mixpanel.get_distinct_id !== undefined) { + state_clause = "&state=" + encodeURIComponent(mixpanel.get_distinct_id()); + } + } + + var client_id = keyService[service + 'LoginClientId']; + var scope = keyService[service + 'LoginScope']; + var redirect_uri = keyService[service + 'RedirectUri']; + if (action == 'attach') { + redirect_uri += '/attach'; + } + + var url = keyService[service + 'LoginUrl'] + 'client_id=' + client_id + '&scope=' + scope + + '&redirect_uri=' + redirect_uri + state_clause; + + return url; + }; + return keyService; }]); @@ -2150,6 +2181,41 @@ quayApp.directive('userSetup', function () { }); +quayApp.directive('externalLoginButton', function () { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/external-login-button.html', + replace: false, + transclude: true, + restrict: 'C', + scope: { + 'signInStarted': '&signInStarted', + 'redirectUrl': '=redirectUrl', + 'provider': '@provider', + 'action': '@action' + }, + controller: function($scope, $location, $timeout, ApiService, KeyService, UserService, CookieService, Features, Config) { + $scope.startSignin = function(service) { + $scope.signInStarted({'service': service}); + + var url = KeyService.getExternalLoginUrl(service, $scope.action || 'login'); + + // Save the redirect URL in a cookie so that we can redirect back after the service returns to us. + var redirectURL = $scope.redirectUrl || window.location.toString(); + CookieService.putPermanent('quay.redirectAfterLoad', redirectURL); + + // Needed to ensure that UI work done by the started callback is finished before the location + // changes. + $timeout(function() { + document.location = url; + }, 250); + }; + } + }; + return directiveDefinitionObject; +}); + + quayApp.directive('signinForm', function () { var directiveDefinitionObject = { priority: 0, @@ -2163,29 +2229,6 @@ quayApp.directive('signinForm', function () { 'signedIn': '&signedIn' }, controller: function($scope, $location, $timeout, ApiService, KeyService, UserService, CookieService, Features, Config) { - $scope.showGithub = function() { - if (!Features.GITHUB_LOGIN) { return; } - - $scope.markStarted(); - - var mixpanelDistinctIdClause = ''; - if (Config.MIXPANEL_KEY && mixpanel.get_distinct_id !== undefined) { - $scope.mixpanelDistinctIdClause = "&state=" + encodeURIComponent(mixpanel.get_distinct_id()); - } - - // Save the redirect URL in a cookie so that we can redirect back after GitHub returns to us. - var redirectURL = $scope.redirectUrl || window.location.toString(); - CookieService.putPermanent('quay.redirectAfterLoad', redirectURL); - - // Needed to ensure that UI work done by the started callback is finished before the location - // changes. - $timeout(function() { - var url = 'https://github.com/login/oauth/authorize?client_id=' + encodeURIComponent(KeyService.githubLoginClientId) + - '&scope=user:email' + mixpanelDistinctIdClause; - document.location = url; - }, 250); - }; - $scope.markStarted = function() { if ($scope.signInStarted != null) { $scope.signInStarted(); @@ -2235,18 +2278,9 @@ quayApp.directive('signupForm', function () { scope: { }, - controller: function($scope, $location, $timeout, ApiService, KeyService, UserService, Config, UIService) { + controller: function($scope, $location, $timeout, ApiService, KeyService, UserService, Config, UIService) { $('.form-signup').popover(); - if (Config.MIXPANEL_KEY) { - angulartics.waitForVendorApi(mixpanel, 500, function(loadedMixpanel) { - var mixpanelId = loadedMixpanel.get_distinct_id(); - $scope.github_state_clause = '&state=' + mixpanelId; - }); - } - - $scope.githubClientId = KeyService.githubLoginClientId; - $scope.awaitingConfirmation = false; $scope.registering = false; diff --git a/static/js/controllers.js b/static/js/controllers.js index d77ffb298..7690f79f6 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -1681,6 +1681,10 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use $scope.githubLogin = resp.login; }); } + + if ($scope.cuser.logins[i].service == 'google') { + $scope.hasGoogleLogin = true; + } } } }); @@ -1697,7 +1701,6 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use $scope.convertStep = 0; $scope.org = {}; $scope.githubRedirectUri = KeyService.githubRedirectUri; - $scope.githubClientId = KeyService.githubLoginClientId; $scope.authorizedApps = null; $scope.logsShown = 0; diff --git a/static/partials/user-admin.html b/static/partials/user-admin.html index 783c5f87a..fc0acb89e 100644 --- a/static/partials/user-admin.html +++ b/static/partials/user-admin.html @@ -33,7 +33,7 @@
  • Account E-mail
  • Robot Accounts
  • Change Password
    • -
  • GitHub Login
    • +
  • External Logins
  • Authorized Applications
  • Usage Logs @@ -162,12 +162,14 @@ - -
    + +
    -
    + + +
    GitHub Login:
    @@ -175,12 +177,28 @@ {{githubLogin}}
    -
    - Connect with GitHub +
    +
    + + +
    +
    +
    Google Login:
    +
    +
    + Account tied to your Google account. +
    +
    + +
    +
    +
    +
    +
    diff --git a/templates/githuberror.html b/templates/ologinerror.html similarity index 66% rename from templates/githuberror.html rename to templates/ologinerror.html index acb803f57..cd921eec2 100644 --- a/templates/githuberror.html +++ b/templates/ologinerror.html @@ -1,14 +1,14 @@ {% extends "base.html" %} {% block title %} - Error Logging in with GitHub · Quay.io + Error Logging in with {{ service_name }} · Quay.io {% endblock %} {% block body_content %}
    -

    There was an error logging in with GitHub.

    +

    There was an error logging in with {{ service_name }}.

    {% if error_message %}
    {{ error_message }}
    @@ -16,11 +16,11 @@
    Please register using the registration form to continue. - You will be able to connect your github account to your Quay.io account + You will be able to connect your account to your Quay.io account in the user settings.
    -{% endblock %} \ No newline at end of file +{% endblock %} From 389c88a7c4b2b08b879b772a0deca139e004ef3e Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 11 Aug 2014 18:25:01 -0400 Subject: [PATCH 02/57] Update federated login to store metadata and have the UI pull the information from the metadata --- data/database.py | 1 + data/model/legacy.py | 13 ++++-- endpoints/api/user.py | 8 ++++ endpoints/callbacks.py | 81 ++++++++++++++++++++++++++------- static/js/controllers.js | 9 ++-- static/partials/user-admin.html | 17 +++++-- 6 files changed, 99 insertions(+), 30 deletions(-) diff --git a/data/database.py b/data/database.py index 76a0af9df..6099cf5d9 100644 --- a/data/database.py +++ b/data/database.py @@ -116,6 +116,7 @@ class FederatedLogin(BaseModel): user = ForeignKeyField(User, index=True) service = ForeignKeyField(LoginService, index=True) service_ident = CharField() + metadata_json = TextField(default='{}') class Meta: database = db diff --git a/data/model/legacy.py b/data/model/legacy.py index b5afdfeb8..bfa310046 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -346,7 +346,8 @@ def set_team_org_permission(team, team_role_name, set_by_username): return team -def create_federated_user(username, email, service_name, service_id, set_password_notification): +def create_federated_user(username, email, service_name, service_id, + set_password_notification, metadata={}): if not is_create_user_allowed(): raise TooManyUsersException() @@ -356,7 +357,8 @@ def create_federated_user(username, email, service_name, service_id, set_passwor service = LoginService.get(LoginService.name == service_name) FederatedLogin.create(user=new_user, service=service, - service_ident=service_id) + service_ident=service_id, + metadata_json=json.dumps(metadata)) if set_password_notification: create_notification('password_required', new_user) @@ -364,9 +366,10 @@ def create_federated_user(username, email, service_name, service_id, set_passwor return new_user -def attach_federated_login(user, service_name, service_id): +def attach_federated_login(user, service_name, service_id, metadata={}): service = LoginService.get(LoginService.name == service_name) - FederatedLogin.create(user=user, service=service, service_ident=service_id) + FederatedLogin.create(user=user, service=service, service_ident=service_id, + metadata_json=json.dumps(metadata)) return user @@ -385,7 +388,7 @@ def verify_federated_login(service_name, service_id): def list_federated_logins(user): selected = FederatedLogin.select(FederatedLogin.service_ident, - LoginService.name) + LoginService.name, FederatedLogin.metadata_json) joined = selected.join(LoginService) return joined.where(LoginService.name != 'quayrobot', FederatedLogin.user == user) diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 3d79a806d..23bc3137a 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -39,9 +39,16 @@ def user_view(user): organizations = model.get_user_organizations(user.username) def login_view(login): + print login.metadata_json + try: + metadata = json.loads(login.metadata_json) + except: + metadata = None + return { 'service': login.service.name, 'service_identifier': login.service_ident, + 'metadata': metadata } logins = model.list_federated_logins(user) @@ -88,6 +95,7 @@ class User(ApiResource): """ Operations related to users. """ schemas = { 'NewUser': { + 'id': 'NewUser', 'type': 'object', 'description': 'Fields which must be specified for a new user.', diff --git a/endpoints/callbacks.py b/endpoints/callbacks.py index ba53cbc5c..49fa1e8a6 100644 --- a/endpoints/callbacks.py +++ b/endpoints/callbacks.py @@ -11,6 +11,7 @@ from util.validation import generate_valid_usernames from util.http import abort from auth.permissions import AdministerRepositoryPermission from auth.auth import require_session_login +from peewee import IntegrityError import features @@ -22,7 +23,8 @@ client = app.config['HTTPCLIENT'] callback = Blueprint('callback', __name__) -def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_encode=False): +def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_encode=False, + redirect_suffix=''): code = request.args.get('code') id_config = service_name + '_LOGIN_CLIENT_ID' if for_login else service_name + '_CLIENT_ID' secret_config = service_name + '_LOGIN_CLIENT_SECRET' if for_login else service_name + '_CLIENT_SECRET' @@ -32,9 +34,10 @@ def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_en 'client_secret': app.config[secret_config], 'code': code, 'grant_type': 'authorization_code', - 'redirect_uri': '%s://%s/oauth2/%s/callback' % (app.config['PREFERRED_URL_SCHEME'], - app.config['SERVER_HOSTNAME'], - service_name.lower()) + 'redirect_uri': '%s://%s/oauth2/%s/callback%s' % (app.config['PREFERRED_URL_SCHEME'], + app.config['SERVER_HOSTNAME'], + service_name.lower(), + redirect_suffix) } headers = { @@ -74,14 +77,15 @@ def get_google_user(token): get_user = client.get(app.config['GOOGLE_USER_URL'], params=token_param) return get_user.json() -def conduct_oauth_login(service_name, user_id, username, email): +def conduct_oauth_login(service_name, user_id, username, email, metadata={}): to_login = model.verify_federated_login(service_name.lower(), user_id) if not to_login: # try to create the user try: valid = next(generate_valid_usernames(username)) to_login = model.create_federated_user(valid, email, service_name.lower(), - user_id, set_password_notification=True) + user_id, set_password_notification=True, + metadata=metadata) # Success, tell analytics analytics.track(to_login.username, 'register', {'service': service_name.lower()}) @@ -102,6 +106,15 @@ def conduct_oauth_login(service_name, user_id, username, email): error_message='Unknown error') +def get_google_username(user_data): + username = user_data['email'] + at = username.find('@') + if at > 0: + username = username[0:at] + + return username + + @callback.route('/google/callback', methods=['GET']) @route_show_if(features.GOOGLE_LOGIN) def google_oauth_callback(): @@ -115,12 +128,13 @@ def google_oauth_callback(): return render_page_template('ologinerror.html', service_name = 'Google', error_message='Could not load user data') - username = user_data['email'] - at = username.find('@') - if at > 0: - username = username[0:at] + username = get_google_username(user_data) + metadata = { + 'service_username': username + } - return conduct_oauth_login('Google', user_data['id'], username, user_data['email']) + return conduct_oauth_login('Google', user_data['id'], username, user_data['email'], + metadata=metadata) @callback.route('/github/callback', methods=['GET']) @@ -156,14 +170,20 @@ def github_oauth_callback(): if user_email['primary']: break - return conduct_oauth_login('github', github_id, username, found_email) + metadata = { + 'service_username': username + } + + return conduct_oauth_login('github', github_id, username, found_email, metadata=metadata) @callback.route('/google/callback/attach', methods=['GET']) @route_show_if(features.GOOGLE_LOGIN) @require_session_login def google_oauth_attach(): - token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE') + token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE', + redirect_suffix='/attach', form_encode=True) + user_data = get_google_user(token) if not user_data or not user_data.get('id', None): return render_page_template('ologinerror.html', service_name = 'Google', @@ -171,7 +191,21 @@ def google_oauth_attach(): google_id = user_data['id'] user_obj = current_user.db_user() - model.attach_federated_login(user_obj, 'google', google_id) + + username = get_google_username(user_data) + metadata = { + 'service_username': username + } + + try: + model.attach_federated_login(user_obj, 'google', google_id, metadata=metadata) + except IntegrityError: + err = 'Google account %s is already attached to a %s account' % ( + username, app.config['REGISTRY_TITLE_SHORT']) + + return render_page_template('ologinerror.html', service_name = 'Google', + error_message=err) + return redirect(url_for('web.user')) @@ -187,7 +221,21 @@ def github_oauth_attach(): github_id = user_data['id'] user_obj = current_user.db_user() - model.attach_federated_login(user_obj, 'github', github_id) + + username = user_data['login'] + metadata = { + 'service_username': username + } + + try: + model.attach_federated_login(user_obj, 'github', github_id, metadata=metadata) + except IntegrityError: + err = 'Github account %s is already attached to a %s account' % ( + username, app.config['REGISTRY_TITLE_SHORT']) + + return render_page_template('ologinerror.html', service_name = 'Github', + error_message=err) + return redirect(url_for('web.user')) @@ -198,7 +246,8 @@ def github_oauth_attach(): def attach_github_build_trigger(namespace, repository): permission = AdministerRepositoryPermission(namespace, repository) if permission.can(): - token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB', for_login=False) + token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB', + for_login=False) repo = model.get_repository(namespace, repository) if not repo: msg = 'Invalid repository: %s/%s' % (namespace, repository) diff --git a/static/js/controllers.js b/static/js/controllers.js index 7690f79f6..5278e4efe 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -1673,17 +1673,16 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use UserService.updateUserIn($scope, function(user) { $scope.cuser = jQuery.extend({}, user); - if (Features.GITHUB_LOGIN && $scope.cuser.logins) { + if ($scope.cuser.logins) { for (var i = 0; i < $scope.cuser.logins.length; i++) { if ($scope.cuser.logins[i].service == 'github') { - var githubId = $scope.cuser.logins[i].service_identifier; - $http.get('https://api.github.com/user/' + githubId).success(function(resp) { - $scope.githubLogin = resp.login; - }); + $scope.hasGithubLogin = true; + $scope.githubLogin = $scope.cuser.logins[i].metadata['service_username']; } if ($scope.cuser.logins[i].service == 'google') { $scope.hasGoogleLogin = true; + $scope.googleLogin = $scope.cuser.logins[i].metadata['service_username']; } } } diff --git a/static/partials/user-admin.html b/static/partials/user-admin.html index fc0acb89e..ca76342dd 100644 --- a/static/partials/user-admin.html +++ b/static/partials/user-admin.html @@ -173,11 +173,15 @@
    GitHub Login:
    -
    +
    {{githubLogin}}
    -
    +
    + + Account attached to Github Account +
    +
    @@ -189,8 +193,13 @@
    Google Login:
    -
    - Account tied to your Google account. +
    + + {{ googleLogin }} +
    +
    + + Account attached to Google Account
    From 11176215e10a069045892420a2c41c6492fe11af Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 11 Aug 2014 18:35:26 -0400 Subject: [PATCH 03/57] Commit new DB changes and make sure the metadata is always present in some form --- endpoints/api/user.py | 2 +- test/data/test.db | Bin 614400 -> 614400 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 23bc3137a..d0d089dcd 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -43,7 +43,7 @@ def user_view(user): try: metadata = json.loads(login.metadata_json) except: - metadata = None + metadata = {} return { 'service': login.service.name, diff --git a/test/data/test.db b/test/data/test.db index 4d04283311e6feb845dff3fde9a4d370858119be..b64829db209fca64a5f122a3c96d74f8f7d66845 100644 GIT binary patch delta 6425 zcmds*dwdjCmdCrgs#9HgbucId2x&qhB&5>yeq<6zr;|?7-RVb?&Z7lMRdrPYA!*2y zfEdad{mjhnXI5y}yY4WGpd-=XBWN=0h~m1aJ32Zus581cj2{P=QQ|oJQFdKX_J)Xp zLC4SV-{y}_cb$8__uSt-=hXe&okt6I9xc3eF;cs0i3h2Dq~q*nEa%-fQj;B9_F%5U z=eO*k$G_aQA*YmwbC*94qX; z)1OQZ-j9ZVASWw__FzkozPLL1hoO^L`TJyg^52J#Vfem>Hzap&oz6;Id;QC1kNr!E zc>q$L9vR2hl=bdS_HP@*O1r`Oq;vZotU^IHC68`Di7kBUog0&5Hypzk+*#o=dr?qw z6;Y9#*l`eFgSZcwJywvL4Ik{kaU5Tz{`W$&7y%jciSk3cNBefBV~*n%$48EL9d9^} zADZWU+EIaHsO9QSabJDcoQQ2%>hp2A@%|+1Ig4A7aIn4=y!Q#>tfayLMyhY~ zxC2cQxq*uLLSkKWb)A>-Mrvqxv(M{ou4xa|1gfH~?ez_UJLahs!!em|YWDF`wB7^! zXOR{zB6A5rZEIH>BEhJ;w0a{~9jwu}4b&LJK_92a^hm6?YinFE+$zo0ceS>9MjF5? zXOSYlv77Zbck3+!Ra>>VNDsGjgYrnJU#>QWs{BLEMwgGHShr_$f8Fp_mpI%LYV4;* z0Q)IY!h2+Ggi1751vbZ+E#7*cXWMWi8>(vQ>Z~22wrtv@M+Q6f`r*!iNV&uPyJ8`%JG&NW|*rNvG+v44^YPFwDZ0(n;{On+3tI=L6dMS{74iPG=WTu60iP24U zF_=@WO>QpO=8pQsP(9_1`Poo3)+kaQt~Suv5DZgI6e~8x+8MsN$sLx&04IVUoI{rI zO-)rkPpq`lU*8|@-XyU;$=%#qI>Pq1^%i_!)PK6H zdCt@qERI7C!;zEu0_;nVXO8+Y+=3ux#=La&N=4hg!;gLrOGCeh6?{W zbF|h5ttw-ojZT}Qv#aM8~}SE{MiJF_t25;-rU=CZ$9{gby^-*GHL(=i-F zpqy#w?0=gz1^y;b>rX2YYVudsPcm0FOOn66p)S@G^$GqcTUG0;WhJ?l^E7)Km~b2I zuN4|4p^X7gp0E~zx)avWN~yM`p`8yia?mG)-BlsFIxL1LpWN0U1gqN`o1;=)buAO9 zoq{eOxFWKy$3>NsNlGzvl8ZA6DNCA4a)u6@C&d_)LHFsK6J<)gXG8CR-ajyq=+=7~ zisnd4B5B4+(Pa!(M)Rct&j>6vnT5XLDEKzmTb$^m6HCcTdD!9@$$S_l&pnyo&DB_X z0mpDGEoeHNL5!1}u1TaSN)$=QIYpv%!-&fo$XJ8%qcrVw4|Y0d!f`T88OxTjw3DRZ zUw(-*e;~0wt_Z=&5sSuoqnn)O=2a)?Cz0Tz&!P9TV82j)rd| zs&xe#CBCx8%Pw(pC$`lwO2KWGB|{{6Ru@Sq4NM+Jlwd9~l1Qr*BPjxEby@8eBuc=p zn@UTxtV#kL!6^#KNg72eicFDlgM(Ww@VX?kR@8-@cR(yisuquP0;@`7JT5~AR%1vx zZp2BMk_8^;SWc7~Vd}Ggz(RWpL`9?Gl%SIgEyyIt=y6gKMT3+CN)=T_RpOcmwv^ba zmm3BJCz1?O5aCvG46TwX&kCe2NdhN{hHmiScO^EVU?xnsqdeSR8ZHoyBo&ocNI~ME zt1halqyUDrRTLNk1+Ah&Qi1>j)l{97SXw3pT~p$+pz)HZf+T6HDiCGSkaWo8DVm0p z^BgHFyh>^!C+U*L>+sMAJ|=Ce3OJgkG?_6-M$_PeWC_M0$~?(xA_Em;SxuKfU8&7o zz|xAqiVVy}L4qSCUM3ZW5lAYoC~$SMs2B=3SZZ5R04s=5R1p@h0gIl4@spBFF=Sja z;+mjAnHU93m)hzIC`B?@9abGLD-x6*f>lFeNtUMcIIYtRN2{RSWm~nJRfIUlL5pT^ zB&Wc(po(#dlr&aVIF^CCDuVl5HoichXibqc4eAqzlPfb}sv^t*O5_X}sw{}{Ny>K5 zx$C^X`Y2x)Xb;7_;WkdL^3+7A7B<>K$w5(StFLQqiO2%$W@FJvh;3}*DLG1e+i7o; z+)&pfR<+sZRs&zkvy?>fQ|DYMPdO^DbcV^wlz(#uq&#(s*(&t z#Wh)xc~zV`l1K?%KmX#og`r5i#0bFoV#;s7Y7u!x=0Vy^b1ecdNc7a1mr{ndU*aT{ z@(j<*6xiv+C@rISd#O!N2xss!-r%TM?F27D!;8gLAp4B&;A z9|OG?0jU9Bb9pmF6oE$@@N2%-yc$|4@j_3&II2a!7sRifF$;6d=R+G0;wvw2J{80x zGZuc1BaHWIpoZ|0%bWLwaPhL1sp$}Y|MjC6@A8FM)0d)dWNtoQFsFU9IrW>(#3km@ zFJ287==MwObIka~=D9MbTw~q9MzuA1rTW#)EO+Ji2{ZK_P z7KzX`5gkL3om|ZtT~(5`BG@aeZh&Q8($OTB4`g$Ja}(g1b7#P_i!; zn(VOuhokVT2x~S<5_5)bwy$0G|ML*JT=P-fKUNRlHWc3v}FVzdz;ur6p!=0r<6@C7acMa>f%W&;r!n;K&mRHddm%CXkq=LS z%AeZD@RCpdRW);$f=4nSvgAI9OxyL{3Dd|0kxYpE<$Vx&RsAa;ncgK}d@UUF*a65a zx^~Arux1Q0R~~DDz3G0)oELZ?2AmH-=KNDTaWi)zKn~fAeO77&y!A2kkdwwzl9j$a`>xdFHlkkoA!LG>38b9nXN*UfGs}#o@^G zW~@s;lAe|3Omz{4eFy#~PNh`ZLf9?nPpqF>!$=1lR=LZaUE+#DCyRwpIe4;^IGq$Z z8J@KTgC=E}hi4m_(>Pv+r(Qt^TOPJg<10@Ld|>uS;L`;#o_&wl$MI!%txh$IMKHY( zu6ox5+&X)!?*Z`HgnbW|@j$l^+$}pEbSQ%sR@!Gmk=T<|_x^2I0rxItth2 zl>_(W%nttQ+V=rHWgo*<9grO0_fz&gxL|$hv}w4&qC7a`6F-BRlqS#An%>gn>ArGs z?_r2?>I-ea^)C?RI=EyzaQ+;kYzyDK9%MZZWnNk_-C*VxgTJ6~>a&kSnYrUPbel#I z*o8slPmavUe8>HVOz+C%3&Z8$<|iPM9dGFajwc~tngaSN71FCGCM2mv!{%Do&!tON|}=*H)O8zsiD zMZ@jI;C+xmd)nzkOMLtg}v>!XAw;+q`0cc9Bq&s z1<#%w4^P+%JfxDk&S(myQ39(grm=pu&Ze@6d$C2|e|r*i%p;Crmcv&)4~njWfiQ>b zdcmS>7|5DCpFR#!=M!VN@c2#7n7Ogp0y{IGScex=yZ>k!?Xv|2u7<4n2a2mf$OC7UU5pG3OndyC$8)+Id*+De)-ipow?T+xrGUJwqJB`B$&`uGyOarx(K{Yh;Oo zIR8(*3(THg@a7Wu#KRHeIJflK2D7*a{4y7&Cc_gOF{kb0F0hw}`%(H(+XtXhfYYRh zK8=7P5r(?#iR1|QFOite%3fFWBeSRL%Uk)GyM3@+CdM)6vhLI1xC}SfWuWuG&N2wg ze*W1Tf%7{Mw)l+D1c-77<5sOG1-~hWuqAD;JZSc)U&3at_yM|_IEZ?VnBdTA$jW|Z z(J9ce2C|Sp-ZU{;NsQ*;C68`zGkdn3zsm>UK@Gwozz9TX|&j^93I_6*E6nElt!i1hcB Zz%1^cZ7@42q7wXOJyDF6-1R-{{{h72SJ(gm delta 6334 zcmds*dwf%6n#XfaPA;dkO@S&>C=F1ALQiro@3}#Nq)pQ#ZEj7{rUiwQe+z-9cxRy3XwAdWC&U2S;qjb(a6O zfAo^)eZKGW{@&+(-ly-@BlET%nYVog=Dl%d4dy+t`twah*{lO4xz(LJh!VpyhMTgd zh5A{=4t9+c70j1!*V_sw84`L0H(Or>@7!$#x&P=MB&J-sq$HQ? zx!2e@H;{An?jQ=X>z3r+>ODmW=_gm_p1$fRQF8ljQ*$?M9x0mdcSw4h^{*6tFQPov zH%Js_rs=t^{(VI8o#E$l_ANVzs-W}D+>tG(h~hM7hhCKEW`luig|{)$NadZ(9&l4 zIcXqu@{_y=CdKmi7tjTd*!pa1upYx3hU3_BY)NjiJu=*R1mi5^lDU>?6&2WZ4BY>( z*e~GKzrtv^^`yB7I!>6T55IQ|JHom+DH&?1t!tGVYEvV(%oDS zWTBe&&@BN$6hJbXh`T*>YkjN7@AK7(HSv1*-e;KI6^qmd8@)_(4PTRVvrWEeq+Sj% zk`RoAg+M?O6OEETDf$`$kpPoys^t<~qa;V_8pTkfKgK*7Z*IB&f^Fto$-#WBh@3({jJ%USRHAV}we^Ccb^!r$UtpDd(z!n5?O|(L zWtOkw>KojxYaEiA?&)%F%*0~Jeznu%4u|7FX!6E3_oSVT&c2oy{PC!(ytbyVwuf~@ zvRbw`l1eo6MN(pTV>ILJ?`1U6)8q+tc{Z+zM>i&fPy^E+YTr!vzz@H`m;u%=ib4|; z@V0uBJRJ~de@)!$X^OOjYnwnQ7>L(32c+6SlnXWSv1X}0=@ugHL{ek|b&|x%u`7s4 zhBDnWpEOvlRxPV`t?lVjv*oN(&amZjYoNB>r#RO+*0#0B+Wg(^tlt@SWYX2~mWZZq zSW8b00Xace2f#Y6GsMRF7?4unp)=Tg(8hAHx;05bxh~pK!};LUvzXwLM6R0gNBxbgH&pFO#KK&Rjs+XSHA%6V_O$X+ z5Jesevm)Q>s|CCqa(e?oCR|$=Y6yZxpAX(}7MsZ{iJrvz_5ja!b8Q_$%jR&B@7kcM z&0#OSrZ3sv5o`(9U*&Cxgi@QL8`4cZeoz0J@bK$rF^Ao1qY9VXb`@SfTx7>V;lRbF zc783P7hwPA)V7?@?PnbLTGz#HILtb6{-QM=e#eQI+4GJYY(utAVfZBD*iRE~Mt_qyezyywhhD%xD_jtiXdferNq+#{GA%q; zTjK{bsF9i@Y$z%;#l>i>iD?p|@QIW7Jm@=#hZjU+3E3O3kF(KcZ)?yW^|plSnt`A8 z#yzOSv9-bahIl0AZ5&1}A2_3eAkj*O6)2wJG8E8Qj#AVVpg1W-i;|RPc#+Mjn>rRN znYMN9-D+2NcgF^`ouwH-(IUmlcA8nt(u--}06Y*RFjQoG!&d%nu-oj$Av<9%GQVlC z-Cg+Rx#WQlE+g{G(=rg#EYDGbAfzcE3MncDQZkiMfSgG&ESr%t@U3M8Kfp@%K+i_| zXgGG3UCePP7CS|wuQ`?WIo%zrGD^2PXR&=wu%o@we!*z&g3;G}xgyEVNMj}k12`ZE zNz7_UV5HM+RoM(+BuSx!3@=h3qb^fWV4_Cxn#5~7r)mnr=WUtO)zNlt(q3g_k2+^! zG$%^@@Y&_Wb50h?IBrtB)aWKBjA_iVgLv2${5Cq_^Nk5|T;LPLJ+zhBY#We4ie*!b zLNPp-qJS)mlp@ePC9!mx6=_Cg0BgjZxYd9qNK`HJQVcDj?nt3DhCzFk(EydwXh5;Z zK?W=mgiGN@C(;l(UqXlXB%#Vohj`EBq z^C?;4sf?ON*_la8R7zweiqX=%MvIKh2omh9GctTfkG)_3J58nuq@zUjxtr3bBdBqBZVkNQfLY&yhh0=G>TyuAtR|7nUh%f31wPT zuA!*_t%#JWo(ogpDTP)Tij&a?tc)U&Xy|j8+~qQdmZOS*QdlGo04hx>+-T}?GK=Kn z0Ve~v-(i|r&H})3Da2G1TBZOBn?gfU6eCKsB*-EFX%UV%Oulk~0U8ntMVCSo0|n6~ z6$zkZG8`+YEDcxzwm40T=F*}fr&7qGr3DH|j7CX70aS*gH98}pR6u>>Jx&u}o@QtU zsjE>Lc2p#l2b7F_DV7n?RFVJ`Mj4__cb#34jJj(=;Ygf`OOfV4q&46XVqssMB-g|w z5Q{`&LQ<~vxtn4_qrWch3pTd$VSja$=e<<*QJs=SKiRM6UK7X6l1Sqg1(3ZP77(%SIh4s-a*;;)`t6pDn@l2DQHXC^Q3P!=Y6x8yudv0<<)SWZoR-h&TP5BJWgiw1?F)wo4#(|Fm@T9 zpPabBK2B`oH=8fK)y8=bJ!pCAfu)gVxjtImPzQX8dT%`JT{POa@)U`$?@F)j$Yxy` zwYOVIZB!lUjy9gIX>O?owY-Pn>m;USXtnjXws~Ji;8TqhId14C>&n^xKc8ygP5*3N zPt0BM=$~)dX>BM+g^NiEDo@c0J=)}P=n_LEiBeK&QOl@2tH}!7w9|T;z;7$R0j|E& zdarp(Ymr`B4qvpARj^@~^$JT-^G|-EYja^INiKyWyR3UGQ&zlwSohA!4PIRZU)gQl zMu^*gI{;VRWgQ_Vd#>t%kv)jGr2XA1q3dqzK1;>k@BNEjItv~sK*-8_5YqP2W=Ypd zVYCn-pS=emOKv`NK=;msgDcUPNA@D-6zZu#xP0Fz^NRhiLHb_AT+p)32JQD*_Yn*? z;{<&6KI;x*TJej2f^CD=Q-EI!DU0^PvilJOFZ=!{u;>Bn zX;5_dH%}L!m#L|SPJ_)>=qgoSbiG8(e+Bdb+5L z${x@w#JsULzxl@86I~qq(Lw8aWAXP7!Y>Y@W!N6xeJi~EA+(I*${jOxFBn}$6+HSd zl7ej6wFrhELCDL0_q!*cYY0h^_vf9*AU%wvuu*$IhHnmAcTArA^y2|tb)J6|EH&l^ z`Z;*`kadrF%V+QCUPtb9whHd~DMHS9uJ|M9d=w$gzg&9)+8;wm@TqYK7X1wE*rhAx z@6$^c!ao{O2%rB9?bwWwclYaB1-y|!+TMN`?O4%2y`R&)3v$1@stR8JI6_t|xwRSE zoPLM8hQ!!aX^qMT2pyil{7)-WZl~^8rRLQlG}&{ zKd(6hS7Rg!>yCkWFp86>iRtls2uhYHW zb72gb5JF!5o!5K~|K z$IEr^;<5DYwIRmRU4aj8DI#O@dGrsB!k`9CR8Yr@GDo1KR7OHAm8Jqo(J~Sv>e{NY z<277F?jfp1RuORZWb!D1-%#a-6;n_k3(Ftd21}-*K)Cl#JPq@T$$gd?zifL+FKrz= zS05FV=%^mLzFXH?#?BSI1hI%SGtAI+DPk3bx3G|$hFHX`>m2aYX^2(k`tL^g?q!H& z%KRX#m&V}>_gSmptJ4unzWVLEbS(xq4O*ANt4a{7bkCvNpnV2nP5;ep+u$E&AQsiO z6@#zOM69A%@iF*JDPk@BqaB_15PWMGv4&ta~Wb4efN$PLw0iI zWCFZuZh@;A@@?Z!Zm>f?OCGfl_B~JN?H%y+43un*9J${zSuZ`Vx3$AJXQEGP06A#k zr|tYeU)TnpEk!BH^5lAg%l}{j+{vSTSwtTB6uJcR6aiW;bHWM{gK{4G^hm*4sNUU*~$iXrdi?Ki@G zE0IV`#cOu!?L9Dm8j|GT6-cC6|0(tBZQbyT(NA_?iJ0`u*S6>jyWq%WNTjVa>4Gn< MA{P?No{u#D4?#ajMgRZ+ From 61cb9d46f72eacdbf8072ce8a5406d60329d3180 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 21 Aug 2014 15:12:43 -0400 Subject: [PATCH 04/57] Fix some of the tools --- tools/emailinvoice.py | 2 +- tools/sendconfirmemail.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/emailinvoice.py b/tools/emailinvoice.py index 63a5bd712..e9c9d0861 100644 --- a/tools/emailinvoice.py +++ b/tools/emailinvoice.py @@ -1,4 +1,4 @@ -from app import stripe +import stripe from app import app from util.invoice import renderInvoiceToHtml diff --git a/tools/sendconfirmemail.py b/tools/sendconfirmemail.py index e9333a181..94345c573 100644 --- a/tools/sendconfirmemail.py +++ b/tools/sendconfirmemail.py @@ -1,4 +1,3 @@ -from app import stripe from app import app from util.useremails import send_confirmation_email From 8866b881dba44b97b789556e27791b3c0a3044ef Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 21 Aug 2014 17:44:56 -0400 Subject: [PATCH 05/57] Remove all license code --- .dockerignore | 4 +-- app.py | 10 -------- license.py | 13 ---------- license.pyc | Bin 895 -> 0 bytes tools/createlicense.py | 38 ---------------------------- util/expiration.py | 55 ----------------------------------------- 6 files changed, 2 insertions(+), 118 deletions(-) delete mode 100644 license.py delete mode 100644 license.pyc delete mode 100644 tools/createlicense.py delete mode 100644 util/expiration.py diff --git a/.dockerignore b/.dockerignore index fcc890f76..40ff6c49f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,11 +1,11 @@ conf/stack screenshots +tools test/data/registry venv .git .gitignore Bobfile README.md -license.py requirements-nover.txt -run-local.sh +run-local.sh \ No newline at end of file diff --git a/app.py b/app.py index 78746fbcf..92a2dacc1 100644 --- a/app.py +++ b/app.py @@ -21,7 +21,6 @@ from data.billing import Billing from data.buildlogs import BuildLogs from data.queue import WorkQueue from data.userevent import UserEventsBuilderModule -from license import load_license from datetime import datetime @@ -50,15 +49,6 @@ else: environ_config = json.loads(os.environ.get(OVERRIDE_CONFIG_KEY, '{}')) app.config.update(environ_config) - logger.debug('Applying license config from: %s', LICENSE_FILENAME) - try: - app.config.update(load_license(LICENSE_FILENAME)) - except IOError: - raise RuntimeError('License file %s not found; please check your configuration' % LICENSE_FILENAME) - - if app.config.get('LICENSE_EXPIRATION', datetime.min) < datetime.utcnow(): - raise RuntimeError('License has expired, please contact support@quay.io') - features.import_features(app.config) Principal(app, use_sessions=False) diff --git a/license.py b/license.py deleted file mode 100644 index b45d90cf8..000000000 --- a/license.py +++ /dev/null @@ -1,13 +0,0 @@ -import pickle - -from Crypto.PublicKey import RSA - -n = 24311791124264168943780535074639421876317270880681911499019414944027362498498429776192966738844514582251884695124256895677070273097239290537016363098432785034818859765271229653729724078304186025013011992335454557504431888746007324285000011384941749613875855493086506022340155196030616409545906383713728780211095701026770053812741971198465120292345817928060114890913931047021503727972067476586739126160044293621653486418983183727572502888923949587290840425930251185737996066354726953382305020440374552871209809125535533731995494145421279907938079885061852265339259634996180877443852561265066616143910755505151318370667L -e = 65537L - -def load_license(license_path): - decryptor = RSA.construct((n, e)) - with open(license_path, 'rb') as encrypted_license: - decrypted_data = decryptor.encrypt(encrypted_license.read(), 0) - - return pickle.loads(decrypted_data[0]) diff --git a/license.pyc b/license.pyc deleted file mode 100644 index 83687adfa9c32d46b214c3197998c215e4e9fcfa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 895 zcma)4Z%9*76hH62ZMxF@mtm2!%-Mu=ff&m~C(0!MjC{kCh3WR*lWjit=6iR`)eK}3 z1c803AkmNIhbW4IC?bpKOC%H&8b%pX6n&}>iS*~(=c2D&-ud10&OPUM&OPVy-*PwI z&b{dqA+reXZWO%^LBfv%1;D?d6Hqu9A>b##Nj&&@kWAn=fn_tuMyRY{tU1zG= z6Q6?9vZzcIxg9}$d9={m<620>`3cvJtq+@4o1qZF*ea z(ppgQF0o?2MEQ}n+J2k8b4@*FjRv@zFpLNeD}ul@ z$}~}p>Qsx1l(MR#2FjQgK#j>!GvtiL$xJ!s4(ZQ!}cH8jy4c3!ype zH!R9gT^@)9Xvjz*$ws`Y(E>cqu*uRu#*uD8YsLqyQh){=XaNIs8*0uTUDkAGL>EJ< zra4FBfuu7NToX%fB*hy35@utF&H%V#u8^}4@TbjV4g8G`Y6 i*(C&4kCFTz{7)H-zhOj`;)>kJdTEr9L-L7*vyeZS$mU)E diff --git a/tools/createlicense.py b/tools/createlicense.py deleted file mode 100644 index 53700d4f4..000000000 --- a/tools/createlicense.py +++ /dev/null @@ -1,38 +0,0 @@ -import argparse -import pickle - -from Crypto.PublicKey import RSA -from datetime import datetime, timedelta - -def encrypt(message, output_filename): - private_key_file = 'conf/stack/license_key' - with open(private_key_file, 'r') as private_key: - encryptor = RSA.importKey(private_key) - - encrypted_data = encryptor.decrypt(message) - - with open(output_filename, 'wb') as encrypted_file: - encrypted_file.write(encrypted_data) - -parser = argparse.ArgumentParser(description='Create a license file.') -parser.add_argument('--users', type=int, default=20, - help='Number of users allowed by the license') -parser.add_argument('--days', type=int, default=30, - help='Number of days for which the license is valid') -parser.add_argument('--warn', type=int, default=7, - help='Number of days prior to expiration to warn users') -parser.add_argument('--output', type=str, required=True, - help='File in which to store the license') - -if __name__ == "__main__": - args = parser.parse_args() - print ('Creating license for %s users for %s days in file: %s' % - (args.users, args.days, args.output)) - - license_data = { - 'LICENSE_EXPIRATION': datetime.utcnow() + timedelta(days=args.days), - 'LICENSE_USER_LIMIT': args.users, - 'LICENSE_EXPIRATION_WARNING': datetime.utcnow() + timedelta(days=(args.days - args.warn)), - } - - encrypt(pickle.dumps(license_data, 2), args.output) diff --git a/util/expiration.py b/util/expiration.py deleted file mode 100644 index 3a58885c9..000000000 --- a/util/expiration.py +++ /dev/null @@ -1,55 +0,0 @@ -import calendar -import sys - -from email.utils import formatdate -from apscheduler.schedulers.background import BackgroundScheduler -from datetime import datetime, timedelta - -from data import model - - -class ExpirationScheduler(object): - def __init__(self, utc_create_notifications_date, utc_terminate_processes_date): - self._scheduler = BackgroundScheduler() - self._termination_date = utc_terminate_processes_date - - soon = datetime.now() + timedelta(seconds=1) - - if utc_create_notifications_date > datetime.utcnow(): - self._scheduler.add_job(model.delete_all_notifications_by_kind, 'date', run_date=soon, - args=['expiring_license']) - - local_notifications_date = self._utc_to_local(utc_create_notifications_date) - self._scheduler.add_job(self._generate_notifications, 'date', - run_date=local_notifications_date) - else: - self._scheduler.add_job(self._generate_notifications, 'date', run_date=soon) - - local_termination_date = self._utc_to_local(utc_terminate_processes_date) - self._scheduler.add_job(self._terminate, 'date', run_date=local_termination_date) - - @staticmethod - def _format_date(date): - """ Output an RFC822 date format. """ - if date is None: - return None - return formatdate(calendar.timegm(date.utctimetuple())) - - @staticmethod - def _utc_to_local(utc_dt): - # get integer timestamp to avoid precision lost - timestamp = calendar.timegm(utc_dt.timetuple()) - local_dt = datetime.fromtimestamp(timestamp) - return local_dt.replace(microsecond=utc_dt.microsecond) - - def _generate_notifications(self): - for user in model.get_active_users(): - model.create_unique_notification('expiring_license', user, - {'expires_at': self._format_date(self._termination_date)}) - - @staticmethod - def _terminate(): - sys.exit(1) - - def start(self): - self._scheduler.start() From d2880807b2369b8da4be20c7246f79fe4768ae3c Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 21 Aug 2014 19:21:20 -0400 Subject: [PATCH 06/57] - Further fixes for license stuff - Small fixes to ensure Quay works for Postgres --- data/database.py | 4 ++- ...670cbeced_migrate_existing_webhooks_to_.py | 8 ++--- .../5a07499ce53f_set_up_initial_database.py | 4 +-- data/model/legacy.py | 3 +- endpoints/api/superuser.py | 18 ----------- requirements-nover.txt | 1 + requirements.txt | 1 + static/js/controllers.js | 30 +------------------ static/partials/super-user.html | 16 +--------- test/test_api_security.py | 2 +- test/test_api_usage.py | 2 +- 11 files changed, 16 insertions(+), 73 deletions(-) diff --git a/data/database.py b/data/database.py index 76a0af9df..349ad1b58 100644 --- a/data/database.py +++ b/data/database.py @@ -17,6 +17,8 @@ SCHEME_DRIVERS = { 'mysql': MySQLDatabase, 'mysql+pymysql': MySQLDatabase, 'sqlite': SqliteDatabase, + 'postgresql': PostgresqlDatabase, + 'postgresql+psycopg2': PostgresqlDatabase, } db = Proxy() @@ -32,7 +34,7 @@ def _db_from_url(url, db_kwargs): if parsed_url.username: db_kwargs['user'] = parsed_url.username if parsed_url.password: - db_kwargs['passwd'] = parsed_url.password + db_kwargs['password'] = parsed_url.password return SCHEME_DRIVERS[parsed_url.drivername](parsed_url.database, **db_kwargs) diff --git a/data/migrations/versions/47670cbeced_migrate_existing_webhooks_to_.py b/data/migrations/versions/47670cbeced_migrate_existing_webhooks_to_.py index 6f516e9b9..726145167 100644 --- a/data/migrations/versions/47670cbeced_migrate_existing_webhooks_to_.py +++ b/data/migrations/versions/47670cbeced_migrate_existing_webhooks_to_.py @@ -20,12 +20,12 @@ def get_id(query): def upgrade(): conn = op.get_bind() - event_id = get_id('Select id From externalnotificationevent Where name="repo_push" Limit 1') - method_id = get_id('Select id From externalnotificationmethod Where name="webhook" Limit 1') + event_id = get_id('Select id From externalnotificationevent Where name=\'repo_push\' Limit 1') + method_id = get_id('Select id From externalnotificationmethod Where name=\'webhook\' Limit 1') conn.execute('Insert Into repositorynotification (uuid, repository_id, event_id, method_id, config_json) Select public_id, repository_id, %s, %s, parameters FROM webhook' % (event_id, method_id)) def downgrade(): conn = op.get_bind() - event_id = get_id('Select id From externalnotificationevent Where name="repo_push" Limit 1') - method_id = get_id('Select id From externalnotificationmethod Where name="webhook" Limit 1') + event_id = get_id('Select id From externalnotificationevent Where name=\'repo_push\' Limit 1') + method_id = get_id('Select id From externalnotificationmethod Where name=\'webhook\' Limit 1') conn.execute('Insert Into webhook (public_id, repository_id, parameters) Select uuid, repository_id, config_json FROM repositorynotification Where event_id=%s And method_id=%s' % (event_id, method_id)) diff --git a/data/migrations/versions/5a07499ce53f_set_up_initial_database.py b/data/migrations/versions/5a07499ce53f_set_up_initial_database.py index 23aaf506a..ffc9d28e6 100644 --- a/data/migrations/versions/5a07499ce53f_set_up_initial_database.py +++ b/data/migrations/versions/5a07499ce53f_set_up_initial_database.py @@ -203,7 +203,7 @@ def upgrade(): sa.Column('id', sa.Integer(), nullable=False), sa.Column('user_id', sa.Integer(), nullable=False), sa.Column('service_id', sa.Integer(), nullable=False), - sa.Column('service_ident', sa.String(length=255, collation='utf8_general_ci'), nullable=False), + sa.Column('service_ident', sa.String(length=255), nullable=False), sa.ForeignKeyConstraint(['service_id'], ['loginservice.id'], ), sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), sa.PrimaryKeyConstraint('id') @@ -375,7 +375,7 @@ def upgrade(): sa.Column('command', sa.Text(), nullable=True), sa.Column('repository_id', sa.Integer(), nullable=False), sa.Column('image_size', sa.BigInteger(), nullable=True), - sa.Column('ancestors', sa.String(length=60535, collation='latin1_swedish_ci'), nullable=True), + sa.Column('ancestors', sa.String(length=60535), nullable=True), sa.Column('storage_id', sa.Integer(), nullable=True), sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], ), sa.ForeignKeyConstraint(['storage_id'], ['imagestorage.id'], ), diff --git a/data/model/legacy.py b/data/model/legacy.py index 9feea0738..f415a9d38 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -69,8 +69,7 @@ class TooManyUsersException(DataModelException): def is_create_user_allowed(): - return get_active_user_count() < config.app_config['LICENSE_USER_LIMIT'] - + return True def create_user(username, password, email): """ Creates a regular user, if allowed. """ diff --git a/endpoints/api/superuser.py b/endpoints/api/superuser.py index 3ade5f1ed..5a117289b 100644 --- a/endpoints/api/superuser.py +++ b/endpoints/api/superuser.py @@ -42,24 +42,6 @@ class SuperUserLogs(ApiResource): abort(403) -@resource('/v1/superuser/seats') -@internal_only -@show_if(features.SUPER_USERS) -@hide_if(features.BILLING) -class SeatUsage(ApiResource): - """ Resource for managing the seats granted in the license for the system. """ - @nickname('getSeatCount') - def get(self): - """ Returns the current number of seats being used in the system. """ - if SuperUserPermission().can(): - return { - 'count': model.get_active_user_count(), - 'allowed': app.config.get('LICENSE_USER_LIMIT', 0) - } - - abort(403) - - def user_view(user): return { 'username': user.username, diff --git a/requirements-nover.txt b/requirements-nover.txt index c0979629b..45a086a8d 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -32,5 +32,6 @@ raven python-ldap pycrypto logentries +psycopg2 git+https://github.com/DevTable/aniso8601-fake.git git+https://github.com/DevTable/anunidecode.git diff --git a/requirements.txt b/requirements.txt index 090ade690..4afd0e97b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,6 +44,7 @@ python-dateutil==2.2 python-ldap==2.4.15 python-magic==0.4.6 pytz==2014.4 +psycopg2==2.5.3 raven==5.0.0 redis==2.10.1 reportlab==2.7 diff --git a/static/js/controllers.js b/static/js/controllers.js index aa18c1b40..6c383d5d2 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -2699,35 +2699,7 @@ function SuperUserAdminCtrl($scope, ApiService, Features, UserService) { }, ApiService.errorDisplay('Cannot delete user')); }; - var seatUsageLoaded = function(usage) { - $scope.usageLoading = false; - - if (usage.count > usage.allowed) { - $scope.limit = 'over'; - } else if (usage.count == usage.allowed) { - $scope.limit = 'at'; - } else if (usage.count >= usage.allowed * 0.7) { - $scope.limit = 'near'; - } else { - $scope.limit = 'none'; - } - - if (!$scope.chart) { - $scope.chart = new UsageChart(); - $scope.chart.draw('seat-usage-chart'); - } - - $scope.chart.update(usage.count, usage.allowed); - }; - - var loadSeatUsage = function() { - $scope.usageLoading = true; - ApiService.getSeatCount().then(function(resp) { - seatUsageLoaded(resp); - }); - }; - - loadSeatUsage(); + $scope.loadUsers(); } function TourCtrl($scope, $location) { diff --git a/static/partials/super-user.html b/static/partials/super-user.html index 64b043331..bc21f5c94 100644 --- a/static/partials/super-user.html +++ b/static/partials/super-user.html @@ -8,9 +8,6 @@
    @@ -19,19 +16,8 @@
    - -
    - - -
    -
    - Seat Usage -
    - -
    - -
    +
    {{ usersError }} diff --git a/test/test_api_security.py b/test/test_api_security.py index 5b3e5612d..7f70d0af6 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -37,7 +37,7 @@ from endpoints.api.repository import RepositoryList, RepositoryVisibility, Repos from endpoints.api.permission import (RepositoryUserPermission, RepositoryTeamPermission, RepositoryTeamPermissionList, RepositoryUserPermissionList) -from endpoints.api.superuser import SuperUserLogs, SeatUsage, SuperUserList, SuperUserManagement +from endpoints.api.superuser import SuperUserLogs, SuperUserList, SuperUserManagement try: diff --git a/test/test_api_usage.py b/test/test_api_usage.py index c91005c5c..b113f27d0 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -40,7 +40,7 @@ from endpoints.api.organization import (OrganizationList, OrganizationMember, from endpoints.api.repository import RepositoryList, RepositoryVisibility, Repository from endpoints.api.permission import (RepositoryUserPermission, RepositoryTeamPermission, RepositoryTeamPermissionList, RepositoryUserPermissionList) -from endpoints.api.superuser import SuperUserLogs, SeatUsage, SuperUserList, SuperUserManagement +from endpoints.api.superuser import SuperUserLogs, SuperUserList, SuperUserManagement try: app.register_blueprint(api_bp, url_prefix='/api') From b51022c73945eedd70df4db70f1d8869b10c9f5e Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 21 Aug 2014 20:36:11 -0400 Subject: [PATCH 07/57] Add support for parsing YAML override config, in addition to Python config --- app.py | 42 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index 92a2dacc1..81c59a30c 100644 --- a/app.py +++ b/app.py @@ -1,8 +1,9 @@ import logging import os import json +import yaml -from flask import Flask +from flask import Flask as BaseFlask, Config as BaseConfig from flask.ext.principal import Principal from flask.ext.login import LoginManager from flask.ext.mail import Mail @@ -24,7 +25,34 @@ from data.userevent import UserEventsBuilderModule from datetime import datetime -OVERRIDE_CONFIG_FILENAME = 'conf/stack/config.py' +class Config(BaseConfig): + """ Flask config enhanced with a `from_yamlfile` method """ + + def from_yamlfile(self, config_file): + with open(config_file) as f: + c = yaml.load(f) + if not c: + logger.debug('Empty YAML config file') + return + + if isinstance(c, str): + raise Exception('Invalid YAML config file: ' + str(c)) + + for key in c.iterkeys(): + if key.isupper(): + self[key] = c[key] + +class Flask(BaseFlask): + """ Extends the Flask class to implement our custom Config class. """ + + def make_config(self, instance_relative=False): + root_path = self.instance_path if instance_relative else self.root_path + return Config(root_path, self.default_config) + + +OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml' +OVERRIDE_CONFIG_PY_FILENAME = 'conf/stack/config.py' + OVERRIDE_CONFIG_KEY = 'QUAY_OVERRIDE_CONFIG' LICENSE_FILENAME = 'conf/stack/license.enc' @@ -42,9 +70,13 @@ else: logger.debug('Loading default config.') app.config.from_object(DefaultConfig()) - if os.path.exists(OVERRIDE_CONFIG_FILENAME): - logger.debug('Applying config file: %s', OVERRIDE_CONFIG_FILENAME) - app.config.from_pyfile(OVERRIDE_CONFIG_FILENAME) + if os.path.exists(OVERRIDE_CONFIG_PY_FILENAME): + logger.debug('Applying config file: %s', OVERRIDE_CONFIG_PY_FILENAME) + app.config.from_pyfile(OVERRIDE_CONFIG_PY_FILENAME) + + if os.path.exists(OVERRIDE_CONFIG_YAML_FILENAME): + logger.debug('Applying config file: %s', OVERRIDE_CONFIG_YAML_FILENAME) + app.config.from_yamlfile(OVERRIDE_CONFIG_YAML_FILENAME) environ_config = json.loads(os.environ.get(OVERRIDE_CONFIG_KEY, '{}')) app.config.update(environ_config) From 5b3514b49cc8e5cc626f34e71a4bd2d956f87757 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 21 Aug 2014 20:38:30 -0400 Subject: [PATCH 08/57] Add missing pyyaml dependency --- requirements-nover.txt | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements-nover.txt b/requirements-nover.txt index 45a086a8d..a3c74e89b 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -33,5 +33,6 @@ python-ldap pycrypto logentries psycopg2 +pyyaml git+https://github.com/DevTable/aniso8601-fake.git git+https://github.com/DevTable/anunidecode.git diff --git a/requirements.txt b/requirements.txt index 4afd0e97b..e454e6846 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ Pillow==2.5.1 PyGithub==1.25.0 PyMySQL==0.6.2 PyPDF2==1.22 +PyYAML==3.11 SQLAlchemy==0.9.7 Werkzeug==0.9.6 alembic==0.6.5 From 2a3094cfdef56718ff7bcd5bda3e3f7ca606da16 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 22 Aug 2014 15:24:56 -0400 Subject: [PATCH 09/57] - Fix zero clipboard integration to properly hide the clipboard controls when flash is not available. - Hide the download .dockercfg link in Safari, since it doesn't work there anyway --- static/css/quay.css | 8 +++ static/directives/copy-box.html | 2 +- static/directives/docker-auth-dialog.html | 2 +- static/js/app.js | 68 +++++++++++++++------- static/js/controllers.js | 27 --------- static/lib/ZeroClipboard.min.js | 17 +++--- static/lib/ZeroClipboard.swf | Bin 1635 -> 4036 bytes static/partials/view-repo.html | 9 +-- 8 files changed, 66 insertions(+), 67 deletions(-) mode change 100755 => 100644 static/lib/ZeroClipboard.min.js mode change 100755 => 100644 static/lib/ZeroClipboard.swf diff --git a/static/css/quay.css b/static/css/quay.css index 01fe84e60..e0f3d2a20 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -2257,6 +2257,14 @@ p.editable:hover i { position: relative; } +.copy-box-element.disabled .input-group-addon { + display: none; +} + +.copy-box-element.disabled input { + border-radius: 4px !important; +} + .global-zeroclipboard-container embed { cursor: pointer; } diff --git a/static/directives/copy-box.html b/static/directives/copy-box.html index 1d996cc31..07dea7407 100644 --- a/static/directives/copy-box.html +++ b/static/directives/copy-box.html @@ -1,4 +1,4 @@ -
    +
    diff --git a/static/directives/docker-auth-dialog.html b/static/directives/docker-auth-dialog.html index dcb71a25b..b7a414725 100644 --- a/static/directives/docker-auth-dialog.html +++ b/static/directives/docker-auth-dialog.html @@ -19,7 +19,7 @@ Download .dockercfg file -
    From 34c6d7f5b4d568ded05f432f80dd3a1701f98cb5 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 22 Aug 2014 16:54:53 -0400 Subject: [PATCH 10/57] Change the auth dialog to copy a full docker login command --- static/directives/docker-auth-dialog.html | 5 +++-- static/js/app.js | 10 +++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/static/directives/docker-auth-dialog.html b/static/directives/docker-auth-dialog.html index b7a414725..33b4af8cd 100644 --- a/static/directives/docker-auth-dialog.html +++ b/static/directives/docker-auth-dialog.html @@ -20,9 +20,10 @@ Download .dockercfg file - + +
    diff --git a/static/js/app.js b/static/js/app.js index 250665f60..aa9d8bcba 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -2383,7 +2383,15 @@ quayApp.directive('dockerAuthDialog', function (Config) { 'shown': '=shown', 'counter': '=counter' }, - controller: function($scope, $element) { + controller: function($scope, $element) { + var updateCommand = function() { + $scope.command = 'docker login -e="." -u="' + $scope.username + + '" -p="' + $scope.token + '" ' + Config['SERVER_HOSTNAME']; + }; + + $scope.$watch('username', updateCommand); + $scope.$watch('token', updateCommand); + $scope.isDownloadSupported = function() { var isSafari = /^((?!chrome).)*safari/i.test(navigator.userAgent); if (isSafari) { From 4140e115e532fcfe57f53faeea8accbfb10fe064 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 22 Aug 2014 18:03:22 -0400 Subject: [PATCH 11/57] Put building behind a feature flag --- config.py | 3 +++ static/partials/repo-admin.html | 5 +++-- static/partials/view-repo.html | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index a903fa29a..3712055d2 100644 --- a/config.py +++ b/config.py @@ -153,6 +153,9 @@ class DefaultConfig(object): # Feature Flag: Whether to support GitHub build triggers. FEATURE_GITHUB_BUILD = False + # Feature Flag: Dockerfile build support. + FEATURE_BUILD_SUPPORT = True + DISTRIBUTED_STORAGE_CONFIG = { 'local_eu': ['LocalStorage', {'storage_path': 'test/data/registry/eu'}], 'local_us': ['LocalStorage', {'storage_path': 'test/data/registry/us'}], diff --git a/static/partials/repo-admin.html b/static/partials/repo-admin.html index b3ef4b51b..6bd329091 100644 --- a/static/partials/repo-admin.html +++ b/static/partials/repo-admin.html @@ -18,7 +18,8 @@
    -
    +
    Build Triggers diff --git a/static/partials/view-repo.html b/static/partials/view-repo.html index 03ce2909c..4f588ccf2 100644 --- a/static/partials/view-repo.html +++ b/static/partials/view-repo.html @@ -18,7 +18,7 @@
    + quay-show="Features.BUILD_SUPPORT && (repo.can_write || buildHistory.length)">
    +
    + + Invoice History + + +
    @@ -48,13 +55,6 @@
    -
    - - Invoice History - - -
    @@ -81,7 +81,7 @@
    -
    +
    @@ -93,9 +93,9 @@
    SSL Encryption
    Robot accounts
    Dockerfile Build
    +
    Invoice History
    Teams
    Logging
    -
    Invoice History
    Free Trial
    diff --git a/static/partials/user-admin.html b/static/partials/user-admin.html index 783c5f87a..1b2ad7fd1 100644 --- a/static/partials/user-admin.html +++ b/static/partials/user-admin.html @@ -25,7 +25,7 @@
  • Billing Options
    • -
  • +
  • Billing History
    • From 09a1c4d2b5d054e2185f5a09b8d5242c1cdd5c2f Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 25 Aug 2014 14:23:21 -0400 Subject: [PATCH 15/57] Add test fix and make sure Quay ups the connection count in its container --- Dockerfile.web | 1 + conf/init/doupdatelimits.sh | 5 +++++ endpoints/index.py | 6 +++++- test/specs.py | 2 +- 4 files changed, 12 insertions(+), 2 deletions(-) create mode 100755 conf/init/doupdatelimits.sh diff --git a/Dockerfile.web b/Dockerfile.web index 56b126d53..448a7f748 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -30,6 +30,7 @@ RUN cd grunt && npm install RUN cd grunt && grunt ADD conf/init/svlogd_config /svlogd_config +ADD conf/init/doupdatelimits.sh /etc/my_init.d/ ADD conf/init/preplogsdir.sh /etc/my_init.d/ ADD conf/init/runmigration.sh /etc/my_init.d/ diff --git a/conf/init/doupdatelimits.sh b/conf/init/doupdatelimits.sh new file mode 100755 index 000000000..603559de0 --- /dev/null +++ b/conf/init/doupdatelimits.sh @@ -0,0 +1,5 @@ +#! /bin/bash +set -e + +# Update the connection limit +sysctl -w net.core.somaxconn=1024 \ No newline at end of file diff --git a/endpoints/index.py b/endpoints/index.py index bf37e14b5..4017d47e9 100644 --- a/endpoints/index.py +++ b/endpoints/index.py @@ -428,7 +428,11 @@ def get_search(): if user is not None: username = user.username - matching = model.get_matching_repositories(query, username) + if query: + matching = model.get_matching_repositories(query, username) + else: + matching = [] + results = [result_view(repo) for repo in matching if (repo.visibility.name == 'public' or ReadRepositoryPermission(repo.namespace, repo.name).can())] diff --git a/test/specs.py b/test/specs.py index 33db0493e..8749a025e 100644 --- a/test/specs.py +++ b/test/specs.py @@ -196,7 +196,7 @@ def build_index_specs(): IndexTestSpec(url_for('index.put_repository_auth', repository=PUBLIC_REPO), NO_REPO, 501, 501, 501, 501).set_method('PUT'), - IndexTestSpec(url_for('index.get_search'), NO_REPO, 501, 501, 501, 501), + IndexTestSpec(url_for('index.get_search'), NO_REPO, 200, 200, 200, 200), IndexTestSpec(url_for('index.ping'), NO_REPO, 200, 200, 200, 200), From 99d75bede763f0f4daa2ff1f2383699752395366 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 25 Aug 2014 15:30:29 -0400 Subject: [PATCH 16/57] Handle error cases better for external services --- endpoints/api/billing.py | 32 ++++++++++++++++++++++++++------ endpoints/api/subscribe.py | 21 ++++++++++++++++++--- static/js/app.js | 4 ++-- templates/index.html | 11 +++-------- util/analytics.py | 6 +++++- 5 files changed, 54 insertions(+), 20 deletions(-) diff --git a/endpoints/api/billing.py b/endpoints/api/billing.py index 3e13df6b6..c41bcec77 100644 --- a/endpoints/api/billing.py +++ b/endpoints/api/billing.py @@ -4,7 +4,7 @@ from flask import request from app import billing from endpoints.api import (resource, nickname, ApiResource, validate_json_request, log_action, related_user_resource, internal_only, Unauthorized, NotFound, - require_user_admin, show_if, hide_if) + require_user_admin, show_if, hide_if, abort) from endpoints.api.subscribe import subscribe, subscription_view from auth.permissions import AdministerOrganizationPermission from auth.auth_context import get_authenticated_user @@ -23,7 +23,11 @@ def get_card(user): } if user.stripe_id: - cus = billing.Customer.retrieve(user.stripe_id) + try: + cus = billing.Customer.retrieve(user.stripe_id) + except stripe.APIConnectionError as e: + abort(503, message='Cannot contact Stripe') + if cus and cus.default_card: # Find the default card. default_card = None @@ -46,7 +50,11 @@ def get_card(user): def set_card(user, token): if user.stripe_id: - cus = billing.Customer.retrieve(user.stripe_id) + try: + cus = billing.Customer.retrieve(user.stripe_id) + except stripe.APIConnectionError as e: + abort(503, message='Cannot contact Stripe') + if cus: try: cus.card = token @@ -55,6 +63,8 @@ def set_card(user, token): return carderror_response(exc) except stripe.InvalidRequestError as exc: return carderror_response(exc) + except stripe.APIConnectionError as e: + return carderror_response(e) return get_card(user) @@ -75,7 +85,11 @@ def get_invoices(customer_id): 'plan': i.lines.data[0].plan.id if i.lines.data[0].plan else None } - invoices = billing.Invoice.all(customer=customer_id, count=12) + try: + invoices = billing.Invoice.all(customer=customer_id, count=12) + except stripe.APIConnectionError as e: + abort(503, message='Cannot contact Stripe') + return { 'invoices': [invoice_view(i) for i in invoices.data] } @@ -228,7 +242,10 @@ class UserPlan(ApiResource): private_repos = model.get_private_repo_count(user.username) if user.stripe_id: - cus = billing.Customer.retrieve(user.stripe_id) + try: + cus = billing.Customer.retrieve(user.stripe_id) + except stripe.APIConnectionError as e: + abort(503, message='Cannot contact Stripe') if cus.subscription: return subscription_view(cus.subscription, private_repos) @@ -291,7 +308,10 @@ class OrganizationPlan(ApiResource): private_repos = model.get_private_repo_count(orgname) organization = model.get_organization(orgname) if organization.stripe_id: - cus = billing.Customer.retrieve(organization.stripe_id) + try: + cus = billing.Customer.retrieve(organization.stripe_id) + except stripe.APIConnectionError as e: + abort(503, message='Cannot contact Stripe') if cus.subscription: return subscription_view(cus.subscription, private_repos) diff --git a/endpoints/api/subscribe.py b/endpoints/api/subscribe.py index dd6de9678..2c3fba359 100644 --- a/endpoints/api/subscribe.py +++ b/endpoints/api/subscribe.py @@ -15,6 +15,9 @@ logger = logging.getLogger(__name__) def carderror_response(exc): return {'carderror': exc.message}, 402 +def connection_response(exc): + return {'message': 'Could not contact Stripe. Please try again.'}, 503 + def subscription_view(stripe_subscription, used_repos): view = { @@ -74,19 +77,29 @@ def subscribe(user, plan, token, require_business_plan): log_action('account_change_plan', user.username, {'plan': plan}) except stripe.CardError as e: return carderror_response(e) + except stripe.APIConnectionError as e: + return connection_response(e) response_json = subscription_view(cus.subscription, private_repos) status_code = 201 else: # Change the plan - cus = billing.Customer.retrieve(user.stripe_id) + try: + cus = billing.Customer.retrieve(user.stripe_id) + except stripe.APIConnectionError as e: + return connection_response(e) if plan_found['price'] == 0: if cus.subscription is not None: # We only have to cancel the subscription if they actually have one - cus.cancel_subscription() - cus.save() + try: + cus.cancel_subscription() + cus.save() + except stripe.APIConnectionError as e: + return connection_response(e) + + check_repository_usage(user, plan_found) log_action('account_change_plan', user.username, {'plan': plan}) @@ -101,6 +114,8 @@ def subscribe(user, plan, token, require_business_plan): cus.save() except stripe.CardError as e: return carderror_response(e) + except stripe.APIConnectionError as e: + return connection_response(e) response_json = subscription_view(cus.subscription, private_repos) check_repository_usage(user, plan_found) diff --git a/static/js/app.js b/static/js/app.js index aa9d8bcba..c718375fe 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -5570,8 +5570,8 @@ quayApp.run(['$location', '$rootScope', 'Restangular', 'UserService', 'PlanServi } } - if (!Features.BILLING && response.status == 402) { - $('#overlicenseModal').modal({}); + if (response.status == 503) { + $('#cannotContactService').modal({}); return false; } diff --git a/templates/index.html b/templates/index.html index f69251514..51d687770 100644 --- a/templates/index.html +++ b/templates/index.html @@ -35,23 +35,18 @@ -{% if not has_billing %} -
    The docker username is {{ username }} and the password is the token below. You may use any value for email.
    -
    + +
    + Regenerating Token... + +
    + +
    +
    -
    +
    + +
    +
    Download .dockercfg file diff --git a/static/directives/robots-manager.html b/static/directives/robots-manager.html index c696937d2..c11c07cf8 100644 --- a/static/directives/robots-manager.html +++ b/static/directives/robots-manager.html @@ -31,7 +31,7 @@
    + shown="!!shownRobot" counter="showRobotCounter" supports-regenerate="true" regenerate="regenerateToken(username)"> {{ shownRobot.name }}
    diff --git a/static/js/app.js b/static/js/app.js index ba8eb8a8e..8844007a6 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -2385,7 +2385,9 @@ quayApp.directive('dockerAuthDialog', function (Config) { 'username': '=username', 'token': '=token', 'shown': '=shown', - 'counter': '=counter' + 'counter': '=counter', + 'supportsRegenerate': '@supportsRegenerate', + 'regenerate': '®enerate' }, controller: function($scope, $element) { var updateCommand = function() { @@ -2396,6 +2398,15 @@ quayApp.directive('dockerAuthDialog', function (Config) { $scope.$watch('username', updateCommand); $scope.$watch('token', updateCommand); + $scope.regenerating = true; + + $scope.askRegenerate = function() { + bootbox.confirm('Are you sure you want to regenerate the token? All existing login credentials will become invalid', function(resp) { + $scope.regenerating = true; + $scope.regenerate({'username': $scope.username, 'token': $scope.token}); + }); + }; + $scope.isDownloadSupported = function() { var isSafari = /^((?!chrome).)*safari/i.test(navigator.userAgent); if (isSafari) { @@ -2421,6 +2432,8 @@ quayApp.directive('dockerAuthDialog', function (Config) { }; var show = function(r) { + $scope.regenerating = false; + if (!$scope.shown || !$scope.username || !$scope.token) { $('#dockerauthmodal').modal('hide'); return; @@ -2661,6 +2674,8 @@ quayApp.directive('logsView', function () { return 'Delete notification of event "' + eventData['title'] + '" for repository {repo}'; }, + 'regenerate_robot_token': 'Regenerated token for robot {robot}', + // Note: These are deprecated. 'add_repo_webhook': 'Add webhook in repository {repo}', 'delete_repo_webhook': 'Delete webhook in repository {repo}' @@ -2704,6 +2719,7 @@ quayApp.directive('logsView', function () { 'reset_application_client_secret': 'Reset Client Secret', 'add_repo_notification': 'Add repository notification', 'delete_repo_notification': 'Delete repository notification', + 'regenerate_robot_token': 'Regenerate Robot Token', // Note: these are deprecated. 'add_repo_webhook': 'Add webhook', @@ -2875,6 +2891,20 @@ quayApp.directive('robotsManager', function () { $scope.shownRobot = null; $scope.showRobotCounter = 0; + $scope.regenerateToken = function(username) { + if (!username) { return; } + + var shortName = $scope.getShortenedName(username); + ApiService.regenerateRobotToken($scope.organization, null, {'robot_shortname': shortName}).then(function(updated) { + var index = $scope.findRobotIndexByName(username); + if (index >= 0) { + $scope.robots.splice(index, 1); + $scope.robots.push(updated); + } + $scope.shownRobot = updated; + }, ApiService.errorDisplay('Cannot regenerate robot account token')); + }; + $scope.showRobot = function(info) { $scope.shownRobot = info; $scope.showRobotCounter++; diff --git a/test/data/test.db b/test/data/test.db index 4d04283311e6feb845dff3fde9a4d370858119be..34882e11701b07d1a0e1ea1631cdc6636489f033 100644 GIT binary patch delta 6232 zcmds*eSB2ana49TH<`If$OH+3KwyZ2keHjfZ|A;*2+7Q3-ZL}FWG0y?kUMkl5FoFU z7v7}dZdbeA?ph`4*>%wc)M{!_1WnqlTGY~At>{))t!=GV7b{BqV7IlaV(Am2R0vhK z{I~gIlHBKfh_%9eHZ=5n^_F?Yfa&TPBO=?zNxRyDWdE z==TEU$*p5V$=sUvNBXyo5-9tl;St;R-Nb^m95-@!`!QnWAU`xRdhHQ2mEM%qyG^+A z67s5%gFE({SC-A6)VqwhbPhbY^J`<~)hnL3POmWFStVr6pW71QYxt=92+~|4{Vg9|{&O zFv*4s4=+EYp7Nfq6_Q-S9h71X^-*Ue?BnBXvdP^Th_*F`>$x_k+bz(Mu-BIq>zfa+)$!zW4A_{$y zqiZwT5LUNE@b~^`SdQH5eXUZPYm>9pzkY)l8S37WMQ$xP6ll_VI{F-(%0-juHom#4 z$0a+Z7B}0erCD5Z+JI{PbqvqRiTYR|>IYib9yI!e%erF%PU%K-DjmHifl%qX!v4NyRe?nXBqzL)HZ*X z+fP>+J8Ca>zj4-Kym*Bte&aBHTy@dP_)K{@27^AU!1!0GGyJ*A_Zg>AxyFlC$Y+=) z8I50-hqf+@`A4SdQ_#iA zX7o%2Mt+%SOTV;w$8R)#9A4paT)f#W{j+0=vb4VZVk8i*??Q3+mb$CKx#+^T)ws#f@TzKWdaetp>YDHoOCp* zA}`P?$5Axm(-bdf6iVhqjp7sziM%2RnxB9e~8Rs|dfkiYT-z9G_8CPNi5j zEm6FdQYlH|WJ=0NYD(c{nd3z~Yc(MZBU|0zdfSvYHkPSoq-svKQ8fINRoco1de*7X z*2}7GWeq*ul{WVCquFyt?>HL}vN7Vcsfy<%MwWHLK<2~^q@G)65fwHg%Obdw)-;OG zY~oW?ij@&1%RHN6=nN8QK7V^zPiiBqy1h(+PX;_m!Ebg4{GnvC@aNU=jIFeBYv4(L zPuCd_hm_7it?V!O!EzEmUP~NxtOm(vcxWs&)sYttU}|E1PEL-|`aq8LXVwQmDN=&glBOw;S#+i%` zqb0`>C*jD;=jarz@Q8Bh}1la!Shx6My_R5!r3$p?$J)hNeU$!&p_}DJcd2@*<;(q9RIa z8sWamT(_Oc$O@g2cuHmwj6^h|BpJyR%CH%lVbU6lQuvk1Tx3@SP7onXisRE9kkSgJ zh%Bt1P6-)^Uepv0vsCU?b{ZjpMH!hAGNMB9DKSM!oG4NZ(l}8{(>$*!c!bJzUe2Z& zNtHB-Qjo?|ydY~-iq`}RGD_wl|F{f?|Ch?Gv}>XQsiy!9T!Ie{(vF@Q2r?)Vrz#u- z_3lKlgBXBdF zl0a3O*SNHh%A{C4S(WRvrx^ips>o3c!-`Wb!0IA=2pXGK;6IgSG@NwgR@gz>6kNPS zI8rnfc2xo-5#ZdfG=zQ99mIC%IxxDC!ObTVgSvtHB?Oux=??*B~VPLb6SAhdptBBpwQj!Inl^ zXl>-fu~@xjMtSfE>Klo)G;!LI_qet8VpkZi&AZKE7Zpa;G7O?LMNLsWt3pH^KTPLT1=pm|s1tN2v5qq!g-N;1cgQ7dG#Un8gcPCMF~1@2(yG{4Rf{VBum@ondA|er7KCVsri% zo5^#`&>*MV*;%11=h$bM&F7eB$eeeMd4|ln=bJa3>7{2ECq8eVDYog0&FA#FnX|_a zn4f&W&o%LqFTu9dJ6l@n8J}?F)I}Ciq;X?^x}ztXtyQ(50VUO`Ri%5nT>c1O&-#;b zs4zz2KW&vS=U891o`6>Q6Ki>ii87pzG|n+mG+%SPhTSE_4n z-D3&aksxK#0?SirXzS1kp#RE>N>ePH>l{lF)y5{@Tf{*mfTxyYUla)S$qy zx|`}saIlypFK05msHkxLMFQN(X^o*24X)@sl2}z?MGo!{xbJ{va`viwzZ2IxrL(Wl zOt^O~$>DtuTE_A^9{v}-Ld28vA@rLL!Uh&U^7a+@j|VNg2|Fz!e8(ZnF|*-`_3gTc zp9-c1|M(#gC@*vENgRC`f+^_joR90qL7>W|ALe0Z!ZJ!&@_diuUrktcn~4V!C3?nz z7c2o89{U~$WAW{~N%vHZoXFPTyT1>piUZ|uVaLA#sz%>CAKQKasG9uxmCDSb@X?3Z2ci1 z=^qsA8b53qsJ%i&oS_z#a38KSk)y=C7dqd>@0rNm1WAk^E zi0bKF9KF2OJUOmWObD2{~%cdH=3|)Jxl@i|m6E za*dgI&=S=%$>}1)bEjCLp677wrNELzbP$vCfaU1u{CoV-JYW@-1&`u)<^!wx!r(pwV1!-AM{LYy0|tk0M@D(=iQHO3xQ=gw5kdpUkI$K?SB`3VCt%q@~08F|dQi;n&@U^n$X9w_aMRKy3T{X5$@9H>PkZ0bE z@NQd|4qEJdMZH<9tmOFN`PItZZCt$nyC%>@!`ez`5J(g{-X3< zy({%+*whPhY+3~{wES;t@S#-zTlwLS7UK5R04v$azB67+4qs-r#EvHQu5D*th2tla z5Yg&q#un-oThF`;#|PUWqJothUd6VnVV-4Wt`i@=8k|@p{dXFVUIQwXeEiOhdiNkM yoClg5TmvczZ@f34cMafArk>on7MPd3xqnix=$}4ecG6@metsQULGbxEaDM{%$U6l9 delta 6114 zcmds*Yj_mbdB^uM((VXJD?m2pB3M9x1e)EsoVkI3U1_!2T}gY)KngAR7I25~FjDc9#1{{glG2puN@uj}T_Xg~?p=liCI%(p%zCd3glK`T& z8@^Q^^yr!YoZorh|9Q_juV%}sSzAuc+BVtf-9Dw-=smLHi}l2`sYfPdS8UitOftM+ z*wJy0nDtb6d-m5Ir-^Z&E%|QtXy;H-!5sMkJySTw5ZBAO>H0jpXP*^i|F!F|dD((V z*=+YAVnXB0K-Sf>izw(=y&(HR&pAR!J+n0X{B5U+Ne}ECpWU%>sA!JgA?X?GpD6l4 zpgg;&pD0XE(6gPrhlt`mp_jAv&AW)o2IraVsmAA+C#Sl1W`bVNId5D(5-c?9>335D^|zq@NYgh+Fj9bT|=XnX|Cq06K=N2 z7YWzN0Y(xUq9Gv=ki>YSb%TI!dS)! z*v8&^XHRo`tT)E3jtL%A>v4n<&RBbgBiSv{y{#Qlu`1ju#~K?}*VGA$+KB|reQB)V z6V5e3Z#P@rDzkhoS6A~^r{;??ocR(geGrvV|U8g=-kv2#XlS# zDzB>ERMX8m!W~*iPdFK`-xN-Yp>>h8v$u!QL{F0^*y&lfG8S1E7lQRnZ?J76-Hre7 zrIG1l{h}x|F#&I@H^I{Zf%aF&yq>0TOQ@y^1sejf+U9^%6NqrZCO+CM)g|0Q*d0%Z zOrTbhI5~PNF~%@W&zVCStX8Ymp?0q7?o>NUS*4U=OXb!;O`A_~u6C@-v_&)it~S>1 z3^~%Ns#r@{Q`fGd#|M!dr>g>JHMb$iMmI4iso>)mjC1%5%SCHfCX_Wy%ld9_x>e@X zP1O#M$oi9hF@~aYN6^>hjicsBz?YJ1Bkk3kuMQVqH1aM<pFJfY5^>R^(fKHHepkZf~H03DwjF>l;v`&xh~1Xq>_;@$UGVwgAs} zahZ0ZWn(D8cdk{{=8%_Oxhc`s-p~@NyUkl44kp(})~1@e{hr>Hp~3eq8Xfjr8&$a2 zws)}5ZW0RnZnU)XYY4r-_-B{4<#KJm;4rOn-ROGb4yWnH9UlCZ)AYIHhMn=5a}7qL zUNFY=XSvgN#m@JcMo>AX8+FKM93h!Z-&BY8Zz{xr6(+0QYTHow&$gf1%Hgzpt+1~) z2NE`#ZnUU~uQ>l#KcZ2!)Es7m5uqt2MxsqjlYmdWZ<>vL@0&tpk!W1@#_D2hq}kir;E#A)g0;=a zPkUn?IGNa*hPwJ#IO=U2Jo&!qV`oGVBw9(c0>v|2nnD`OQHq*G6elHVQIb*&FR~r# z`u6!sI{h#Zz)s{A z<(@Iv9xgm{sV?y2i-`Qvl#Ij_%X5?<2q_AQLXt|Nq)eq1B&U-M%ckWt{$LRyjF;?z z?sfLzXzVOIpW`4DT)vo?+;^qGjROJi^9%-q$@ZAR_L%J*uxOtxQf7`Dzl}qtIp#R|Z4w#SLTt44 z$tcOPNk*Xm}KqcVs!o17-A!5Amuyvj>5v;_T(LTL;GtCZFd zmDFfNu`obJEI7m^O=c%}3ImJ=W1b?VRaHVFC8{i#s&X79sVE7*k{YtGtisWPGm zNvNH)02L{*oQO31Zbc4XDyGtiPiiV9Y7)#3OG!zZqFt&jF(n|GXK2h)ISWf!nbRbw zGl*1WD8wjfN|BL5DPj@{Nu;nW;&7JAshY_tN-6~wQH-R}6jFGNk|8#VVHhDTscD&$ z@TXKxMX3feAzBeBRlO9Zz*7pXFcc@l16GDWBpUl1Iqp)KgLG68Q3?y@Af(ci!VQ-q zC$nH5k2o3O!;YLOr7S`mmjtGw&@zP}Y6@OSQj93kk|2u+r9?dB$nlj54AQ_T2rdZ| zBL(P^3i&JPG{*`mOCwgmEzX>|r7|rlaxw{$mJ%o=F&ZTy1yN~^*66eVwE&&o0Vmj* zVrT~J)u=Q(Y?8_&N`|o%%Lp)+Bp`)R2I!oJE-pz#+||KQIL5@JaC0Es8t@3wkgryf ztD_Q%hQm=IA=miaO;MrIUmNo^G`8{~e^rF%y+Tbmh`G1uc4J+vK z>3C&SlW-ttnLR9XqvT7W^#?6w*O&i2Xo(Cf?5Ked?@c%vvQ%7O-W#%r*Hs3GLY5yd z>$`H5FUOic)$BHoF38LBn6H=fzFsD;lHnHn>V~>Zw_c@>k}X%sqioK-N*-l%&b9K| zk?ZmDcxri+u4qZ!F}UH{6Bt6lq%1nOfD} z(cwy~JzYw2o$5%nXL!20xupu#@E(S*m6(>1tMYAw?Y!-6xGGZb?Nl~pC2*sj=#08A!aUl^7r@cw$_)zNya1vm8WQh9&Y$JxYgi@qLgGx)Y2-? zYO;dY@3x*NOgl^O!7KJy4_PY4wHE0mrT8@~S&8fST5q)!HUH=zbZsWyK#~jb&|d3) z%eW=)J*|6ZWczQg#Bc7iZY9KR|I~+dT+sI6GVFR7kmU!z`}cau zRD7fWkQE02X?uO6q-!NOQV7VG4ghk}y~mH}-YK|$DZKNOgTNd|J=>2L9~x%fdiZTj z{}`BMEn98a{)qJu!Elq$;+Gz=?jj}>zxF$v>9?M-eTS`g6Vu1d zJBX(}3JlY<@BKF}ddzwr6+Qj#^99gojW=vC*ldN~f?M;S&Mz8c&vTNR^-jxK3zO^4 z2@~HjKWMsOYBVO{wW{szn<|`f7-TSzoC2*7v<#HWYH($%NT6g6NtD2H3^Y!>3O6KP z|G0I?!ufV*^o)eRoB;9cIb!WEnnraT(aXiWkl$>T@VCR#7 z%zbgyS!{m_kmxh>0519&tlZ5@<{Z*X=HlO)A%rh}1}ist=)=RhR*tt5kkbQC!^#!? z%|}_?TbBLD+bZ$hCjeQ#;QnT8I|<0*yUpYwf_wD-IR3~? z?jjtfy&vK46XZF|q`Nk**1g_KQ4Hh&LSFpLtJswb2;1y)Q!!%!#Q66+1U%geh$8dC z_wdVBK(Lg{hTBL$5HH;QeckK2RMq$80itB<#J8}03?RyaAH?z4d_a^ueEU*tEC9s# zZMB{F(E>o^J(}i=o}vY$x#?+-9!a?VJIMh zl9Fi&dM!;wlA@(0M%1 zdY1!#HUU`ET>sUGKfDE4IqAO+=_N7z>Lb=l{MJNZ$+!R8L%J5l>-((>@okfURkHv1 zPHdkHtcmaL+=_oa8CX^W6w*{|Y$aBQpAKhceewI9KA?*84>TT`#{A4J$Mvgpe z8LOAPqi5RinJMt38j<}Le!}jL^|=}RQVEnO%adyeF8|{)yqkxGnMa=b47&tiqLy2n zxLkyoD_x)7g?}fKLq){T@vC}f)#Y6>@{JnLl*xXgVC&FU{Bs#rdD_Gi^YE7W0GnEO z>>Rc)0GO=bdjXS`0Fw_>|A}9(1X!7R;5j{${1a^WQ#Cd(0$91p@H{@Y2w=tU=uW(1 zF~CahS^nyPi|m_d$t}JE>6zY3AEoiLEs&9H%AcXn-E`@zG(Or2897ScID+j22LOfB2X=dMmgz^*^LuJ<~OE@a$U#O!|#GTlBe| Xc<2^zX$wud@axOTxx}JZ!p;8!0)7n{ diff --git a/test/test_api_security.py b/test/test_api_security.py index 7f70d0af6..9a3bcfac3 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -14,7 +14,9 @@ from endpoints.api.search import FindRepositories, EntitySearch from endpoints.api.image import RepositoryImageChanges, RepositoryImage, RepositoryImageList from endpoints.api.build import (FileDropResource, RepositoryBuildStatus, RepositoryBuildLogs, RepositoryBuildList) -from endpoints.api.robot import UserRobotList, OrgRobot, OrgRobotList, UserRobot +from endpoints.api.robot import (UserRobotList, OrgRobot, OrgRobotList, UserRobot, + RegenerateOrgRobot, RegenerateUserRobot) + from endpoints.api.trigger import (BuildTriggerActivate, BuildTriggerSources, BuildTriggerSubdirs, TriggerBuildList, ActivateBuildTrigger, BuildTrigger, BuildTriggerList, BuildTriggerAnalyze) @@ -1632,6 +1634,19 @@ class TestOrgRobotBuynlargeZ7pd(ApiTestCase): ApiTestCase.setUp(self) self._set_url(OrgRobot, orgname="buynlarge", robot_shortname="Z7PD") + def test_get_anonymous(self): + self._run_test('GET', 401, None, None) + + def test_get_freshuser(self): + self._run_test('GET', 403, 'freshuser', None) + + def test_get_reader(self): + self._run_test('GET', 403, 'reader', None) + + def test_get_devtable(self): + self._run_test('GET', 400, 'devtable', None) + + def test_put_anonymous(self): self._run_test('PUT', 401, None, None) @@ -1644,6 +1659,7 @@ class TestOrgRobotBuynlargeZ7pd(ApiTestCase): def test_put_devtable(self): self._run_test('PUT', 400, 'devtable', None) + def test_delete_anonymous(self): self._run_test('DELETE', 401, None, None) @@ -3040,6 +3056,19 @@ class TestUserRobot5vdy(ApiTestCase): ApiTestCase.setUp(self) self._set_url(UserRobot, robot_shortname="robotname") + def test_get_anonymous(self): + self._run_test('GET', 401, None, None) + + def test_get_freshuser(self): + self._run_test('GET', 400, 'freshuser', None) + + def test_get_reader(self): + self._run_test('GET', 400, 'reader', None) + + def test_get_devtable(self): + self._run_test('GET', 400, 'devtable', None) + + def test_put_anonymous(self): self._run_test('PUT', 401, None, None) @@ -3052,6 +3081,7 @@ class TestUserRobot5vdy(ApiTestCase): def test_put_devtable(self): self._run_test('PUT', 201, 'devtable', None) + def test_delete_anonymous(self): self._run_test('DELETE', 401, None, None) @@ -3065,6 +3095,42 @@ class TestUserRobot5vdy(ApiTestCase): self._run_test('DELETE', 400, 'devtable', None) +class TestRegenerateUserRobot(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(RegenerateUserRobot, robot_shortname="robotname") + + def test_post_anonymous(self): + self._run_test('POST', 401, None, None) + + def test_post_freshuser(self): + self._run_test('POST', 400, 'freshuser', None) + + def test_post_reader(self): + self._run_test('POST', 400, 'reader', None) + + def test_post_devtable(self): + self._run_test('POST', 400, 'devtable', None) + + +class TestRegenerateOrgRobot(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(RegenerateOrgRobot, orgname="buynlarge", robot_shortname="robotname") + + def test_post_anonymous(self): + self._run_test('POST', 401, None, None) + + def test_post_freshuser(self): + self._run_test('POST', 403, 'freshuser', None) + + def test_post_reader(self): + self._run_test('POST', 403, 'reader', None) + + def test_post_devtable(self): + self._run_test('POST', 400, 'devtable', None) + + class TestOrganizationBuynlarge(ApiTestCase): def setUp(self): ApiTestCase.setUp(self) diff --git a/test/test_api_usage.py b/test/test_api_usage.py index b113f27d0..bd8bb29cd 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -16,7 +16,8 @@ from endpoints.api.tag import RepositoryTagImages, RepositoryTag from endpoints.api.search import FindRepositories, EntitySearch from endpoints.api.image import RepositoryImage, RepositoryImageList from endpoints.api.build import RepositoryBuildStatus, RepositoryBuildLogs, RepositoryBuildList -from endpoints.api.robot import UserRobotList, OrgRobot, OrgRobotList, UserRobot +from endpoints.api.robot import (UserRobotList, OrgRobot, OrgRobotList, UserRobot, + RegenerateUserRobot, RegenerateOrgRobot) from endpoints.api.trigger import (BuildTriggerActivate, BuildTriggerSources, BuildTriggerSubdirs, TriggerBuildList, ActivateBuildTrigger, BuildTrigger, BuildTriggerList, BuildTriggerAnalyze) @@ -1572,6 +1573,30 @@ class TestUserRobots(ApiTestCase): robots = self.getRobotNames() assert not NO_ACCESS_USER + '+bender' in robots + def test_regenerate(self): + self.login(NO_ACCESS_USER) + + # Create a robot. + json = self.putJsonResponse(UserRobot, + params=dict(robot_shortname='bender'), + expected_code=201) + + token = json['token'] + + # Regenerate the robot. + json = self.postJsonResponse(RegenerateUserRobot, + params=dict(robot_shortname='bender'), + expected_code=200) + + # Verify the token changed. + self.assertNotEquals(token, json['token']) + + json2 = self.getJsonResponse(UserRobot, + params=dict(robot_shortname='bender'), + expected_code=200) + + self.assertEquals(json['token'], json2['token']) + class TestOrgRobots(ApiTestCase): def getRobotNames(self): @@ -1601,6 +1626,31 @@ class TestOrgRobots(ApiTestCase): assert not ORGANIZATION + '+bender' in robots + def test_regenerate(self): + self.login(ADMIN_ACCESS_USER) + + # Create a robot. + json = self.putJsonResponse(OrgRobot, + params=dict(orgname=ORGANIZATION, robot_shortname='bender'), + expected_code=201) + + token = json['token'] + + # Regenerate the robot. + json = self.postJsonResponse(RegenerateOrgRobot, + params=dict(orgname=ORGANIZATION, robot_shortname='bender'), + expected_code=200) + + # Verify the token changed. + self.assertNotEquals(token, json['token']) + + json2 = self.getJsonResponse(OrgRobot, + params=dict(orgname=ORGANIZATION, robot_shortname='bender'), + expected_code=200) + + self.assertEquals(json['token'], json2['token']) + + class TestLogs(ApiTestCase): def test_user_logs(self): self.login(ADMIN_ACCESS_USER) From 67905c277eca5901124f54b75a36eef466329e91 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 25 Aug 2014 19:13:40 -0400 Subject: [PATCH 20/57] Remove webhook worker --- Dockerfile.web | 3 --- conf/init/webhookworker/log/run | 2 -- conf/init/webhookworker/run | 8 ------- workers/webhookworker.py | 41 --------------------------------- 4 files changed, 54 deletions(-) delete mode 100755 conf/init/webhookworker/log/run delete mode 100755 conf/init/webhookworker/run delete mode 100644 workers/webhookworker.py diff --git a/Dockerfile.web b/Dockerfile.web index 448a7f748..e1d253632 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -39,9 +39,6 @@ ADD conf/init/nginx /etc/service/nginx ADD conf/init/diffsworker /etc/service/diffsworker ADD conf/init/notificationworker /etc/service/notificationworker -# TODO: Remove this after the prod CL push -ADD conf/init/webhookworker /etc/service/webhookworker - # Download any external libs. RUN mkdir static/fonts static/ldn RUN venv/bin/python -m external_libraries diff --git a/conf/init/webhookworker/log/run b/conf/init/webhookworker/log/run deleted file mode 100755 index 6738f16f8..000000000 --- a/conf/init/webhookworker/log/run +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec svlogd -t /var/log/webhookworker/ \ No newline at end of file diff --git a/conf/init/webhookworker/run b/conf/init/webhookworker/run deleted file mode 100755 index 04521552a..000000000 --- a/conf/init/webhookworker/run +++ /dev/null @@ -1,8 +0,0 @@ -#! /bin/bash - -echo 'Starting webhook worker' - -cd / -venv/bin/python -m workers.webhookworker - -echo 'Webhook worker exited' \ No newline at end of file diff --git a/workers/webhookworker.py b/workers/webhookworker.py deleted file mode 100644 index ccff884c2..000000000 --- a/workers/webhookworker.py +++ /dev/null @@ -1,41 +0,0 @@ -import logging -import argparse -import requests -import json - -from app import webhook_queue -from workers.worker import Worker - - -root_logger = logging.getLogger('') -root_logger.setLevel(logging.DEBUG) - -FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - %(funcName)s - %(message)s' -formatter = logging.Formatter(FORMAT) - -logger = logging.getLogger(__name__) - - -class WebhookWorker(Worker): - def process_queue_item(self, job_details): - url = job_details['url'] - payload = job_details['payload'] - headers = {'Content-type': 'application/json'} - - try: - resp = requests.post(url, data=json.dumps(payload), headers=headers) - if resp.status_code/100 != 2: - logger.error('%s response for webhook to url: %s' % (resp.status_code, - url)) - return False - except requests.exceptions.RequestException as ex: - logger.exception('Webhook was unable to be sent: %s' % ex.message) - return False - - return True - -logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False) - -worker = WebhookWorker(webhook_queue, poll_period_seconds=15, - reservation_seconds=3600) -worker.start() \ No newline at end of file From 510bbe7889e8854ca591473d8d8be7f6aa4eff43 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 26 Aug 2014 12:41:43 -0400 Subject: [PATCH 21/57] Add more check conditions for unhealthy workers and make the messaging better. --- workers/dockerfilebuild.py | 10 ++++++---- workers/worker.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index a4de1cc47..9d3e92ae7 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -41,12 +41,13 @@ def matches_system_error(status_str): """ Returns true if the given status string matches a known system error in the Docker builder. """ - KNOWN_MATCHES = ['lxc-start: invalid', 'lxc-start: failed to', 'lxc-start: Permission denied'] + KNOWN_MATCHES = ['lxc-start: invalid', 'lxc-start: failed to', 'lxc-start: Permission denied', + 'lxc-start: The container failed'] for match in KNOWN_MATCHES: - # 4 because we might have a Unix control code at the start. - found = status_str.find(match[0:len(match) + 4]) - if found >= 0 and found <= 4: + # 10 because we might have a Unix control code at the start. + found = status_str.find(match[0:len(match) + 10]) + if found >= 0 and found <= 10: return True return False @@ -613,6 +614,7 @@ class DockerfileBuildWorker(Worker): except WorkerUnhealthyException as exc: # Spawn a notification that the build has failed. + log_appender('Worker has become unhealthy. Will retry shortly.', build_logs.ERROR) spawn_failure(exc.message, event_data) # Raise the exception to the queue. diff --git a/workers/worker.py b/workers/worker.py index e7750c232..c29d10f41 100644 --- a/workers/worker.py +++ b/workers/worker.py @@ -135,8 +135,8 @@ class Worker(object): except WorkerUnhealthyException: logger.error('The worker has encountered an error and will not take new jobs. Job is being requeued.') - self._stop.set() self.mark_current_incomplete(restore_retry=True) + self._stop.set() finally: # Close the db handle periodically From c1b0b2383a9902f1366c6c1c8dd8b281505a1425 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 26 Aug 2014 15:18:59 -0400 Subject: [PATCH 22/57] Add missing dependency to the builder Dockerfile --- Dockerfile.buildworker | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.buildworker b/Dockerfile.buildworker index c18c24589..04efe38f0 100644 --- a/Dockerfile.buildworker +++ b/Dockerfile.buildworker @@ -4,10 +4,10 @@ ENV DEBIAN_FRONTEND noninteractive ENV HOME /root # Install the dependencies. -RUN apt-get update # 06AUG2014 +RUN apt-get update # 21AUG2014 # New ubuntu packages should be added as their own apt-get install lines below the existing install commands -RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62-dev libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap2-dev libsasl2-dev +RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62-dev libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap2-dev libsasl2-dev libpq-dev # Build the python dependencies ADD requirements.txt requirements.txt From d76d4704a08501abb607d5e43e46c9b801c0f159 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 26 Aug 2014 15:19:39 -0400 Subject: [PATCH 23/57] Add pagination to the notifications API and make the UI only show a maximum of 5 notifications (beyond that, it shows "5+"). --- data/model/legacy.py | 12 +++++++--- endpoints/api/user.py | 24 +++++++++++++++----- endpoints/notificationevent.py | 2 ++ static/css/quay.css | 2 +- static/directives/header-bar.html | 16 ++----------- static/directives/notification-bar.html | 5 ++++- static/directives/notifications-bubble.html | 7 ++++++ static/js/app.js | 25 +++++++++++++++++++-- 8 files changed, 67 insertions(+), 26 deletions(-) create mode 100644 static/directives/notifications-bubble.html diff --git a/data/model/legacy.py b/data/model/legacy.py index 866587f7e..cc32b8979 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -1717,19 +1717,20 @@ def create_notification(kind_name, target, metadata={}): def create_unique_notification(kind_name, target, metadata={}): with config.app_config['DB_TRANSACTION_FACTORY'](db): - if list_notifications(target, kind_name).count() == 0: + if list_notifications(target, kind_name, limit=1).count() == 0: create_notification(kind_name, target, metadata) def lookup_notification(user, uuid): - results = list(list_notifications(user, id_filter=uuid, include_dismissed=True)) + results = list(list_notifications(user, id_filter=uuid, include_dismissed=True, limit=1)) if not results: return None return results[0] -def list_notifications(user, kind_name=None, id_filter=None, include_dismissed=False): +def list_notifications(user, kind_name=None, id_filter=None, include_dismissed=False, + page=None, limit=None): Org = User.alias() AdminTeam = Team.alias() AdminTeamMember = TeamMember.alias() @@ -1767,6 +1768,11 @@ def list_notifications(user, kind_name=None, id_filter=None, include_dismissed=F .switch(Notification) .where(Notification.uuid == id_filter)) + if page: + query = query.paginate(page, limit) + elif limit: + query = query.limit(limit) + return query diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 3d79a806d..e2e6a0ff4 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -7,8 +7,9 @@ from flask.ext.principal import identity_changed, AnonymousIdentity from app import app, billing as stripe, authentication from endpoints.api import (ApiResource, nickname, resource, validate_json_request, request_error, - log_action, internal_only, NotFound, require_user_admin, - InvalidToken, require_scope, format_date, hide_if, show_if, license_error) + log_action, internal_only, NotFound, require_user_admin, parse_args, + query_param, InvalidToken, require_scope, format_date, hide_if, show_if, + license_error) from endpoints.api.subscribe import subscribe from endpoints.common import common_login from data import model @@ -403,11 +404,24 @@ class Recovery(ApiResource): @internal_only class UserNotificationList(ApiResource): @require_user_admin + @parse_args + @query_param('page', 'Offset page number. (int)', type=int, default=0) + @query_param('limit', 'Limit on the number of results (int)', type=int, default=5) @nickname('listUserNotifications') - def get(self): - notifications = model.list_notifications(get_authenticated_user()) + def get(self, args): + page = args['page'] + limit = args['limit'] + + notifications = list(model.list_notifications(get_authenticated_user(), page=page, limit=limit + 1)) + has_more = False + + if len(notifications) > limit: + has_more = True + notifications = notifications[0:limit] + return { - 'notifications': [notification_view(notification) for notification in notifications] + 'notifications': [notification_view(notification) for notification in notifications], + 'additional': has_more } diff --git a/endpoints/notificationevent.py b/endpoints/notificationevent.py index f1cbec42c..e393dc134 100644 --- a/endpoints/notificationevent.py +++ b/endpoints/notificationevent.py @@ -184,6 +184,8 @@ class BuildFailureEvent(NotificationEvent): return 'build_failure' def get_sample_data(self, repository): + build_uuid = 'fake-build-id' + return build_event_data(repository, { 'build_id': build_uuid, 'build_name': 'some-fake-build', diff --git a/static/css/quay.css b/static/css/quay.css index 721253ab9..224029444 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -745,7 +745,7 @@ i.toggle-icon:hover { } .user-notification.notification-animated { - width: 21px; + min-width: 21px; transform: scale(0); -moz-transform: scale(0); diff --git a/static/directives/header-bar.html b/static/directives/header-bar.html index d440e3a86..3f395b34d 100644 --- a/static/directives/header-bar.html +++ b/static/directives/header-bar.html @@ -37,15 +37,7 @@ {{ user.username }} - - {{ notificationService.notifications.length }} - +
      @@ -58,11 +50,7 @@ Notifications - - {{ notificationService.notifications.length }} - +
    • Organizations
      • diff --git a/static/directives/notification-bar.html b/static/directives/notification-bar.html index 5d25a40b4..c6841a7f5 100644 --- a/static/directives/notification-bar.html +++ b/static/directives/notification-bar.html @@ -3,7 +3,10 @@
      -

      Notifications

      +

      + Notifications + +

      diff --git a/static/directives/notifications-bubble.html b/static/directives/notifications-bubble.html new file mode 100644 index 000000000..cf10cccf2 --- /dev/null +++ b/static/directives/notifications-bubble.html @@ -0,0 +1,7 @@ + + + {{ notificationService.notifications.length }}+ + + diff --git a/static/js/app.js b/static/js/app.js index 8844007a6..0c91c7c6d 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1155,7 +1155,8 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading 'user': null, 'notifications': [], 'notificationClasses': [], - 'notificationSummaries': [] + 'notificationSummaries': [], + 'additionalNotifications': false }; var pollTimerHandle = null; @@ -1251,7 +1252,9 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading 'uuid': notification.id }; - ApiService.updateUserNotification(notification, params); + ApiService.updateUserNotification(notification, params, function() { + notificationService.update(); + }, ApiService.errorDisplay('Could not update notification')); var index = $.inArray(notification, notificationService.notifications); if (index >= 0) { @@ -1308,6 +1311,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading ApiService.listUserNotifications().then(function(resp) { notificationService.notifications = resp['notifications']; + notificationService.additionalNotifications = resp['additional']; notificationService.notificationClasses = notificationService.getClasses(notificationService.notifications); }); }; @@ -5021,6 +5025,23 @@ quayApp.directive('twitterView', function () { }); +quayApp.directive('notificationsBubble', function () { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/notifications-bubble.html', + replace: false, + transclude: false, + restrict: 'C', + scope: { + }, + controller: function($scope, UserService, NotificationService) { + $scope.notificationService = NotificationService; + } + }; + return directiveDefinitionObject; +}); + + quayApp.directive('notificationView', function () { var directiveDefinitionObject = { priority: 0, From 97aa2c5aaa68c9e3f45686707057917c884829a2 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 27 Aug 2014 13:04:31 -0400 Subject: [PATCH 24/57] Make sure the regen confirm dialog result is actually used :-/ --- static/js/app.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/static/js/app.js b/static/js/app.js index 0c91c7c6d..c26e5800d 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -2406,8 +2406,10 @@ quayApp.directive('dockerAuthDialog', function (Config) { $scope.askRegenerate = function() { bootbox.confirm('Are you sure you want to regenerate the token? All existing login credentials will become invalid', function(resp) { - $scope.regenerating = true; - $scope.regenerate({'username': $scope.username, 'token': $scope.token}); + if (resp) { + $scope.regenerating = true; + $scope.regenerate({'username': $scope.username, 'token': $scope.token}); + } }); }; From 551539dbc582de9096dca37ff256a7a35be8c715 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 27 Aug 2014 16:41:30 -0400 Subject: [PATCH 25/57] Update the nginx config to allow for request bodies up to 20gb. --- conf/server-base.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/server-base.conf b/conf/server-base.conf index 6aeaa689e..a13cf1424 100644 --- a/conf/server-base.conf +++ b/conf/server-base.conf @@ -1,4 +1,4 @@ -client_max_body_size 8G; +client_max_body_size 20G; client_body_temp_path /var/log/nginx/client_body 1 2; server_name _; From 463a3c55c3ed70220dfd402cea94d4b46ca1945f Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 27 Aug 2014 19:02:53 -0400 Subject: [PATCH 26/57] Make worker error messages more descriptive --- workers/dockerfilebuild.py | 1 + workers/worker.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 9d3e92ae7..2b79aa084 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -478,6 +478,7 @@ class DockerfileBuildWorker(Worker): container['Id'], container['Command']) docker_cl.kill(container['Id']) self._timeout.set() + except ConnectionError as exc: raise WorkerUnhealthyException(exc.message) diff --git a/workers/worker.py b/workers/worker.py index c29d10f41..57d4a02d0 100644 --- a/workers/worker.py +++ b/workers/worker.py @@ -102,8 +102,8 @@ class Worker(object): logger.debug('Running watchdog.') try: self.watchdog() - except WorkerUnhealthyException: - logger.error('The worker has encountered an error and will not take new jobs.') + except WorkerUnhealthyException as exc: + logger.error('The worker has encountered an error via watchdog and will not take new jobs: %s' % exc.message) self.mark_current_incomplete(restore_retry=True) self._stop.set() @@ -133,8 +133,8 @@ class Worker(object): logger.warning('An error occurred processing request: %s', current_queue_item.body) self.mark_current_incomplete(restore_retry=False) - except WorkerUnhealthyException: - logger.error('The worker has encountered an error and will not take new jobs. Job is being requeued.') + except WorkerUnhealthyException as exc: + logger.error('The worker has encountered an error via the job and will not take new jobs: %s' % exc.message) self.mark_current_incomplete(restore_retry=True) self._stop.set() From 5744f0f8881d56c645eea49b27910de7396139fb Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 28 Aug 2014 16:07:56 -0400 Subject: [PATCH 27/57] Make the dockerfilebuild error checking less harsh --- workers/dockerfilebuild.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 2b79aa084..b373a00a9 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -41,8 +41,7 @@ def matches_system_error(status_str): """ Returns true if the given status string matches a known system error in the Docker builder. """ - KNOWN_MATCHES = ['lxc-start: invalid', 'lxc-start: failed to', 'lxc-start: Permission denied', - 'lxc-start: The container failed'] + KNOWN_MATCHES = ['lxc-start: invalid', 'lxc-start: failed to', 'lxc-start: Permission denied'] for match in KNOWN_MATCHES: # 10 because we might have a Unix control code at the start. From 5028172c51a4b5ef27b03399c7a4583e6ff9fade Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 28 Aug 2014 16:10:06 -0400 Subject: [PATCH 28/57] Fix Stripe dialog in IE and mobile safari --- static/js/app.js | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/static/js/app.js b/static/js/app.js index c26e5800d..51434ce39 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1543,7 +1543,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading }); }; - planService.changePlan = function($scope, orgname, planId, callbacks) { + planService.changePlan = function($scope, orgname, planId, callbacks, opt_async) { if (!Features.BILLING) { return; } if (callbacks['started']) { @@ -1556,7 +1556,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading planService.getCardInfo(orgname, function(cardInfo) { if (plan.price > 0 && (previousSubscribeFailure || !cardInfo.last4)) { var title = cardInfo.last4 ? 'Subscribe' : 'Start Trial ({{amount}} plan)'; - planService.showSubscribeDialog($scope, orgname, planId, callbacks, title); + planService.showSubscribeDialog($scope, orgname, planId, callbacks, title, /* async */true); return; } @@ -1629,9 +1629,34 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading return email; }; - planService.showSubscribeDialog = function($scope, orgname, planId, callbacks, opt_title) { + planService.showSubscribeDialog = function($scope, orgname, planId, callbacks, opt_title, opt_async) { if (!Features.BILLING) { return; } + // If the async parameter is true and this is a browser that does not allow async popup of the + // Stripe dialog (such as Mobile Safari or IE), show a bootbox to show the dialog instead. + var isIE = navigator.appName.indexOf("Internet Explorer") != -1; + var isMobileSafari = navigator.userAgent.match(/(iPod|iPhone|iPad)/) && navigator.userAgent.match(/AppleWebKit/); + + if (opt_async && (isIE || isMobileSafari)) { + bootbox.dialog({ + "message": "Please click 'Subscribe' to continue", + "buttons": { + "subscribe": { + "label": "Subscribe", + "className": "btn-primary", + "callback": function() { + planService.showSubscribeDialog($scope, orgname, planId, callbacks, opt_title, false); + } + }, + "close": { + "label": "Cancel", + "className": "btn-default" + } + } + }); + return; + } + if (callbacks['opening']) { callbacks['opening'](); } @@ -3904,7 +3929,7 @@ quayApp.directive('planManager', function () { return true; }; - $scope.changeSubscription = function(planId) { + $scope.changeSubscription = function(planId, opt_async) { if ($scope.planChanging) { return; } var callbacks = { @@ -3918,7 +3943,7 @@ quayApp.directive('planManager', function () { } }; - PlanService.changePlan($scope, $scope.organization, planId, callbacks); + PlanService.changePlan($scope, $scope.organization, planId, callbacks, opt_async); }; $scope.cancelSubscription = function() { @@ -3981,7 +4006,7 @@ quayApp.directive('planManager', function () { if ($scope.readyForPlan) { var planRequested = $scope.readyForPlan(); if (planRequested && planRequested != PlanService.getFreePlan()) { - $scope.changeSubscription(planRequested); + $scope.changeSubscription(planRequested, /* async */true); } } }); From 85ab7a8c8dd73ec126ae83ff0550e1099c96cf48 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 28 Aug 2014 18:40:33 -0400 Subject: [PATCH 29/57] Fix migration downgrade for the regenerating robot kind --- .../43e943c0639f_add_log_kind_for_regenerating_robot_.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data/migrations/versions/43e943c0639f_add_log_kind_for_regenerating_robot_.py b/data/migrations/versions/43e943c0639f_add_log_kind_for_regenerating_robot_.py index 2c91902f0..6ee041e4c 100644 --- a/data/migrations/versions/43e943c0639f_add_log_kind_for_regenerating_robot_.py +++ b/data/migrations/versions/43e943c0639f_add_log_kind_for_regenerating_robot_.py @@ -29,6 +29,7 @@ def upgrade(): def downgrade(): schema = gen_sqlalchemy_metadata(all_models) + logentrykind = schema.tables['logentrykind'] op.execute( (logentrykind.delete() .where(logentrykind.c.name == op.inline_literal('regenerate_robot_token'))) From ce7e3a8733037a38fbb4e84cca76c98e57768a88 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Fri, 29 Aug 2014 13:16:32 -0400 Subject: [PATCH 30/57] Do not link against layers that are still marked as uploading, there is no guarantee that they will ever be completed and their ancestry may be incomplete. --- data/model/legacy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/model/legacy.py b/data/model/legacy.py index cc32b8979..52723bd11 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -1037,7 +1037,8 @@ def find_create_or_link_image(docker_image_id, repository, username, translation .join(Repository) .join(Visibility) .switch(Repository) - .join(RepositoryPermission, JOIN_LEFT_OUTER)) + .join(RepositoryPermission, JOIN_LEFT_OUTER) + .where(ImageStorage.uploading == False)) query = (_filter_to_repos_for_user(query, username) .where(Image.docker_image_id == docker_image_id)) From 584f6b9635c8d073292747ffcb2a16a8dd00f1df Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 29 Aug 2014 13:59:54 -0400 Subject: [PATCH 31/57] Add a spinner when a tag is being deleted --- static/js/controllers.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/static/js/controllers.js b/static/js/controllers.js index 41e1443ea..4d1c8484f 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -523,16 +523,24 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi $scope.deleteTag = function(tagName) { if (!$scope.repo.can_admin) { return; } - $('#confirmdeleteTagModal').modal('hide'); var params = { 'repository': namespace + '/' + name, 'tag': tagName }; + var errorHandler = ApiService.errorDisplay('Cannot delete tag', function() { + $('#confirmdeleteTagModal').modal('hide'); + $scope.deletingTag = false; + }); + + $scope.deletingTag = true; + ApiService.deleteFullTag(null, params).then(function() { loadViewInfo(); - }, ApiService.errorDisplay('Cannot delete tag')); + $('#confirmdeleteTagModal').modal('hide'); + $scope.deletingTag = false; + }, errorHandler); }; $scope.getImagesForTagBySize = function(tag) { From d1b2ff588af1beb02910b0f247c2ad30cd9e913b Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 29 Aug 2014 14:00:07 -0400 Subject: [PATCH 32/57] Add a spinner when a tag is being deleted --- static/partials/view-repo.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/static/partials/view-repo.html b/static/partials/view-repo.html index 4f588ccf2..e5f2cecc6 100644 --- a/static/partials/view-repo.html +++ b/static/partials/view-repo.html @@ -391,7 +391,10 @@ ?
      -
      +
      + +
      +
      Are you sure you want to delete tag {{ tagToDelete }} @@ -401,7 +404,7 @@ The following images and any other images not referenced by a tag will be deleted:
      -
      +
      From 2c20fca37e49720afbec7840023d84a31a47842d Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Fri, 29 Aug 2014 14:30:49 -0400 Subject: [PATCH 33/57] Fix sharing tests and add a test to ensure that uploading images are not shared. --- test/test_image_sharing.py | 86 ++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 37 deletions(-) diff --git a/test/test_image_sharing.py b/test/test_image_sharing.py index ab278f9f4..ef2458ccc 100644 --- a/test/test_image_sharing.py +++ b/test/test_image_sharing.py @@ -46,25 +46,30 @@ class TestImageSharing(unittest.TestCase): preferred = storage.preferred_locations[0] image = model.find_create_or_link_image(docker_image_id, repository_obj, username, {}, preferred) - return image.storage.id + image.storage.uploading = False + image.storage.save() + return image.storage - def assertSameStorage(self, docker_image_id, storage_id, repository=REPO, username=ADMIN_ACCESS_USER): - new_storage_id = self.createStorage(docker_image_id, repository, username) - self.assertEquals(storage_id, new_storage_id) + def assertSameStorage(self, docker_image_id, existing_storage, repository=REPO, + username=ADMIN_ACCESS_USER): + new_storage = self.createStorage(docker_image_id, repository, username) + self.assertEquals(existing_storage.id, new_storage.id) - def assertDifferentStorage(self, docker_image_id, storage_id, repository=REPO, username=ADMIN_ACCESS_USER): - new_storage_id = self.createStorage(docker_image_id, repository, username) - self.assertNotEquals(storage_id, new_storage_id) + def assertDifferentStorage(self, docker_image_id, existing_storage, repository=REPO, + username=ADMIN_ACCESS_USER): + new_storage = self.createStorage(docker_image_id, repository, username) + self.assertNotEquals(existing_storage.id, new_storage.id) def test_same_user(self): - """ The same user creates two images, each which should be shared in the same repo. This is a sanity check. """ + """ The same user creates two images, each which should be shared in the same repo. This is a + sanity check. """ # Create a reference to a new docker ID => new image. - first_storage_id = self.createStorage('first-image') + first_storage = self.createStorage('first-image') # Create a reference to the same docker ID => same image. - self.assertSameStorage('first-image', first_storage_id) + self.assertSameStorage('first-image', first_storage) # Create a reference to another new docker ID => new image. second_storage_id = self.createStorage('second-image') @@ -73,68 +78,68 @@ class TestImageSharing(unittest.TestCase): self.assertSameStorage('second-image', second_storage_id) # Make sure the images are different. - self.assertNotEquals(first_storage_id, second_storage_id) + self.assertNotEquals(first_storage, second_storage_id) def test_no_user_private_repo(self): """ If no user is specified (token case usually), then no sharing can occur on a private repo. """ # Create a reference to a new docker ID => new image. - first_storage_id = self.createStorage('the-image', username=None, repository=SHARED_REPO) + first_storage = self.createStorage('the-image', username=None, repository=SHARED_REPO) # Create a areference to the same docker ID, but since no username => new image. - self.assertDifferentStorage('the-image', first_storage_id, username=None, repository=RANDOM_REPO) + self.assertDifferentStorage('the-image', first_storage, username=None, repository=RANDOM_REPO) def test_no_user_public_repo(self): """ If no user is specified (token case usually), then no sharing can occur on a private repo except when the image is first public. """ # Create a reference to a new docker ID => new image. - first_storage_id = self.createStorage('the-image', username=None, repository=PUBLIC_REPO) + first_storage = self.createStorage('the-image', username=None, repository=PUBLIC_REPO) # Create a areference to the same docker ID. Since no username, we'd expect different but the first image is public so => shaed image. - self.assertSameStorage('the-image', first_storage_id, username=None, repository=RANDOM_REPO) + self.assertSameStorage('the-image', first_storage, username=None, repository=RANDOM_REPO) def test_different_user_same_repo(self): """ Two different users create the same image in the same repo. """ # Create a reference to a new docker ID under the first user => new image. - first_storage_id = self.createStorage('the-image', username=PUBLIC_USER, repository=SHARED_REPO) + first_storage = self.createStorage('the-image', username=PUBLIC_USER, repository=SHARED_REPO) # Create a reference to the *same* docker ID under the second user => same image. - self.assertSameStorage('the-image', first_storage_id, username=ADMIN_ACCESS_USER, repository=SHARED_REPO) + self.assertSameStorage('the-image', first_storage, username=ADMIN_ACCESS_USER, repository=SHARED_REPO) def test_different_repo_no_shared_access(self): """ Neither user has access to the other user's repository. """ # Create a reference to a new docker ID under the first user => new image. - first_storage_id = self.createStorage('the-image', username=RANDOM_USER, repository=RANDOM_REPO) + first_storage = self.createStorage('the-image', username=RANDOM_USER, repository=RANDOM_REPO) # Create a reference to the *same* docker ID under the second user => new image. second_storage_id = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=REPO) # Verify that the users do not share storage. - self.assertNotEquals(first_storage_id, second_storage_id) + self.assertNotEquals(first_storage, second_storage_id) def test_public_than_private(self): """ An image is created publicly then used privately, so it should be shared. """ # Create a reference to a new docker ID under the first user => new image. - first_storage_id = self.createStorage('the-image', username=PUBLIC_USER, repository=PUBLIC_REPO) + first_storage = self.createStorage('the-image', username=PUBLIC_USER, repository=PUBLIC_REPO) # Create a reference to the *same* docker ID under the second user => same image, since the first was public. - self.assertSameStorage('the-image', first_storage_id, username=ADMIN_ACCESS_USER, repository=REPO) + self.assertSameStorage('the-image', first_storage, username=ADMIN_ACCESS_USER, repository=REPO) def test_private_than_public(self): """ An image is created privately then used publicly, so it should *not* be shared. """ # Create a reference to a new docker ID under the first user => new image. - first_storage_id = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=REPO) + first_storage = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=REPO) # Create a reference to the *same* docker ID under the second user => new image, since the first was private. - self.assertDifferentStorage('the-image', first_storage_id, username=PUBLIC_USER, repository=PUBLIC_REPO) + self.assertDifferentStorage('the-image', first_storage, username=PUBLIC_USER, repository=PUBLIC_REPO) def test_different_repo_with_access(self): @@ -143,64 +148,71 @@ class TestImageSharing(unittest.TestCase): be shared since the user has access. """ # Create the image in the shared repo => new image. - first_storage_id = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=SHARED_REPO) + first_storage = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=SHARED_REPO) # Create the image in the other user's repo, but since the user (PUBLIC) still has access to the shared # repository, they should reuse the storage. - self.assertSameStorage('the-image', first_storage_id, username=PUBLIC_USER, repository=PUBLIC_REPO) + self.assertSameStorage('the-image', first_storage, username=PUBLIC_USER, repository=PUBLIC_REPO) def test_org_access(self): """ An image is accessible by being a member of the organization. """ # Create the new image under the org's repo => new image. - first_storage_id = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO) + first_storage = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO) # Create an image under the user's repo, but since the user has access to the organization => shared image. - self.assertSameStorage('the-image', first_storage_id, username=ADMIN_ACCESS_USER, repository=REPO) + self.assertSameStorage('the-image', first_storage, username=ADMIN_ACCESS_USER, repository=REPO) # Ensure that the user's robot does not have access, since it is not on the permissions list for the repo. - self.assertDifferentStorage('the-image', first_storage_id, username=ADMIN_ROBOT_USER, repository=SHARED_REPO) + self.assertDifferentStorage('the-image', first_storage, username=ADMIN_ROBOT_USER, repository=SHARED_REPO) def test_org_access_different_user(self): """ An image is accessible by being a member of the organization. """ # Create the new image under the org's repo => new image. - first_storage_id = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO) + first_storage = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO) # Create an image under a user's repo, but since the user has access to the organization => shared image. - self.assertSameStorage('the-image', first_storage_id, username=PUBLIC_USER, repository=PUBLIC_REPO) + self.assertSameStorage('the-image', first_storage, username=PUBLIC_USER, repository=PUBLIC_REPO) # Also verify for reader. - self.assertSameStorage('the-image', first_storage_id, username=READ_ACCESS_USER, repository=PUBLIC_REPO) + self.assertSameStorage('the-image', first_storage, username=READ_ACCESS_USER, repository=PUBLIC_REPO) def test_org_no_access(self): """ An image is not accessible if not a member of the organization. """ # Create the new image under the org's repo => new image. - first_storage_id = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO) + first_storage = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO) # Create an image under a user's repo. Since the user is not a member of the organization => new image. - self.assertDifferentStorage('the-image', first_storage_id, username=RANDOM_USER, repository=RANDOM_REPO) + self.assertDifferentStorage('the-image', first_storage, username=RANDOM_USER, repository=RANDOM_REPO) def test_org_not_team_member_with_access(self): """ An image is accessible to a user specifically listed as having permission on the org repo. """ # Create the new image under the org's repo => new image. - first_storage_id = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO) + first_storage = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO) # Create an image under a user's repo. Since the user has read access on that repo, they can see the image => shared image. - self.assertSameStorage('the-image', first_storage_id, username=OUTSIDE_ORG_USER, repository=OUTSIDE_ORG_REPO) + self.assertSameStorage('the-image', first_storage, username=OUTSIDE_ORG_USER, repository=OUTSIDE_ORG_REPO) def test_org_not_team_member_with_no_access(self): """ A user that has access to one org repo but not another and is not a team member. """ # Create the new image under the org's repo => new image. - first_storage_id = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ANOTHER_ORG_REPO) + first_storage = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ANOTHER_ORG_REPO) # Create an image under a user's repo. The user doesn't have access to the repo (ANOTHER_ORG_REPO) so => new image. - self.assertDifferentStorage('the-image', first_storage_id, username=OUTSIDE_ORG_USER, repository=OUTSIDE_ORG_REPO) + self.assertDifferentStorage('the-image', first_storage, username=OUTSIDE_ORG_USER, repository=OUTSIDE_ORG_REPO) + + def test_no_link_to_uploading(self): + still_uploading = self.createStorage('an-image', repository=PUBLIC_REPO) + still_uploading.uploading = True + still_uploading.save() + + self.assertDifferentStorage('an-image', still_uploading) From 417fec0b68617f3c93b224a6ddbcf30a55d2f5da Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 29 Aug 2014 15:46:43 -0400 Subject: [PATCH 34/57] Fix namespace selector bug from the landing page and make the namespace selector update the URL if need be --- static/js/app.js | 8 ++++++-- static/partials/landing-login.html | 2 +- static/partials/landing-normal.html | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/static/js/app.js b/static/js/app.js index 51434ce39..dfdb9a879 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1749,7 +1749,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading when('/repository/:namespace/:name/build', {templateUrl: '/static/partials/repo-build.html', controller:RepoBuildCtrl, reloadOnSearch: false}). when('/repository/:namespace/:name/build/:buildid/buildpack', {templateUrl: '/static/partials/build-package.html', controller:BuildPackageCtrl, reloadOnSearch: false}). when('/repository/', {title: 'Repositories', description: 'Public and private docker repositories list', - templateUrl: '/static/partials/repo-list.html', controller: RepoListCtrl}). + templateUrl: '/static/partials/repo-list.html', controller: RepoListCtrl, reloadOnSearch: false}). when('/user/', {title: 'Account Settings', description:'Account settings for ' + title, templateUrl: '/static/partials/user-admin.html', reloadOnSearch: false, controller: UserAdminCtrl}). when('/superuser/', {title: 'Superuser Admin Panel', description:'Admin panel for ' + title, templateUrl: '/static/partials/super-user.html', @@ -4037,7 +4037,7 @@ quayApp.directive('namespaceSelector', function () { 'namespace': '=namespace', 'requireCreate': '=requireCreate' }, - controller: function($scope, $element, $routeParams, CookieService) { + controller: function($scope, $element, $routeParams, $location, CookieService) { $scope.namespaces = {}; $scope.initialize = function(user) { @@ -4074,6 +4074,10 @@ quayApp.directive('namespaceSelector', function () { if (newNamespace) { CookieService.putPermanent('quay.namespace', newNamespace); + + if ($routeParams['namespace'] && $routeParams['namespace'] != newNamespace) { + $location.search({'namespace': newNamespace}); + } } }; diff --git a/static/partials/landing-login.html b/static/partials/landing-login.html index e2500815a..0a3046d2a 100644 --- a/static/partials/landing-login.html +++ b/static/partials/landing-login.html @@ -24,7 +24,7 @@ {{repository.namespace}}/{{repository.name}}
      - See All Repositories + See All Repositories
      diff --git a/static/partials/landing-normal.html b/static/partials/landing-normal.html index 8a0badad1..6b9b6e42e 100644 --- a/static/partials/landing-normal.html +++ b/static/partials/landing-normal.html @@ -34,7 +34,7 @@ {{repository.namespace}}/{{repository.name}}
      - See All Repositories + See All Repositories
    From 07c7cdd51d53a3b5009192bae3cd69b927f6f823 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 29 Aug 2014 16:25:11 -0400 Subject: [PATCH 35/57] Fix PingService when loading results from cache --- static/js/app.js | 53 +++++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/static/js/app.js b/static/js/app.js index dfdb9a879..f5c612c5f 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -441,6 +441,29 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading var pingService = {}; var pingCache = {}; + var invokeCallback = function($scope, pings, callback) { + if (pings[0] == -1) { + setTimeout(function() { + $scope.$apply(function() { + callback(-1, false, -1); + }); + }, 0); + return; + } + + var sum = 0; + for (var i = 0; i < pings.length; ++i) { + sum += pings[i]; + } + + // Report the average ping. + setTimeout(function() { + $scope.$apply(function() { + callback(Math.floor(sum / pings.length), true, pings.length); + }); + }, 0); + }; + var reportPingResult = function($scope, url, ping, callback) { // Lookup the cached ping data, if any. var cached = pingCache[url]; @@ -453,28 +476,15 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading // If an error occurred, report it and done. if (ping < 0) { cached['pings'] = [-1]; - setTimeout(function() { - $scope.$apply(function() { - callback(-1, false, -1); - }); - }, 0); + invokeCallback($scope, pings, callback); return; } // Otherwise, add the current ping and determine the average. cached['pings'].push(ping); - var sum = 0; - for (var i = 0; i < cached['pings'].length; ++i) { - sum += cached['pings'][i]; - } - - // Report the average ping. - setTimeout(function() { - $scope.$apply(function() { - callback(Math.floor(sum / cached['pings'].length), true, cached['pings'].length); - }); - }, 0); + // Invoke the callback. + invokeCallback($scope, cached['pings'], callback); // Schedule another check if we've done less than three. if (cached['pings'].length < 3) { @@ -510,12 +520,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading pingService.pingUrl = function($scope, url, callback) { if (pingCache[url]) { - cached = pingCache[url]; - setTimeout(function() { - $scope.$apply(function() { - callback(cached.result, cached.success); - }); - }, 0); + invokeCallback($scope, pingCache[url]['pings'], callback); return; } @@ -5401,7 +5406,9 @@ quayApp.directive('locationView', function () { $scope.getLocationTooltip = function(location, ping) { var tip = $scope.getLocationTitle(location) + '
    '; - if (ping < 0) { + if (ping == null) { + tip += '(Loading)'; + } else if (ping < 0) { tip += '
    Note: Could not contact server'; } else { tip += 'Estimated Ping: ' + (ping ? ping + 'ms' : '(Loading)'); From 066b3ed8f042701a0a5cbd394021f9f37b4dc7af Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 2 Sep 2014 14:26:35 -0400 Subject: [PATCH 36/57] Add client side handling of user login throttling --- static/directives/signin-form.html | 29 +++++++++++++++---------- static/js/app.js | 35 +++++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/static/directives/signin-form.html b/static/directives/signin-form.html index f56b8f8db..d8f822968 100644 --- a/static/directives/signin-form.html +++ b/static/directives/signin-form.html @@ -4,18 +4,25 @@ placeholder="Username or E-mail Address" ng-model="user.username" autofocus> - - - - - Sign In with GitHub - - +
    + Too many attempts have been made to login. Please try again in {{ tryAgainSoon }} seconds. +
    + + + + + + + + Sign In with GitHub + + +
    Invalid username or password.
    diff --git a/static/js/app.js b/static/js/app.js index f5c612c5f..5fb21205b 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -2245,7 +2245,10 @@ quayApp.directive('signinForm', function () { 'signInStarted': '&signInStarted', 'signedIn': '&signedIn' }, - controller: function($scope, $location, $timeout, ApiService, KeyService, UserService, CookieService, Features, Config) { + controller: function($scope, $location, $timeout, $interval, ApiService, KeyService, UserService, CookieService, Features, Config) { + $scope.tryAgainSoon = 0; + $scope.tryAgainInterval = null; + $scope.showGithub = function() { if (!Features.GITHUB_LOGIN) { return; } @@ -2275,7 +2278,15 @@ quayApp.directive('signinForm', function () { } }; + $scope.$on('$destroy', function() { + if ($scope.tryAgainInterval) { + $interval.cancel($scope.tryAgainInterval); + } + }); + $scope.signin = function() { + if ($scope.tryAgainSoon > 0) { return; } + $scope.markStarted(); ApiService.signinUser($scope.user).then(function() { @@ -2298,8 +2309,26 @@ quayApp.directive('signinForm', function () { $location.path($scope.redirectUrl ? $scope.redirectUrl : '/'); }, 500); }, function(result) { - $scope.needsEmailVerification = result.data.needsEmailVerification; - $scope.invalidCredentials = result.data.invalidCredentials; + if (result.status == 429 /* try again later */) { + $scope.tryAgainSoon = result.headers('Retry-After'); + + // Cancel any existing interval. + if ($scope.tryAgainInterval) { + $interval.cancel($scope.tryAgainInterval); + } + + // Setup a new interval. + $scope.tryAgainInterval = $interval(function() { + $scope.tryAgainSoon--; + if ($scope.tryAgainSoon <= 0) { + $scope.tryAgainInterval = null; + $scope.tryAgainSoon = 0; + } + }, 1000, $scope.tryAgainSoon); + } else { + $scope.needsEmailVerification = result.data.needsEmailVerification; + $scope.invalidCredentials = result.data.invalidCredentials; + } }); }; } From 2dcdd7ba5b93d6deece46760c5b219f21be264fe Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Tue, 2 Sep 2014 15:27:05 -0400 Subject: [PATCH 37/57] Add exponential backoff of login attempts. --- data/database.py | 2 ++ data/model/legacy.py | 32 +++++++++++++++++++++++++++++++- endpoints/api/__init__.py | 8 ++++++++ test/data/test.db | Bin 614400 -> 231424 bytes util/backoff.py | 5 +++++ 5 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 util/backoff.py diff --git a/data/database.py b/data/database.py index 349ad1b58..69932273d 100644 --- a/data/database.py +++ b/data/database.py @@ -76,6 +76,8 @@ class User(BaseModel): organization = BooleanField(default=False, index=True) robot = BooleanField(default=False, index=True) invoice_email = BooleanField(default=False) + invalid_login_attempts = IntegerField(default=0) + last_invalid_login = DateTimeField(default=datetime.utcnow) class TeamRole(BaseModel): diff --git a/data/model/legacy.py b/data/model/legacy.py index 52723bd11..2e703b003 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -1,12 +1,17 @@ import bcrypt import logging -import datetime import dateutil.parser import json +from datetime import datetime, timedelta + from data.database import * from util.validation import * from util.names import format_robot_username +from util.backoff import exponential_backoff + + +EXPONENTIAL_BACKOFF_SCALE = timedelta(seconds=1) logger = logging.getLogger(__name__) @@ -68,6 +73,12 @@ class TooManyUsersException(DataModelException): pass +class TooManyLoginAttemptsException(Exception): + def __init__(self, message, retry_after): + super(TooManyLoginAttemptsException, self).__init__(message) + self.retry_after = retry_after + + def is_create_user_allowed(): return True @@ -551,11 +562,30 @@ def verify_user(username_or_email, password): except User.DoesNotExist: return None + now = datetime.utcnow() + + if fetched.invalid_login_attempts > 0: + can_retry_at = exponential_backoff(fetched.invalid_login_attempts, EXPONENTIAL_BACKOFF_SCALE, + fetched.last_invalid_login) + + if can_retry_at > now: + retry_after = can_retry_at - now + raise TooManyLoginAttemptsException('Too many login attempts.', retry_after.total_seconds()) + if (fetched.password_hash and bcrypt.hashpw(password, fetched.password_hash) == fetched.password_hash): + + if fetched.invalid_login_attempts > 0: + fetched.invalid_login_attempts = 0 + fetched.save() + return fetched + fetched.invalid_login_attempts += 1 + fetched.last_invalid_login = now + fetched.save() + # We weren't able to authorize the user return None diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index e8dab28dc..854c3cad1 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -87,6 +87,14 @@ def handle_api_error(error): return response +@api_bp.app_errorhandler(model.TooManyLoginAttemptsException) +@crossdomain(origin='*', headers=['Authorization', 'Content-Type']) +def handle_too_many_login_attempts(error): + response = make_response('Too many login attempts', 429) + response.headers['Retry-After'] = int(error.retry_after) + return response + + def resource(*urls, **kwargs): def wrapper(api_resource): if not api_resource: diff --git a/test/data/test.db b/test/data/test.db index 34882e11701b07d1a0e1ea1631cdc6636489f033..3e631b5d72e5c2c290e95affc3a9794853a6a573 100644 GIT binary patch delta 19857 zcmc(H2Ygh;_VArKcX#hiA)$m2l8_}N1PGh$yLTah^g?Z6GWq9`H=7E}}jd)M#Wy9)$F-ur)k-|wFv%$YlV&Y3xL=A1LT zXj|f<#l5tZi2$!zaavbDw9QVNzZD#hzRE5!Kn1>F}d;%Z9Q8)x|z-zD@{ta8!diF?mcawS;4ZiwZiegOT9~)FxAraCb@xJS z;r6*0+}wu2^-UObO~YWm2ZK5F7_?2nVCEzYrqy83P>I3SWf;^HVo;HbL188a*{K+e zv0>maVqn!_U>t!#auNm!@fgI$V9-ATgRpQ6f`Y;@6x5*@68M&bZ*fqcgYVn8a20&V z!FTw13Vh$`!$M#^2VcSa@H$Rvb8o`~;9Ea-LI^z0!Q(jT1F#KK(;o!iN~*qW}Q)lgB(`=91ddAPVgOUN$w|is^)OGHf}$eL^X5`)cSgB&~;SC^N~&r-fhL;&`b>8kdghm3xn6DV6gWp z3|^gp!R{&ycFB0&S%ks%d<K4`K#r-4s6 z_y#_~gm=OB$-_~-WPB3%8AI-E_!-mBLf2dFApBfFv>g0`3;8V4?jv{)m+=936?PzH z{{>GXl~=(dxPO<$l-(2>rPI-9oA>C{E9rW0rd zEuy(JosOawnnH)scp5|d({LI@dGZ_ifqX?yk&nr7a+thHUL(86OXNB76xm4DkVnZw z^m^ zkzxE3?uI+z7GxI-VGgvx3~)mo)Iuc`Lmp(nXt07FhQknug-GZP!JwkQ(;w;A^fdj1 zzE9t!2kAb#o4!n+r`>cDT}xNchiTV+)JJcpf1_TyfX<>Vw2?N@$+U)+(?Xg<$5ID1 z<3dcLgK0F4pkY*`Kz=3PlP}5Vq-W>q%A%jC6{T|lX5A3}#k|e@0BC_P`hz$hMA!U{<`Iw2%S*%u&nu5cb}BTwzh&!yhLj z2*E-d$N=9WniM0CA_#IWFw*sSA}m6(PJ$fle>kulJ=qosxfht~Ot=cXa4+uaTr4pH zSngP|mGBt<3GOK)sw+nq6TXjz1aTyY%pqUXdGrLBU=7ab8{dAMP_#U>3|90KF_4&Z z!xY&=FOjRsApdtXaj=q?Nq-W@+%Mxawf<>t+lE|dF#Ba2SwQ01+!u*iEw5@WFtEi$ zCBla?W>3L~IM#!}K75$O_L9p0e8j=`$YD-n##_KX+Df!R;HSf93N>Xq93w+TI4&3b z9FAkb1Hg`-A=;k&>(Ri1=)YVeBUa6AL#|9~>1O&16v0}Y<+=ga_m9;?MnsUC!aAMx zGu^W~-0kxJA>F6{O>`t`q&B{>dA4(AqpNOa%hbl^I%j96yQ#IagO#r${d+q*I_oaV z_mx9mHgE^gl0-iq%65=+GL$XZLGnq``6LzlW(ThLVRCw3ma~)Ok>URM(w!t;D`O!9 z%pC6KVlX3rK89R&Ep)(}$X~C6+i|*mftkbTfoRw+^W?AKH1de|;2kWn4^i?dyaL;? z)Cyo5yHq^e9Y<56=~xbQ@CfeT8{tOy6j|>3Fc37*LHA(MILgMRP*qocWsal9ZYefp z6Vs)Oi zs3s$?%27Shm{qH>jWd`Fq)PTm3Z0g2G+K*mswS2@Y<63QxhTIpBfBiWGP|gA!wh_;;&yVTtV`+LcEaTt=N;dDpTgb0pg`Mz{BHDchYq69U*s@aU3Z{oR zcnZ(PczOs^chRnVUX4PeH#CI)Rs83JzolUMKcGR^#Y70uAfNi1QqjLkhkW91iPFo{ zBK5x~hP;ziTnopwko9=vKL%I%dTS@Mmv)k5cKQ_}bbq@OcN`hXY`e)qqGM0)Cby6j z|IW+aO`_P$SBRJBdnCl{A@*>2%kgmX{E>@fc?Xn3h<(vQBZDu^@@b#v+37AiAeg?- z;mP|9Zl>W_>2=zbr$R!Kzf4Fz(9ZvWj0zRI&hMoF86nWnUnL^Vl_w_t2?@_r(D)zs zlONZJwjc3MbQ0c|(FW}OhiSsO!%99L$U_`?i2eWzFncj!m3vS;Fm&Iz2ag_N@{{k! zS4pedoK=yNQN_-_il^1Bdr9&|=S+_Sa6eUPD2l(=Q$_rx?Bn-n#IRhOy&$Ke#4Z&V z<ayBWxk^SaMX(?-9EqXFVJgn07hzlW5Zp>7aMMasbs92usAebSztNZs*KE{95&bi1K2hR`0jCb7)nq& zYLj3&$&^7N8|8ozfm!wFwFSg;E4#;n>=BjMCfv`#3vfRU{0?CEZvao=s)1MdLX?_M z(^PsL01nqa;5xfq5n662SA>>Mx3fw9pVfVh1M&&UVcSQ+)g+fCjD|vz$67~2HOcpH znIog2ffTUxRCp(_!v3(HwnB(TzT4wf4+ry^f0g~T4}vxDnJlq=hNC9l=t+L$nBI0~be3`#|BU;epASInK=elK{B`MUh*5D2A}8 zG#Ej~v2|$>#U`gg2`Q4}t)$pLfq$hzAC^4^YDh^pO4u&KjF0dsq?Ap0gl{8dGKglU zAK^y?WXVgqj^iUl_*~vXU&H4(oTI=#*Yg84bR!4z=tfL=l(LOu_|PE^$!u|%OuBlz zr$J|Nx=lJ~z1yL)TMTBi+2b(OJ0*5}3~%VFn&lp;#hp^=Zq>>gM{9J9GTBC1tXiD` z|Kd@G>};uXIXm6)qqOmVC~Y!aRC-~Ly=S!gvvBV@91fe8Cn7hVh+b8OP+#w|+3aSQ z&N9&CFzYNPr(GvW4U*1=<7%)w4K};W#8!{xv$V=K?$H!7H=Cci$od>bMn1<%N_iyx z_cF1*M@)PkkRE@oooMl&>FxCPYIWu%i%c84yNgkbKvzlC1vr}@pJQVaqsX*{*<}ft4-A1=r=kS=3)RG%3*f3@?m@W25 zd1;s&5`auofG=h0eW`5S4`~-<_!1}ap|ZPU0eEClSVR=%d8)X* zsOKUJ=n%RdMj?@&Wp6yj=aXtb-p8!uZy^)T(?DeFS0c53TFK8Kwf@wbSMk%xL^=Kt zndDDuSdAnuTgk_=PgnDU0-F;DE9*sNHPeTI#rEw!K3s$|0YMBi-uE-0uaEJ4l=L%! z7mi`^Lv@EiH3n#G&+~g<2R51$w`bJYgSj}m{AVfos8Ovl?|K8*XT~!&>j~I9S_ARd#~Q<2KsdxWPSU$*Qy1Y!02% zVsPp#o_be<&0;sYYzFquNR>6gCb>**6Rr=N%Ye(p)6k%koOZj;BRM?yXRz5U5;N&k zX$c;w0dp-*oz-N+-X*(7=WsL_bxyP7^f((Nsa~=(pH7uEL~3a87+lzp#fD3^!EBM& zxJBo38YPdV!7aJWR`#P#m7HL~s!}~JZWorqCT&)oRB!X>Tm}Qq&SSLL><*TnqRL3H zS*<3gJVH-{-HoD&(WR@GJT{$M^3)qG^$i}U&B@lMsA3WtEN+LV!DH81U6S0U$D?!F zJrZAP7~!G$H=4cMm5V%3>k4pTkO(d>3PSgl?) zBEjJ{*(`30S?4yGao0*t6pXBr9oai#$m%vbjEMM^dX+WP;BcAEMns~)V8nyKh?`up zI1%xZQF2%%v&mRrFYl{h*v6sK;+)E^hk6uO;3L4cjaB7D0nb4+NT>{b0k4HFv;**{ zCS9y3U_Ykae5s0%!7xKsYkYzC$iv8OcEEG+1gu7`<3qh_7iz0#V1qfpX4I+f4v3dY z97&?@K?`O?5tg)`x08BhxiZ}ws$Oi>9x|kR)p~vwAuj*I4Lzs|W*e8{w$0zbdx*P7 z>VXZsDLxFgD5tLww)t$h$w5ZAk1D>6DVqn?#;Rxy}55S9P(s&hB zJ0IidpM@=0_Eu$kj~@)HiI37?m`K z@XG%R4(mBY(BruC?}dl4`*vXWZcsH1h9#V$r#l4)aUv^W2_oww*o%sQpRn|HrR)q~ zOSY;y&q<4@o(thQ2ij}sQ2G=&eW(Lu?t`k)WJWh~qb~_*lBbZ!@OTVjdk(9ESo0xO zI6LyTs+2VQvxGycPyD@rf@JCtxQ~PT5HYxZ z?$fJ7+a1@Wvw@B6rRzq%$v8vosQ&gT+S(#_FmlafJ zNF|_``cwcYr{p_(LNbcGG-F)X8Y84BA9hactuv*g{dQ`Q9lDTZt zyXsguSzrr}`H3CQx{jPHI^``j&mz)5RJT z5-EqBjl(_sPez_>-5($miIJElhpjXjT(1FDgh~N!v)I{bbQx<+i z9m&SLi=+sDR~<$QS1FYrSQYSKjJU|)KFU#Umc z(UiN%UjRQs*mXzMNdfd+Jmat8G<`3J1@>K-CQO7|QDJ~*;Rc-dHNb8itC`(X?f-$Y zyw5_P_k)$i%%Nd?65Wg@u-lQ2g~AQvn>CTKv6O8(s*a@G-_RDmZq&u6Y0Rjfes;&a<{Q`hma$0 zvX~I~7l%gCbF#oi<~tnp{Q+6`g*r`MG`z^+v9SZsO#c}wy9k`Zqv0*sgGHACd$C26 z9WC=PIsaJ{mp{id@EGd)-@wCTC%lNI{)w`19Is;Ax->a|qNrAmZY+u>f$eu{M)#Ld zqv0lU`0ov3CX9Z7w%Pl9y|vkzIqjOrG@RDo9YPZ2{_e-Z9*1B`y1GLhsf(pujm<9i zTr`Ndn>*X*)y=@qx<=QzZ|)9%jZve7_~G@Pnn-;t^}@wfN^DDh#CD5|*4Q*fo>p z;P^1HF;&2&b{D%bRd9&zxk%JV>J4ToX~F<@ELBiZ&c}k@P>&4|>lqHMtbY6IRy>IP z_SR=)@*%?R^GhSoKdi}*rnf7XV0aV_C?2%14nu9%XLJu*sb!l?6ICw8T|N1VBgJ4f zpr9Bs2r|eY52|z7!B@@>x^lL7QFfO}8@TPCYhAb~{}0)k3)zy+Vlu+58&}yD{Y6LwYGrHoY$PvO&VEf$Vo? zW0C#SG1zl820LUInHObmo#$mYji+UIm?wv0uwgI;t0OU36(U=aDXhVeJAlIH8f@u7 z;QytC9OivRGced@)~93MGb3R#Grp=B-ZfN~4F1>LB$jsl@488l^KKF+XcrlB?oMzT z$?-ng!ww<}ccXCpJaoe*v}Ue=hXMB%x;y<14cH4{7JA_{LIX^O8YqWC$bqroK5KGPx`l3_tLbw3AZ2t(7rl+%NW0LW z-9ek^RO+OY=y+O63uqQiqjqYf$uyD1(J0!NhN5Q?AwQFE$r*Bzd`OOxx5?{d57~}( z^k>LsvYxCY%g8^;-Q-R*C0&nJr8%UH%s}&h9jPUiq!_KY8EEacqEdf2&RhQT_19`; z?RLN~&;8IS4t-%1Xq19*gW_;YI$~D=fW0 z%n!!2fBOwML2S_hAzf90DO*|i*Md^M`=0Opkxgva8GmdYTlBdh&i-ESAt$bIK;L_uu#SfMu)B*9!Q_mf2M;`kNbhTg`T(DUe~vJ^>tJziR; zq7zUtq$5ESkd$g9;%Df7@*2ANJWd}%ZmTkiNVHSq(~?gi-{9XmP3?()&Aj(m2a^qy{(4j@F2QmEP>nb1n7c! zXp?WksX5U>V?2~X0c1fM*ueHxmQL;O8Fz&D_j zunQe{u0^-0N3pSKxcn<{=}XwaAn1#D`WgM4UP70JQuJ~%qJL9g%A-E%7=0aGbr`(` zJ#=JcTH@ABBBOlNLg||5riQdLZ?Ez~)(2+~$MEinHymag`vT09qnctlIQJFs;lc;z22&?`^ zx*||d>uhLfnbq7`*D%f5Jk?#-&_JVM^rfk-&W?^bE$uG6Je$Q!YP8ODHur_(3sqX0 zXS>@w`2k=hPM6E?t%P;l9UY2wu1^%${TaSne~G9)U!>LD-qhI9f$mJbWF(zWpX08d z*3vQ~QBG7O{5n@l!wh%3r*Wn`Vjwt3VC0Hd)P+XEe4BoYj_eOc($MZkd9BXb+B!4f zFE!Y=^jEE~?e~7>Kwf)GeM_gR7YsgExYIc`SgsrJxav`Nh;PcT+VC(KaxPEqH^_I* zZ!rNPNF0@DC7nU9N3Xx-=$y76T?2oCaNHC5a5cJ3%tyDkXHd-~>vjd|d-b}n=wb+|k0cp4SwQP) zF?#qV($HjdpFD_%;KS&$SBL&f1`4TZAoM_}koY>OuGik@X8F&;teu zgZ`xXr=kP*j0GIbM6bgZ@(r2A{lLi{lkTns=$19us~RqF0{&&pl2<)?=umK2A@gvRg%>MnopqQCccXS1uNN%s8;^gef9iKnr( zeom%ow#jDC%g)R+Nwa32)7vuJnjN)ySvjWpolORy1Z5HJ8pa7Pif)o>A3WIj_;$*f1???(Fv2#q%p>7)p$m z+`{IT|1ehz>ewwdFM&`l%BxBXbly6v9@)q_EN43^Yi8QYtDW--yUY{kPHZ#I$e8Q) z%y8y5>z(5>3OqHn_TpKyrnaP%Stc5%Ia}LoQqy#2LrZ;1Q%lD*ck_&v^r=nG#+iRc zZlhhYI?QuuD5txC?{jBfva3_!T#8##*I)1vyBz9`HiyY>^7ba7T&!1R6^+U=Ki72t z0rT92h9bLElR10ph{haidHu}JsdF1ki(IbEf(m!RyfS-k!^|A(__@`N*+quJSp_AY zwsEuA;or&7k$>#eoZ>JVC6hrB`N(q|%GN>jDLRTghdXi&>AD8>i}nj>sGl{jd8V^{ zsyn&aji=L81s8wS1MQEaLgx{qK{8A8d!a_ge4%?*9!C3i{ZAd7F75N|WEk$uE8-xCYhYeG*S(tA<)5!)9NmS4M0 zqb<2@i+2QJq5Z^EcGttg^!~l)KU?Q-@Gu9FbPjoiJIu+Rd&T5h4CjYN1Y|TYR2_Fq%sy zbI)`Almkf~>Iz@_><9S7lDo=$$7i45lOFtfu5b68?Yy{c{aW8$b5AE|)8^I7^+UK4 z*%z}B2BIJQLaImi_1j4X_ciwjSAy-NbeFs;JOzV;IW+yRh6%uamhyuP9v4o>j}4pMTP8^m=jBjAP)UD7#aa#d9p7r?awjm+x+G&D z@nbr5i>B+`3@(dl@aD#_bywj6+_Flzi68d&Yxc0QtA*42fXVz0W>_Pf;*&n?RnN3* zh1EkwG{tKE8^me+H*uczJ}RVlhn*6l&W+(uuvoSN`!Vk1uVLwr39B`O+P59{#tvee zLUCLpR|=CgVRy6~@Ooldc`wAr-j%|Nm@z-RqP^0>25)QvdtJZ=iXOoRl20z{<@F3< zZ6Y>s`VnD;CLF$cz?&QITX#(=JGdO@KIGlu@3Oi_h0}(Fd1Ytiwgs*eO>m&cn`H$q z_(tw5Cdki{F8=eQdgCU|-_K3+&rm(g_v)NfcH;}!8@c1rvn*^YVtUA#+Hlso6fqF` z{4F7D$}(Yf|DhW{`P4tVKf##xSu8(Nd%PbDe-1Hbi|_TcH+BU35^%!5JckobijN)X z^`N^vj}w0Id11wn@RQS)|2NcQt!ISA?CIY%`KDi`4mnMx zkY3y~oNQ|?5UY(EJg)Gm6+3eZDZcKgP(`~R--rq$<&jP*873Q=^H2v=iMpUV)CtWX zZKxkwhW!A6{%Aewk)Av?~w!Ju~sVkZ66K+K|V48&~u8G)Fj zZ}P{C27OB)X41C>VitWzAm+5`X9c2?evSYU*rBBF>+f088y)(7J>uxgEssb^kM0rI z_lR5M8)Bf0$)Jy5q5DutZq^TAufL|zMcDN*y&>s*0lVHL>0{ZQuW2Iim6ci61^46Q z@Omi6BJoAcnAfnjtTx8)SU!FT6v{CzKV&GB%dsJR!f>dOV?#$L$}2gF$bMUBmhq4W z$U$5q<#-c5Oi$9E@fJJ`6->G4K{5gTN19Ltbu-+L>tQE+fD2+bKKE*n8yd+cnV?aQ z>G@$+Xpv(kez*j)7K>gdKjKiny70?l)u;48 zh{diq^q4+=S`eACav^{*e~QJSH_12hOB1aIy}3tgR+HX(@uy8$L{`08rfI(3s<-Ji z3T@<=T`wpEwp#U)KhdhU+Vl>k2~5Pe)vh0guh z*tM@{1}SXdPYcm=9VXtyCmI@eu-1(z0fc=$MG?w^xeWAYhYWfYUwe)q z?;2UW$!-wld$SPAnXgk#z(BaEtK8CBf?TwZ1~lio2#6#PpQPe!KSR!EL|<0 zwi;eBel8C^gfq$y0&7u?vxRzSKRjqUNCJ0&TZkE#{&goG5cd5i>=nst`|h_8dHr_m z8qTH=MBc#LzRzLh6p`U5Ngly60Fh_g+cNTs$U7W^<)go;FK4gBimNs1>vQoqFJZ^K zg;e&`AaSzBx={Su>nUc7o)*$rO`NzQCZ%{pw7<>Eim zWUcG4fjI-k6@1jA4{TxDXmL9~nB4FaI~gsW9uofLlo$T1Rek(7;W(4N5pvlJzYCdc zrYtRJ^F_u=^Td4Coi8qCnmjS%f`jKKd~q9uFO_T1WN|Mp)4eFD$_}30@9M-vD%MU= zBnTa0bwy%&_n!U26mq2-0!#0Y1MgQdu7Qb>h;`Mp@*wt8Bw|szd&?2_*#N|PihI(g zD`I_MUo4-h?|hyG^+T*jCN5s-jh*NhJHP9PNE`D|))ud))-QIl2*i4tNA=#7u+FB8 z6(7)qzWL$rUQeB0+UyQTWRJZ5?PfNmH;&nSBq@;%?<1ZbJaG3*n`LANbDwg5dJEap zeRF}>LfF<^#4AJOc7H!kj3SEc8^RTFsAhLVUo`E_q*}6{+<>V4l)G2CXpQVn8X-P; z#Wl>tR2%9p|{)hCH_JVeVMf2z@_4`JSYni#O?!&t#SO>Fu<9K)y|YKWUKy%C-6824uC!xN|1j;s+K{i|Y={HbMyf0CI`)PG z+5V7@Pk(2NMj>Y&R+V^=X-A7Ey80$0Im(o?JPJfzi zqr1?2aflvA6UJG5&BlR%BHRF!28ZGAAV_Es&4)4+<))zNuN7T^7oj+}7~Mpd;wst* zPs27e5AH{Ar{k!+KMOxx#LPDj(w-ssyjLV5)yVJRDfL63pnUU`SD*@?7s>f*s#c(e zY7{6?L4hI_6&OT=6c|i{6&ON86c|cF71)dRQeYSjQ(!m^*L2AZ_ols-gg&&70{haw z3hYPwDKLUYD6l{6ufRweslWkrfC8gvlmZ9RfeMVK(F%;AF$#>Ou?if758&kf;%J-# zwN%@sL@p(P3|rKJiiqh$&#r{xN) zpcM+Nq?HP+qE!gH2pLbuD+$%KT7fmRMu8LP1O?X8S_Mv|6BRg#PEz1i^eP2Trjr$T zHN9Gab+k@_Q|J^0I;m5E^|W4r4YWalF6z?o*gtVox02wY9+^G4@p;I(SFw0}yp9et zFQTP$6KWopp}uAb9xqY6p6wgABo+_xKWTitnRy z%uUMIWpM?9V(oA4(*mXZ6&5){O}`12iYq>(sL;Zcr1-H|~YsFEB`2BUSb zHxcotkbdO8?E^0`CE z?{*>Idm8!QD&&LrB0szt`QlvUkJFG(PJl98!ZF|*zHZ}{FCI&+$gOQt*51w1ZQ==D zYW-pf8*Y~+l|{Q^*jIM(bVBI03+*!ELr`kR{#?)x`Sn~nk{lt6Njmop_b4YnQj@y7 zhKj2oTHew@#K2ufu_`}Ah#&z4VKCd4B&M^td{JO?P`>F#+3!cfKFtx+uPFN!n8fWG zN6hTu-sTo|DjF|r6=re0rgy~d!``N5b~FZ4vMk~{js5D|9NrO4iei?Ttm5?kBX3-O z=t?IX-T^!PcfjRWV*^89-Kz4A=wPP@;9Yc)0S9}aP8`Aa$^J2!Z8IR=VkQRqzL0lw)!fZi8OKIQ{LuTEJVI~J~3Hb-=B_- zdQfglMd$XwH`3EXyT4BmZzh2UE$sob1Loih&$}jwr!`S`dVlsd%~z}p?B!aV!{D@f zuXn^eznS5QiQ@Dq=>yXtnMN9XZFfOGP6g$e$k&A3n)5}!(q(qD*1A{q9 zb_SUb3HU2G_%l=FN#aDx-@!?!ZXQ+@jK7Y8uki3jI#mq1%-hC*pCg-6q{-=f8P`vC zv{*Bo8IV)46GfWD%ND`9(M&4VjJ<3wHZX!cS*+>*XStc-mpX0AqlsqYN;Jd6FLl`T zk7r$pCi#z9pOk2-djG)*Q*Nw3t1s20TqbW>sb-Y^4^MGRyBk@1KmoUaJ{6FFs!~g&Q delta 21587 zcmd^nd3+Sbw(wN-OlIruVh<#Q1PFvA472tuEJF5uCkY9JFqtI@kUe3&8U{o~uLz7< zh>Ex(3WB2Mx}YGJtDv|cD!724tm2M{UZ3CTo(Y76sQ2@|_xQhyxPn|kd zb?Tg2vWj1_b5vAOL3VAcC(7Hkg0 za{u>D7Bs}at8}n`O0&a1zsZ0GLu$BxZ&RFK(-eQ4ziP`qz zasIze7>)Y+ADmF(=O)CXK4Nz61ik;}@uLv&ZyR6Xf6$>WHvjX{;r_2<2)$_q zit?{)8S3{(#~>Bpzwa-Pe8m6Qux)kt8s*-Qqajlie))sUQ}{HNp;@x+ zG6A?E{ash=ixnSe<5$>B!=A)b=)NcMCgsrnWk2{L`q7=I@I?CTQ@BDodM|s2&)b*I zI*ljM((U+RIk|EFIlAc~yhlFf%VE1{^TYU@vd`_CPW$S3`pJ8M_246Tv$Efzfs=f3 z9Nm8qYFe`q&zARJU-2y6u@>JW$M^CbwBj!Oh4PwLQ_lKQHFWdafVF%bzCqsCHM5eE zyYV^s!26?rqla(DU&;pzocJx>cL&fXH2<^R7ZE~tz9!OlC(su*(`@#66?E2KkvsqnlO%eLB~iX4eFS z;b%=H{H)G~pT9fc=a$j%b8~-DpfEWhBj1w~y>v;I|3rMGlAM&0AIQmGaI*a;^+Vdl z$`-S+GIA00z-j2`Lm*#!$u9C7d6H}*_mjKGYO;d($U@RYYKfDSkphxI5{QEsNh}#o zhLFBQO%Q*P|CT?^f5so;-{JT2yZGn$C;3hM{rp}0YJLUp;}`Nxd@b+f%lHC5gHPZc zypfOP+lTW*_`bZFN8Cm3TkbUX8Fz?#huh2T;-2H4CCo{LsOX;rd=a)hKrJse}j(v2FA(f=aDzlmSU z|Bau=H}bQ2H$Rh~#uxJ0d@4VMAJ1F(ar_v51V5A?#P{JfyqxYq>St-?jmx^ZY#*wgWSCw1qr)_Th6tEjJ0re zTs279bgr1o1363LCUG{9w0Le5e7E?weNfGrE*cdW(@ejKL5l6slRn04w#nNUvvD%= zGpOn_po0&C!0aQhfVezG9s^-nN7jI-EG3ITP-c@V5R*cZ2|_ZSm_S5EkT4Js4Z$ED z-|=TaI1cmgf@r+LKM#WO82$rEGWkT11`{6#vJl3%_XAPD z+|M8gXSib^2JdqFKnR}ao&phgfLjLwa5J|Qy1$v54V_=k6++i1a^s=n{f<4u9%B!)@3Q;YSJ>y-r`X5X2iSG&8un&(DZ7YmW@ociY&l!VX0nOwc-92p zE%Ikz&DkJ-3~U8MUn^j?w0p5vDMtu-ut$P!?3SPlyChhJt0d^eP6^J!vm{uFDSr{n1oEXU;%EW>3IoQ9`KuoRbyupO1)5-Fn?7fWy|o+`m2TqMCl zTqwZ;Tp+=GoG-yVoF~CtoGZZ`oFl<(oGrmDoF&0boGHN!oFTzA6c(2UIzG+~njjo2tb12#xdkM$B9hsQ}!hjkJh zi^obZ9>+^C4#!C_7RO3(3?3uF7#yQ)7yE299xY{z!lNV@jiV(v5|5PN2s}c9!|`wl zM&T$4M&d{b4#UGF7=a@sI1~?+U^ot!U>FXQ;1E1Sf`jp335Md(cIjad9wfnmc%TFa z-~kfskNZooAMPi?zPPUh``|tjB$!B$$GikN%t?^NtOT`KD?ts`NKlQ{5>#Q81eLg5 zDLsVX5D6-fQE}b*L~GVtr)}@QCp+i35_}%8|Ma_8zE-9nmygJhj(wQwYj#s+S42*qPKWj z+ZqG(1+ydR(~EIbR3wvx>Kfd&-i4Ko%?+&$tqU7Hm5rX}y4sc&__PR`16YFwGJ4d| zTwMv(RJMAYb(PHxb3NfSz8y!+=Ddr6lKf0=|cCNe9 z-Qb$zY4+C6^$Z-$2qJX>+T5NNS95Kn$kYMh%mhT2EXARyKfPrU9zw$x<8XNx<3RN7 zrFanPb0?(fON;Q}2|P2rlgq8nYK~dQaR%;0}vJ+M8M-83Xj)zu> zxj`f`SJ{UdbqTAXs-ZPR%;-cAGZf-5x~_=_2TYURtQ zPxm*$#nzlPZpHup%LNKGIWF?$OLDvy!czUm7e_cm1vLo0EkkeP+4wwjzdTv~y<%Mm zRa%t0)k^ilnpAB+EoBw3RQQQo!7E{@=jYcDp3uHKQS5f>(0T+v`m;UR4yFDZFt zo$?j6Ozl%&)NIujv0FefKLZ=8YSbqCSoR^Bj3$6-B|khXEF?5?AZn_ucY79yYM`>- zS?3WShOTMJN5lLDEsIg4e}9WvwKOS089Hzvy2AXDc13FDB_mB93iGdOZb02@8QfXR z^|h_es=1!F7EiPIAGQ6%zw3~H+R7+2Tr`cWTsZ}e@V~V(9gP%|y4w;_w7>DTax}{S z`fY;0;n8?Bdi&jvrXdvLcRm)e=CN653_Pqrv4Fnj@pULpEVk|OeE<4gMijsO?Oihw z8Y|`{?`{myB^9P5O!ZIL9qvE4JKjI^9X%SiJ^P*g2H9<>=|;T zmtKGoyKJR+9hdjZslzg{*dk47Mm^`8~WB_6ZMg3GCy3JY_K@-rRg{4`5qR<0#IFSV@L zkx*J{D=xAdGLurL358aTMiwd4tBh)uivDvY9@}22*2;$a6tQZVN~LOOYi+4@dw~9E zlQY_AiWX`XWjC~C7B$spS{B)=9OZLc9km`?UQTh+xZh?4TE6;9jOlmiJ>{T9fQ;{=YC^i?C z)FzIboip1d)HLMUr%tUaE3u~KnOcMtZ-Z-rd46h=C9A%{*roPLWq^F>1eK~yoETu8 z;Bg0ASd=?1)6_gwu#_~qOY%MT;hPw8&+4ka+5{t`_T0YyH zm|;(>YpZV%y0#F~g+a~EdUr#eI3R*86yzrqTC$td^=-`tw=*#}WuZB%WL#5m>74xX z>YTd5tTN-gW_Q`RI>F%0FwQHUpOIn7t8cJ$HqZw+HYVp6 zr4=S6CTHfCmE@%4CYKl6tmfRTl7y1;&n+%VOUlX1%PlUn z30W0#tt`@~h><$g+w5tn=|XaoscBqI>(nMgQnn*)ZjQ@1wb7N^HrLczkTlm*Yf7sv zXvi%yE^Jv~vo_iFb@PSVT=ycNxszm|*uzMDI!UU(v8@W$Nv5UruAA}T_UTwF({-}J z?U^U(pLlnxG?X$Dj25$_B&8@bH$68wEwMBs(QFcOO-55*QK>CEEzgjZTwa=ACKMzV z=9Lu{*|M??R*SvZnvtDtPfN45A+0RJrx4Uekdw~JgEVKQ)+Bk{`c!A;w1V`E3{#rX zknd>DODaq?yXIT%^)2?K9LIw4By(bUQ(3isL2Bt-PhuL~c?%vLCC&vbG0VV~kweU2 zS~~)kpvOrIc@JzrOUbQWm4;~KHLz~RZ}z_%PWFgar0>W%u$O!a7Pj}uTcD#(fni`z z_cFohe-Pc~2UGiAIx(D|EaNA`7C^@@C(FqxFdKaaRstny;a~4w*uXYo|KxRw_P?r( z*c7>sL;u$bNEy^X{(b9erudw;6{_yX)Z%R11Uq-$Fd-+G?eXx1GL0$#J&Ld#$Izt}sR<0vTj^AVa4?b9>nHwTW%aO5UtrRSHIq?K5T%q5mg zQ&LKLrXj(WoK|4XNGunU?ZzBiiNXK!TciBxZ{>|K8j1=t4AXK;GfbJuc0+ktdU~ck zrz|rmy}U?(^7Y>z6B6;y6RzV(+kq54Z zwK`v2GXrM1Vzqksz|aJabK5LdgWGJ@83c<sIDGLD>%v~n_`$^ z^`B`sP}NT|^3&Bm1?=aQDN1IbS|mzbmJdKq8FHeZaOWc1e^jrc>P?SO=l395jbN)F z_V>SGozhDux%~hBTCMoY7{93hr^Y^6CnGPBbv?;H!q9bERc#q)SL z`RiGobQU&U9HaE-P^49oGor!dOfQIe^h`JkSBl5AKlFZFOHV|iNVQm#bZ*p3T{-m3 zNEEIn-vrMjdqFD_=RY_PgdS!71g;(pRFku!Li(PZ?WM|Gde)4>l>9asS;%kej$~lZ zW4g_XxRGXy*XcI9btaR`uCsWm+&a76?9kc0c6XK2;&7NPHhR>G4DC4$^<$$MS{xaGucb;?WpW5Qhr?p3G8(*s)nK8UY$z!TRN@s? znap;J$WSafa*$uTvmmGxCDlzYP7vkig4jzJdO@5at?%hClaWSAcFW+PaCa}5W9Y5p zv1W32g2Zpb5ln*m>Ysj@-mRd*$<|&fA3(QeAw~J0=r$~B`^yaMoqqvSO`Qz3IO|DW zk9NmyV`yC-(zct-wyG+d!K-uGy+)lyunD>MPS{SUyDj6zP!va}j zHdGm$7Qq5@nn}=E3|^DY=@tw+0lH2w+Px0D%Zb65E)_sBq!El3x8U*EbS}5is1JLv}EraNN_n`q&o*a+@QW&AEpBi|6KOV*5`Xo9u z9yEJpGU|`gq!)70p#C41qAQb80ZJEZXhazu1t};8qf9!h9F3ss%aEGxOG8$=e;Uj~ z*@4_|rlEc`vkc8gIhWA-&^^;p1pO)v3WSxT6qGBzx*6pKFy1alN;+UVYD4*A#u)0F z4yO=MM+7}K9Sx<G0qpKye#diVDSPYF!)3MnyEg01xi?tqtiBntJC}G?=E%hxx8J z$UC5IKF}7k0@+ZBgi9x_N16`b3Um)brBZu?&pc8kto5&%;$2s*n%aOecD$!jngT^^HAMGq}NLbTIr zw%S0C=*$+E8ESGlbxxZJ2Dq!r>IL;;^El1av=B{-HVA^%BzPSvjRn zF00w;anX&7k&x(iz+3_Z=!_N-6Fx%c@Cq>T*ewPpC{ByXYZ66XOLofOFr?E3sEbon zx8CnOaO+QZwxg6GL@p!Y#139jFM-v>4JXWU@RwTE(?QL^+L#XcGH@9fPUeW#{I9_3 zau_C*e~}l#jb;-}EPimJX(z2@PLC$X-pSB8H=;X6i=-0rj5G^(v-G?rTHeS`@(g*t zM|l|Q^qJdG{%`_|9hktO=x5|(aB_MVoZ_~D(dSEY0tk75Jkzu2M~pbwi-(g(#iGCC zHP*A}0Y)18n<*-b68$%~L-B9H)btiG>k0V$v%rA8!1LW;Jfy=P#S^>vk#3I~yG3eU zLAT$Ha)MvKR@65?fZg!~*eX8+o7Fz3;6?cECt&!lCHM89W9)8*u6+>I4JWH*(#hqQ zHFx#}K?xhBq>q%t!JK3Xif-c}#Vy zdaU|2ZK3u>@F9AH6!RNNhHNt&h%KQTPoPvZN7B<9D*LotaA#u9+h`KYH8(tFh%68hcCHeX1-jdlR7q| z38*44C*8Uk9YUxQ%8#Ode*(3jS@8G>a!MI*Y(*A&!%mcps-*NcJ7M?b>PX*(Zbir~ zPH$&lK^4eDckLFZ_T6X(@`}%sHOt33m7xv?TG2wk{R!3vzkLW)z5XE@gsKDco9-m) z)1f(pZpWww>K;rb7A^h=KJG{)4xy8au!f3Abop*X?%aUnSoXhu+T17)LF6tOxvPhy ziR0|9R}t$;F{B&p`G(@s1bb3Zp^%@PnN?a;l%AB6QIuy&wwER3SW7ajW##td+|=TX zyyEO!TUKd-!8)zLl3P@iqUx~#wg|BH7i$^j8&rHriXS<69&uKWOTBb-LJ$7>XHaI{vXvzXkVqRlWt zS}iWCV07DIxl4C^fRcjS`QL9&qUiT_{Jgl({WIToRwVrQYWic_0sZ&)NUi3QWMm*8 z$0c>=fPuXM(WLWe?|?2FA?Rqq0H>m68ucUkpxYaZLoiy+RC@t^kbh+v3v5XRgXArv z6A##>vtUJ-OcADdEu=*`Q2Bs9?)lD_ZP$3@q2z~MdXr4{K zz$T@kvw(tr{0oZK_}E%yXc{1y14ygXm=~3*CPZRv<7ISsV^jB=tqjcq>7fY#cl!To8L2KQ65bFOG$kp@Sg=F3GO&C>-ZU=wvng9oUYxysN4Xdoug zfJTeMP|Ia(NNZ`X7LQh|>y)7>fT#~3zM;h&U9W|1XuXULnU<*Qr`9V&BO=gEizP$c z+=gmTeQWc=PS^%fCxbfH48_B7(A08+jNCxBUd>>-m!UWOhT8vl{ZGHqi;H>&!=nDX z;PRvUPzahDiUa$KANyl)zyCwr?+EVyC(y)0M;fT|X3=hN^VKW|`x$z3D4rMep(v5T zYG4tG21o7bypnsGbFx3MD_Og?qzmB&wwj^CZe#}a?>ZFi<7{S|*-F!HWNy#-W6fq9 zaqU#CMvQ4fvIJSRTGP|^Vd<=L))FS=s%?GZV`8bn$H?gyOPEntMS-&6)b3*@U-d0i z5J#W(_3|n3HnEQ}XoTuYO*qY5%Eb0Dbblq$vw0~K-}BYUrOed6Mo0G&K;9r)wTu~e zm3Q|qW5%cSDB@T=LNpUshw)gn4H;!m z$i(12nJgO)Zwd8%ijz8i{S-5(qxB8^cc#Pl58Tdl)NR0irX%+r{3z27xGSf@&#m#| zP+P3rD{jR;fPGz(HcV5h30EIcKcl`y^_uEA)iPC!%B3n&C8^A+k*fYGM)|$+xbgs4 zbWW2`R0~zLsxnoE$^oXuAu6@W57Y7#YO&2UXWO^Et}`U~~@>V4`L)SJ~C)T`A?)$`QV>SFa2 z^%!8{*Qz(c4`u_I2cF?RQUmc}r5e@Cs%@%=RBKg1UpfQ(F{Wca!(r_Th9La{H4}>T zMx$LnUrNslrmgxmDcu@OTl6hbx;dCO>6@fYVRCc1Cy?Pk5*qPGiryH#(u z>8JH&Vy?_Al}NgTRw9uRNES=&!t+$A-69Fo=Fr;>db?3yC}9SY1rjDa=S!G*fZ2}p zHbHN*>TNcCZU9wE<^)g$eYS*}B@tvZ=xs*5&7`-P^_hWYI&(4tWu;_#psYckCY4Q< z%31}z)vCAJ^j5n*MJg-hM2T6+QsDqbl2jO;6Q#ll^ob-qoQYs2(=mX0ennx0v+t^qw!UgAtepG0P;6H8Hi~lU_bX&$z^=vGSO) zOojLqD<2)pl!{NI!8oz?IF4Z;5JF5x$w!K!2~QDV+|NFahf(8khyjp3XVr6fKw_8A zG1>GcyWZr`59{(dfy{_5p948VyL?VKedQR|5wm`9m&{Ol|1mslm_a|VOXdKH`@p6C z68HPjN7IL zN*dPUvr@tl^dSv_AdP#Bj5&_x{=EMK#5YXR?0CoMp z4FMh5X&XHUffk>EL)<}_M)rbnaR-DAJqeS+!{j~)9rlx3$&D~2EGBKF0m6q}q=J+} z0C5&cA(J75*i3XJ27-vg$v{G2AH(p!@)sbG_$>bge*{8_5Agf>S0R}A1^yX+D})n2 z$lnVAPpjMcTlnRCJH!;X@O6AOL={ixi}^f=D^B7k@ivGoj)!QdNC++N&vU#If{TCQ z&U0Twc=2)W6Yc{DFn*oe194K%aof2q+(rm8zMH#)yA8sOmvPr}^C8f94(EXgsxq#K z%i+=>*mwdbaC!(g9tm^#)j~k$iR;9Um<;?0ZpWlXM7je4glm8-j@!fEt6i?0#8$8m za{@b%J;zX>{hY4OMk0#HIlCyUH>yqr{i|x)l~lx zJmr56q4oMhSVcGPgygHc@GAQ17Iv%7rJxE_gb57+#c$KQHfwyvC0p zFGj@Mc?h1`55bF#KjAc5@(RwPr(eOT)F$Sgd;)%mSnNKj0x|jWRhJl_FZD$ zN04`FC**DXP(*%3{CNUrP{UTJ>cnI4bl+o;XWN3)5JR00;wgxyV;;wpZrm)Ee_SlT z8Rt@7#C>%W)6#DzGE)>{|J+6mKyis^ z{E6Yqfz{S2kgk7mJ&pzp>^9W#Ofk;K1te5l2>v%lPj}zA7}$Q|S%QKN zosLfzkRgGQ+GDFFo?w8rViS3g+zW9{tH~{}uW5&@(s%?$_~T|`vkhOrLaK|#&%vc=#ifJXk7t7|sCW*Tfh05bZ?ho~?$jJSm^RPG1?VQpN&`6g z^sQQ)if+8JT*_>$qswY=BDx`9CI7Sre@kZ-;fZMZCCnF#AojMp0W540h(13Ik3jS2 zr^6u5wysebYJ{~*71}P8_<7Za2z?Zd_FWEirTwaCq>JoXr-)mOVwT%Fi*CcAsY8; z*2W%2$%O%XWdiDA$aJD*qERzXJY6V^;o*SqJZ$Mdh3))Sm>pk(=$h-`a)gIKrfNX* z7lPKegB%Sb{b4Zu0=F5w!fyw)Jp~l@2vF5>I8QzTyO@VT$FG2})*jv?2KG19A+N#+ zE6D~iM0G=VX$+Vt6m&y19y!EZXtAf4m`YO;OG{IX>DGAB6IGZEL_u5@b*~rqXLRbT*LGeS?7h@hE;%aGpIab)SPT)^ z)2oA69NjY?PY8@IxIBdvK}c;gyGgrK8>MMcFT463D{zFi{Qvj6RAyTS9U4DT(n_*M z8R^4EawTk?c0cgVpq{Lnr@S@f9NbNlFRx_o#9J^2chfY=5>_V+(Cdq(8C5MH;V4RL%h%B z0Nghd*FPnc?syIEn;CNPd@Zfm3-`^8{ovmoU+M_Dd6jtI%s#kp#+bU}-;}&A-Z%56 z{0u$(Pq=TU@A$b(=)OI0-^|!K)4%jZ45K^$E#5ctFSrLGdRmy?=Z&DVeiiSVc@^%P z8UEgzFVRge!hJLPW9P1=%{$<}8S_(PzxUOJ(NBI5>3a$2i$3zrLSNhvy8paL-^)Os z{lKivbjQ;`pYGkHduhcpKwr!O{G%^*5ZzqEOrkeDiA>O}z zLz~Y5leCR<2m9(w^pkBOlfD8bjU9e5#206z`=1n<^ffR^6_UP-?l=WZl5Z^;Kr2oI zlT_&stoEgjqnjTBtmS8*6XUkUo}lC`FlnOi{=xL{d(ep!H7Td*zJov?+xO%IUql?; zxmKj_eV~usu+8T4#?o1LiS&H{^u^|_E1{d-2KvV5{}4@^-@)gq3@>Qch-{7rI}{wj z*OD-}8wKoJ|9$2yP!{XCFhyvm^x+{egn_P z@#tgl3=?FZ$d&>DQ5~gm%qG-f9KgKAJd?n@%yg7|gCAo$oFC(Rv2-C%oLrE}1^Ht6 zD+;wDU$ItkG-OOjW5_?0GG)4QmGVPXl&S{oPCtQR>L&Hun!%b`nnyL?Xzki1+Sk}V zK|CUF zK$yrZhR)5y=^Yz(ijPt7NH=_lR|P-7Ad@eU?^LM%`^c9Y{`3c^rL957br7dI(*x0e#g)68BBXY~IDHPZL( z8K-H*ub@UkJJ1GS>Ug^OMNy6X7rwy~cik7i1h_k2Mnd`d+&=cO_B3$UrlzVh%Ab{L z;BKkB1Fky=!?D2K`LguY3E`KR>#OPv{mAG*~SQ68L! zOu!rYld625w=6ghnSnPXyWuUmX{^|FHsLAStOMTYYv$hUt1Ai4L*sxq{D`dmzPRGx zJfsKSjQ{4w+v$!d=(=H+^?#=o!-2kQ4u(GAOD&|EzZ7{h0_a2AuNguta8&ZRqM0PCYtZ+V7RQ~>J@+Pty8)Y-w=Z6>gO?8FbZQBn!4H(j%N6g`{{ zte3ZLZl(J&fc2I)EGv8w-r($(DGr;b_GJ6Kp5W}31@y&N-fyIvrT~5M|Jpf>HYWgm z5g$dI_0?4cmr#j7pAd8XL|>dUxP(dq`X*fbXE@zqVeTm&u*7smw9JiCtd)uCUYRZ` ze5I_b%7+PI)n&D>{4k{PN(HYtDk}ewF#+|jSliX54uZuHHFy{$CD$<-^xb*PWZ1y1 zy^aC9hlc?Z!+7|4IfhA~M@K{6ePYrky%^1`qWT#4xsZbsI!@V{2GntKJQyzNLpeBo z<4zBQ=>xG)T{+E%>N-wd2e+vVQmfWnWkG*=%K%cfi8e0;KBZKX_alkKUR#DOd d{b)V*RVzI_7y7Z^@h#`*zB*=;+@e`&{y$O#TS))_ diff --git a/util/backoff.py b/util/backoff.py new file mode 100644 index 000000000..15429936e --- /dev/null +++ b/util/backoff.py @@ -0,0 +1,5 @@ +def exponential_backoff(attempts, scaling_factor, base): + backoff = 5 * (pow(2, attempts) - 1) + backoff_time = backoff * scaling_factor + retry_at = backoff_time/10 + base + return retry_at From 2cfab6e252cf298caf067145762dba2acf17cb5a Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 2 Sep 2014 15:28:56 -0400 Subject: [PATCH 38/57] Reshow the sign in button when the username is changed --- static/js/app.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/static/js/app.js b/static/js/app.js index 5fb21205b..f44e4294e 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -2278,6 +2278,14 @@ quayApp.directive('signinForm', function () { } }; + $scope.$watch('user.username', function() { + $scope.tryAgainSoon = 0; + + if ($scope.tryAgainInterval) { + $interval.cancel($scope.tryAgainInterval); + } + }); + $scope.$on('$destroy', function() { if ($scope.tryAgainInterval) { $interval.cancel($scope.tryAgainInterval); @@ -2325,6 +2333,9 @@ quayApp.directive('signinForm', function () { $scope.tryAgainSoon = 0; } }, 1000, $scope.tryAgainSoon); + + $scope.needsEmailVerification = false; + $scope.invalidCredentials = false; } else { $scope.needsEmailVerification = result.data.needsEmailVerification; $scope.invalidCredentials = result.data.invalidCredentials; From 53939f596d3c2d6fd68f74b5d3522628c5aa7995 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 2 Sep 2014 16:45:25 -0400 Subject: [PATCH 39/57] Properly escape the $ in $token for the auth dialog command --- static/js/app.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/static/js/app.js b/static/js/app.js index f5c612c5f..3838d561c 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -2425,7 +2425,10 @@ quayApp.directive('dockerAuthDialog', function (Config) { }, controller: function($scope, $element) { var updateCommand = function() { - $scope.command = 'docker login -e="." -u="' + $scope.username + + var escape = function(v) { + return v.replace('$', '\\$'); + }; + $scope.command = 'docker login -e="." -u="' + escape($scope.username) + '" -p="' + $scope.token + '" ' + Config['SERVER_HOSTNAME']; }; From 232e3cc1dadddb7451df63386681bc237426ca5a Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 3 Sep 2014 12:10:36 -0400 Subject: [PATCH 40/57] Move cancelInterval into its own method to remove code duplication --- static/js/app.js | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/static/js/app.js b/static/js/app.js index f44e4294e..1f7eb060f 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -2278,24 +2278,29 @@ quayApp.directive('signinForm', function () { } }; - $scope.$watch('user.username', function() { + $scope.cancelInterval = function() { $scope.tryAgainSoon = 0; if ($scope.tryAgainInterval) { $interval.cancel($scope.tryAgainInterval); } + + $scope.tryAgainInterval = null; + }; + + $scope.$watch('user.username', function() { + $scope.cancelInterval(); }); $scope.$on('$destroy', function() { - if ($scope.tryAgainInterval) { - $interval.cancel($scope.tryAgainInterval); - } + $scope.cancelInterval(); }); $scope.signin = function() { if ($scope.tryAgainSoon > 0) { return; } $scope.markStarted(); + $scope.cancelInterval(); ApiService.signinUser($scope.user).then(function() { $scope.needsEmailVerification = false; @@ -2318,24 +2323,18 @@ quayApp.directive('signinForm', function () { }, 500); }, function(result) { if (result.status == 429 /* try again later */) { + $scope.needsEmailVerification = false; + $scope.invalidCredentials = false; + + $scope.cancelInterval(); + $scope.tryAgainSoon = result.headers('Retry-After'); - - // Cancel any existing interval. - if ($scope.tryAgainInterval) { - $interval.cancel($scope.tryAgainInterval); - } - - // Setup a new interval. $scope.tryAgainInterval = $interval(function() { $scope.tryAgainSoon--; if ($scope.tryAgainSoon <= 0) { - $scope.tryAgainInterval = null; - $scope.tryAgainSoon = 0; + $scope.cancelInterval(); } }, 1000, $scope.tryAgainSoon); - - $scope.needsEmailVerification = false; - $scope.invalidCredentials = false; } else { $scope.needsEmailVerification = result.data.needsEmailVerification; $scope.invalidCredentials = result.data.invalidCredentials; From 0bd9ba523e1d5604492fe3cc7b49a4c19d227003 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 3 Sep 2014 13:07:53 -0400 Subject: [PATCH 41/57] Add a migration for the brute force prevention fields to the user table. --- ...add_brute_force_prevention_metadata_to_.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py diff --git a/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py b/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py new file mode 100644 index 000000000..55351a50a --- /dev/null +++ b/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py @@ -0,0 +1,28 @@ +"""Add brute force prevention metadata to the user table. + +Revision ID: 4fdb65816b8d +Revises: 43e943c0639f +Create Date: 2014-09-03 12:35:33.722435 + +""" + +# revision identifiers, used by Alembic. +revision = '4fdb65816b8d' +down_revision = '43e943c0639f' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('user', sa.Column('invalid_login_attempts', sa.Integer(), nullable=False, server_default=0)) + op.add_column('user', sa.Column('last_invalid_login', sa.DateTime(), nullable=False, server_default=sa.func.now())) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('user', 'last_invalid_login') + op.drop_column('user', 'invalid_login_attempts') + ### end Alembic commands ### From 18ec0c3e0ace4c699369bcc73cf59c18c8abcf51 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 3 Sep 2014 13:09:17 -0400 Subject: [PATCH 42/57] Update the audit ancestry tool to not affect pushes in progress. --- tools/auditancestry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/auditancestry.py b/tools/auditancestry.py index dce445e4e..95e082642 100644 --- a/tools/auditancestry.py +++ b/tools/auditancestry.py @@ -20,7 +20,7 @@ query = (Image .join(ImageStorage) .switch(Image) .join(Repository) - .where(Repository.name == 'userportal', Repository.namespace == 'crsinc')) + .where(ImageStorage.uploading == False)) bad_count = 0 good_count = 0 From 21f7acf7ca14a1870336664ce7beaa5723b4cecb Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 3 Sep 2014 13:34:36 -0400 Subject: [PATCH 43/57] Fix the default value for the migration to use a string --- .../4fdb65816b8d_add_brute_force_prevention_metadata_to_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py b/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py index 55351a50a..a1c8c95dd 100644 --- a/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py +++ b/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py @@ -16,7 +16,7 @@ from sqlalchemy.dialects import mysql def upgrade(): ### commands auto generated by Alembic - please adjust! ### - op.add_column('user', sa.Column('invalid_login_attempts', sa.Integer(), nullable=False, server_default=0)) + op.add_column('user', sa.Column('invalid_login_attempts', sa.Integer(), nullable=False, server_default="0")) op.add_column('user', sa.Column('last_invalid_login', sa.DateTime(), nullable=False, server_default=sa.func.now())) ### end Alembic commands ### From 8910c6ff019f859c3f2f72c4f7f8f996b7f76712 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 3 Sep 2014 13:44:05 -0400 Subject: [PATCH 44/57] Add a migration to remove the webhooks table. --- ...2b0ea7a4d_remove_the_old_webhooks_table.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py diff --git a/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py b/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py new file mode 100644 index 000000000..79ea17be0 --- /dev/null +++ b/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py @@ -0,0 +1,35 @@ +"""Remove the old webhooks table. + +Revision ID: f42b0ea7a4d +Revises: 4fdb65816b8d +Create Date: 2014-09-03 13:43:23.391464 + +""" + +# revision identifiers, used by Alembic. +revision = 'f42b0ea7a4d' +down_revision = '4fdb65816b8d' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('webhook') + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('webhook', + sa.Column('id', mysql.INTEGER(display_width=11), nullable=False), + sa.Column('public_id', mysql.VARCHAR(length=255), nullable=False), + sa.Column('repository_id', mysql.INTEGER(display_width=11), autoincrement=False, nullable=False), + sa.Column('parameters', mysql.LONGTEXT(), nullable=False), + sa.ForeignKeyConstraint(['repository_id'], [u'repository.id'], name=u'fk_webhook_repository_repository_id'), + sa.PrimaryKeyConstraint('id'), + mysql_default_charset=u'latin1', + mysql_engine=u'InnoDB' + ) + ### end Alembic commands ### From 6c60e078fc207b0fb751f45586845825115bc421 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 3 Sep 2014 15:35:29 -0400 Subject: [PATCH 45/57] Fix NPE --- static/js/app.js | 1 + 1 file changed, 1 insertion(+) diff --git a/static/js/app.js b/static/js/app.js index c55cf5eb5..4e51e4708 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -2465,6 +2465,7 @@ quayApp.directive('dockerAuthDialog', function (Config) { controller: function($scope, $element) { var updateCommand = function() { var escape = function(v) { + if (!v) { return v; } return v.replace('$', '\\$'); }; $scope.command = 'docker login -e="." -u="' + escape($scope.username) + From 1e7e012b923b5b4cbf7fc850ecd54c4c487240e6 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 3 Sep 2014 15:41:25 -0400 Subject: [PATCH 46/57] Add a requirement for the current password to change the user's password or email address --- endpoints/api/user.py | 22 +++++++++++++++++++++- static/js/app.js | 2 +- static/js/controllers.js | 2 ++ static/partials/user-admin.html | 11 ++++++++--- test/test_api_usage.py | 26 +++++++++++++++++++++++--- 5 files changed, 55 insertions(+), 8 deletions(-) diff --git a/endpoints/api/user.py b/endpoints/api/user.py index e2e6a0ff4..054db7041 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -117,6 +117,10 @@ class User(ApiResource): 'type': 'object', 'description': 'Fields which can be updated in a user.', 'properties': { + 'current_password': { + 'type': 'string', + 'description': 'The user\'s current password', + }, 'password': { 'type': 'string', 'description': 'The user\'s password', @@ -152,8 +156,22 @@ class User(ApiResource): user = get_authenticated_user() user_data = request.get_json() - try: + def verify_current_password(user, user_data): + current_password = user_data.get('current_password', '') + + verified = False + try: + verified = model.verify_user(user.username, current_password) + except: + pass + + if not verified: + raise request_error(message='Current password does not match') + + try: if 'password' in user_data: + verify_current_password(user, user_data) + logger.debug('Changing password for user: %s', user.username) log_action('account_change_password', user.username) model.change_password(user, user_data['password']) @@ -163,6 +181,8 @@ class User(ApiResource): model.change_invoice_email(user, user_data['invoice_email']) if 'email' in user_data and user_data['email'] != user.email: + verify_current_password(user, user_data) + new_email = user_data['email'] if model.find_user_by_email(new_email): # Email already used. diff --git a/static/js/app.js b/static/js/app.js index 4e51e4708..72eabc41a 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -384,7 +384,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading var uiService = {}; uiService.hidePopover = function(elem) { - var popover = $('#signupButton').data('bs.popover'); + var popover = $(elem).data('bs.popover'); if (popover) { popover.hide(); } diff --git a/static/js/controllers.js b/static/js/controllers.js index 4d1c8484f..485b7f529 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -1763,6 +1763,7 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use // Reset the form. delete $scope.cuser['repeatEmail']; + delete $scope.cuser['current_password']; $scope.changeEmailForm.$setPristine(); }, function(result) { @@ -1784,6 +1785,7 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use // Reset the form delete $scope.cuser['password'] delete $scope.cuser['repeatPassword'] + delete $scope.cuser['current_password']; $scope.changePasswordForm.$setPristine(); diff --git a/static/partials/user-admin.html b/static/partials/user-admin.html index 1b2ad7fd1..260aa47e2 100644 --- a/static/partials/user-admin.html +++ b/static/partials/user-admin.html @@ -128,6 +128,8 @@
    +
    @@ -138,18 +140,21 @@
    -
    - -
    Change Password
    +
    + +
    + Password changed successfully
    + Date: Wed, 3 Sep 2014 17:24:52 -0400 Subject: [PATCH 47/57] Up the gunicorn worker count (under protest) --- conf/gunicorn_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/gunicorn_config.py b/conf/gunicorn_config.py index 4d9d50499..ca8ad5363 100644 --- a/conf/gunicorn_config.py +++ b/conf/gunicorn_config.py @@ -1,5 +1,5 @@ bind = 'unix:/tmp/gunicorn.sock' -workers = 8 +workers = 16 worker_class = 'gevent' timeout = 2000 logconfig = 'conf/logging.conf' From e783df31e0471346d25affaf74172038a17fcba6 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 4 Sep 2014 14:24:20 -0400 Subject: [PATCH 48/57] Add the concept of require_fresh_login to both the backend and frontend. Sensitive methods will now be marked with the annotation, which requires that the user has performed a login within 10 minutes or they are asked to do so in the UI before running the operation again. --- endpoints/api/__init__.py | 28 ++++++++++- endpoints/api/discovery.py | 5 ++ endpoints/api/user.py | 54 ++++++++++++--------- endpoints/common.py | 4 +- static/js/app.js | 84 +++++++++++++++++++++++++++++---- static/js/controllers.js | 7 ++- static/partials/user-admin.html | 4 -- test/test_api_security.py | 27 +++++++++-- test/test_api_usage.py | 22 ++------- 9 files changed, 174 insertions(+), 61 deletions(-) diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index 854c3cad1..9f9a8c941 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -1,7 +1,8 @@ import logging import json +import datetime -from flask import Blueprint, request, make_response, jsonify +from flask import Blueprint, request, make_response, jsonify, session from flask.ext.restful import Resource, abort, Api, reqparse from flask.ext.restful.utils.cors import crossdomain from werkzeug.exceptions import HTTPException @@ -66,6 +67,11 @@ class Unauthorized(ApiException): ApiException.__init__(self, 'insufficient_scope', 403, 'Unauthorized', payload) +class FreshLoginRequired(ApiException): + def __init__(self, payload=None): + ApiException.__init__(self, 'fresh_login_required', 401, "Requires fresh login", payload) + + class ExceedsLicenseException(ApiException): def __init__(self, payload=None): ApiException.__init__(self, None, 402, 'Payment Required', payload) @@ -264,6 +270,26 @@ def require_user_permission(permission_class, scope=None): require_user_read = require_user_permission(UserReadPermission, scopes.READ_USER) require_user_admin = require_user_permission(UserAdminPermission, None) +require_fresh_user_admin = require_user_permission(UserAdminPermission, None) + +def require_fresh_login(func): + @add_method_metadata('requires_fresh_login', True) + @wraps(func) + def wrapped(*args, **kwargs): + user = get_authenticated_user() + if not user: + raise Unauthorized() + + logger.debug('Checking fresh login for user %s', user.username) + + last_login = session.get('login_time', datetime.datetime.now() - datetime.timedelta(minutes=60)) + valid_span = datetime.datetime.now() - datetime.timedelta(minutes=10) + + if last_login >= valid_span: + return func(*args, **kwargs) + + raise FreshLoginRequired() + return wrapped def require_scope(scope_object): diff --git a/endpoints/api/discovery.py b/endpoints/api/discovery.py index ee8702636..1995c6b42 100644 --- a/endpoints/api/discovery.py +++ b/endpoints/api/discovery.py @@ -119,6 +119,11 @@ def swagger_route_data(include_internal=False, compact=False): if internal is not None: new_operation['internal'] = True + if include_internal: + requires_fresh_login = method_metadata(method, 'requires_fresh_login') + if requires_fresh_login is not None: + new_operation['requires_fresh_login'] = True + if not internal or (internal and include_internal): operations.append(new_operation) diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 054db7041..eb99ba0fa 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -9,7 +9,7 @@ from app import app, billing as stripe, authentication from endpoints.api import (ApiResource, nickname, resource, validate_json_request, request_error, log_action, internal_only, NotFound, require_user_admin, parse_args, query_param, InvalidToken, require_scope, format_date, hide_if, show_if, - license_error) + license_error, require_fresh_login) from endpoints.api.subscribe import subscribe from endpoints.common import common_login from data import model @@ -117,10 +117,6 @@ class User(ApiResource): 'type': 'object', 'description': 'Fields which can be updated in a user.', 'properties': { - 'current_password': { - 'type': 'string', - 'description': 'The user\'s current password', - }, 'password': { 'type': 'string', 'description': 'The user\'s password', @@ -148,6 +144,7 @@ class User(ApiResource): return user_view(user) @require_user_admin + @require_fresh_login @nickname('changeUserDetails') @internal_only @validate_json_request('UpdateUser') @@ -156,22 +153,8 @@ class User(ApiResource): user = get_authenticated_user() user_data = request.get_json() - def verify_current_password(user, user_data): - current_password = user_data.get('current_password', '') - - verified = False - try: - verified = model.verify_user(user.username, current_password) - except: - pass - - if not verified: - raise request_error(message='Current password does not match') - try: if 'password' in user_data: - verify_current_password(user, user_data) - logger.debug('Changing password for user: %s', user.username) log_action('account_change_password', user.username) model.change_password(user, user_data['password']) @@ -181,8 +164,6 @@ class User(ApiResource): model.change_invoice_email(user, user_data['invoice_email']) if 'email' in user_data and user_data['email'] != user.email: - verify_current_password(user, user_data) - new_email = user_data['email'] if model.find_user_by_email(new_email): # Email already used. @@ -377,6 +358,37 @@ class Signin(ApiResource): return conduct_signin(username, password) +@resource('/v1/signin/verify') +@internal_only +class VerifyUser(ApiResource): + """ Operations for verifying the existing user. """ + schemas = { + 'VerifyUser': { + 'id': 'VerifyUser', + 'type': 'object', + 'description': 'Information required to verify the signed in user.', + 'required': [ + 'password', + ], + 'properties': { + 'password': { + 'type': 'string', + 'description': 'The user\'s password', + }, + }, + }, + } + + @require_user_admin + @nickname('verifyUser') + @validate_json_request('VerifyUser') + def post(self): + """ Verifies the signed in the user with the specified credentials. """ + signin_data = request.get_json() + password = signin_data['password'] + return conduct_signin(get_authenticated_user().username, password) + + @resource('/v1/signout') @internal_only class Signout(ApiResource): diff --git a/endpoints/common.py b/endpoints/common.py index fe09104ca..52715a1d1 100644 --- a/endpoints/common.py +++ b/endpoints/common.py @@ -2,8 +2,9 @@ import logging import urlparse import json import string +import datetime -from flask import make_response, render_template, request, abort +from flask import make_response, render_template, request, abort, session from flask.ext.login import login_user, UserMixin from flask.ext.principal import identity_changed from random import SystemRandom @@ -112,6 +113,7 @@ def common_login(db_user): logger.debug('Successfully signed in as: %s' % db_user.username) new_identity = QuayDeferredPermissionUser(db_user.username, 'username', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=new_identity) + session['login_time'] = datetime.datetime.now() return True else: logger.debug('User could not be logged in, inactive?.') diff --git a/static/js/app.js b/static/js/app.js index 72eabc41a..d23235a66 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -713,7 +713,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading return config; }]); - $provide.factory('ApiService', ['Restangular', function(Restangular) { + $provide.factory('ApiService', ['Restangular', '$q', function(Restangular, $q) { var apiService = {}; var getResource = function(path, opt_background) { @@ -810,6 +810,65 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading } }; + var freshLoginFailCheck = function(opName, opArgs) { + return function(resp) { + var deferred = $q.defer(); + + // If the error is a fresh login required, show the dialog. + if (resp.status == 401 && resp.data['error_type'] == 'fresh_login_required') { + bootbox.dialog({ + "message": 'It has been more than a few minutes since you last logged in, ' + + 'so please verify your password to perform this sensitive operation:' + + '' + + '' + + '', + "title": 'Please Verify', + "buttons": { + "verify": { + "label": "Verify", + "className": "btn-success", + "callback": function() { + var info = { + 'password': $('#freshPassword').val() + }; + + $('#freshPassword').val(''); + + // Conduct the sign in of the user. + apiService.verifyUser(info).then(function() { + // On success, retry the operation. if it succeeds, then resolve the + // deferred promise with the result. Otherwise, reject the same. + apiService[opName].apply(apiService, opArgs).then(function(resp) { + deferred.resolve(resp); + }, function(resp) { + deferred.reject(resp); + }); + }, function(resp) { + // Reject with the sign in error. + deferred.reject({'data': {'message': 'Invalid verification credentials'}}); + }); + } + }, + "close": { + "label": "Cancel", + "className": "btn-default", + "callback": function() { + deferred.reject(resp); + } + } + } + }); + + // Return a new promise. We'll accept or reject it based on the result + // of the login. + return deferred.promise; + } + + // Otherwise, we just 'raise' the error via the reject method on the promise. + return $q.reject(resp); + }; + }; + var buildMethodsForOperation = function(operation, resource, resourceMap) { var method = operation['method'].toLowerCase(); var operationName = operation['nickname']; @@ -823,7 +882,15 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading 'ignoreLoadingBar': true }); } - return one['custom' + method.toUpperCase()](opt_options); + + var opObj = one['custom' + method.toUpperCase()](opt_options); + + // If the operation requires_fresh_login, then add a specialized error handler that + // will defer the operation's result if sudo is requested. + if (operation['requires_fresh_login']) { + opObj = opObj.catch(freshLoginFailCheck(operationName, arguments)); + } + return opObj; }; // If the method for the operation is a GET, add an operationAsResource method. @@ -3923,9 +3990,11 @@ quayApp.directive('billingOptions', function () { var save = function() { $scope.working = true; + + var errorHandler = ApiService.errorDisplay('Could not change user details'); ApiService.changeDetails($scope.organization, $scope.obj).then(function(resp) { $scope.working = false; - }); + }, errorHandler); }; var checkSave = function() { @@ -5699,11 +5768,10 @@ quayApp.run(['$location', '$rootScope', 'Restangular', 'UserService', 'PlanServi // Handle session expiration. Restangular.setErrorInterceptor(function(response) { - if (response.status == 401) { - if (response.data['session_required'] == null || response.data['session_required'] === true) { - $('#sessionexpiredModal').modal({}); - return false; - } + if (response.status == 401 && response.data['error_type'] == 'invalid_token' && + response.data['session_required'] !== false) { + $('#sessionexpiredModal').modal({}); + return false; } if (response.status == 503) { diff --git a/static/js/controllers.js b/static/js/controllers.js index 485b7f529..d95a760a9 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -1763,12 +1763,11 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use // Reset the form. delete $scope.cuser['repeatEmail']; - delete $scope.cuser['current_password']; $scope.changeEmailForm.$setPristine(); }, function(result) { $scope.updatingUser = false; - UIService.showFormError('#changeEmailForm', result); + UIService.showFormError('#changeEmailForm', result); }); }; @@ -1778,14 +1777,14 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use $scope.updatingUser = true; $scope.changePasswordSuccess = false; - ApiService.changeUserDetails($scope.cuser).then(function() { + ApiService.changeUserDetails($scope.cuser).then(function(resp) { + $scope.updatingUser = false; $scope.changePasswordSuccess = true; // Reset the form delete $scope.cuser['password'] delete $scope.cuser['repeatPassword'] - delete $scope.cuser['current_password']; $scope.changePasswordForm.$setPristine(); diff --git a/static/partials/user-admin.html b/static/partials/user-admin.html index 260aa47e2..4349d9df9 100644 --- a/static/partials/user-admin.html +++ b/static/partials/user-admin.html @@ -128,8 +128,6 @@
    -
    @@ -153,8 +151,6 @@
    - Date: Thu, 4 Sep 2014 17:54:51 -0400 Subject: [PATCH 49/57] Code review fixes --- endpoints/api/user.py | 1 - endpoints/callbacks.py | 43 +++++++++++++++++--------------------- templates/ologinerror.html | 2 +- 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/endpoints/api/user.py b/endpoints/api/user.py index d0d089dcd..fb09d012a 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -39,7 +39,6 @@ def user_view(user): organizations = model.get_user_organizations(user.username) def login_view(login): - print login.metadata_json try: metadata = json.loads(login.metadata_json) except: diff --git a/endpoints/callbacks.py b/endpoints/callbacks.py index 49fa1e8a6..1cbd46192 100644 --- a/endpoints/callbacks.py +++ b/endpoints/callbacks.py @@ -4,7 +4,7 @@ from flask import request, redirect, url_for, Blueprint from flask.ext.login import current_user from endpoints.common import render_page_template, common_login, route_show_if -from app import app, analytics +from app import app, analytics, get_app_url from data import model from util.names import parse_repository_name from util.validation import generate_valid_usernames @@ -22,6 +22,11 @@ client = app.config['HTTPCLIENT'] callback = Blueprint('callback', __name__) +def render_ologin_error(service_name, + error_message='Could not load user data. The token may have expired.'): + return render_page_template('ologinerror.html', service_name=service_name, + error_message=error_message, + service_url=get_app_url()) def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_encode=False, redirect_suffix=''): @@ -96,15 +101,12 @@ def conduct_oauth_login(service_name, user_id, username, email, metadata={}): analytics.alias(to_login.username, state) except model.DataModelException, ex: - return render_page_template('ologinerror.html', service_name=service_name, - error_message=ex.message) + return render_ologin_error(service_name, ex.message) if common_login(to_login): return redirect(url_for('web.index')) - return render_page_template('ologinerror.html', service_name=service_name, - error_message='Unknown error') - + return render_ologin_error(service_name) def get_google_username(user_data): username = user_data['email'] @@ -120,17 +122,16 @@ def get_google_username(user_data): def google_oauth_callback(): error = request.args.get('error', None) if error: - return render_page_template('ologinerror.html', service_name='Google', error_message=error) + return render_ologin_error('Google', error) token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE', form_encode=True) user_data = get_google_user(token) if not user_data or not user_data.get('id', None) or not user_data.get('email', None): - return render_page_template('ologinerror.html', service_name = 'Google', - error_message='Could not load user data') - + return render_ologin_error('Google') + username = get_google_username(user_data) metadata = { - 'service_username': username + 'service_username': user_data['email'] } return conduct_oauth_login('Google', user_data['id'], username, user_data['email'], @@ -142,13 +143,12 @@ def google_oauth_callback(): def github_oauth_callback(): error = request.args.get('error', None) if error: - return render_page_template('ologinerror.html', service_name = 'GitHub', error_message=error) + return render_ologin_error('GitHub', error) token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB') user_data = get_github_user(token) if not user_data: - return render_page_template('ologinerror.html', service_name = 'GitHub', - error_message='Could not load user data') + return render_ologin_error('GitHub') username = user_data['login'] github_id = user_data['id'] @@ -186,15 +186,14 @@ def google_oauth_attach(): user_data = get_google_user(token) if not user_data or not user_data.get('id', None): - return render_page_template('ologinerror.html', service_name = 'Google', - error_message='Could not load user data') + return render_ologin_error('Google') google_id = user_data['id'] user_obj = current_user.db_user() username = get_google_username(user_data) metadata = { - 'service_username': username + 'service_username': user_data['email'] } try: @@ -202,9 +201,7 @@ def google_oauth_attach(): except IntegrityError: err = 'Google account %s is already attached to a %s account' % ( username, app.config['REGISTRY_TITLE_SHORT']) - - return render_page_template('ologinerror.html', service_name = 'Google', - error_message=err) + return render_ologin_error('Google', err) return redirect(url_for('web.user')) @@ -216,8 +213,7 @@ def github_oauth_attach(): token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB') user_data = get_github_user(token) if not user_data: - return render_page_template('ologinerror.html', service_name = 'GitHub', - error_message='Could not load user data') + return render_ologin_error('GitHub') github_id = user_data['id'] user_obj = current_user.db_user() @@ -233,8 +229,7 @@ def github_oauth_attach(): err = 'Github account %s is already attached to a %s account' % ( username, app.config['REGISTRY_TITLE_SHORT']) - return render_page_template('ologinerror.html', service_name = 'Github', - error_message=err) + return render_ologin_error('GitHub', err) return redirect(url_for('web.user')) diff --git a/templates/ologinerror.html b/templates/ologinerror.html index cd921eec2..304b7f554 100644 --- a/templates/ologinerror.html +++ b/templates/ologinerror.html @@ -15,7 +15,7 @@ {% endif %}
    - Please register using the registration form to continue. + Please register using the registration form to continue. You will be able to connect your account to your Quay.io account in the user settings.
    From b9a4d2835f89da831abf3ce35ce79808a2cd79b4 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 4 Sep 2014 18:18:19 -0400 Subject: [PATCH 50/57] Add migration for the new DB field --- ...a_add_metadata_field_to_external_logins.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py diff --git a/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py b/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py new file mode 100644 index 000000000..c642dcbee --- /dev/null +++ b/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py @@ -0,0 +1,26 @@ +"""add metadata field to external logins + +Revision ID: 1594a74a74ca +Revises: f42b0ea7a4d +Create Date: 2014-09-04 18:17:35.205698 + +""" + +# revision identifiers, used by Alembic. +revision = '1594a74a74ca' +down_revision = 'f42b0ea7a4d' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('federatedlogin', sa.Column('metadata_json', sa.Text(), nullable=False)) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('federatedlogin', 'metadata_json') + ### end Alembic commands ### From 6fa5a365b3f664346d150fd18a7fd0255194034b Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 4 Sep 2014 18:45:23 -0400 Subject: [PATCH 51/57] Add loginservice for Google --- ...4ca_add_metadata_field_to_external_logins.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py b/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py index c642dcbee..a59116c7f 100644 --- a/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py +++ b/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py @@ -12,6 +12,9 @@ down_revision = 'f42b0ea7a4d' from alembic import op import sqlalchemy as sa +from sqlalchemy.dialects import mysql +from data.model.sqlalchemybridge import gen_sqlalchemy_metadata +from data.database import all_models def upgrade(): @@ -19,8 +22,22 @@ def upgrade(): op.add_column('federatedlogin', sa.Column('metadata_json', sa.Text(), nullable=False)) ### end Alembic commands ### + schema = gen_sqlalchemy_metadata(all_models) + + op.bulk_insert(schema.tables['loginservice'], + [ + {'id':4, 'name':'google'}, + ]) def downgrade(): ### commands auto generated by Alembic - please adjust! ### op.drop_column('federatedlogin', 'metadata_json') ### end Alembic commands ### + + schema = gen_sqlalchemy_metadata(all_models) + loginservice = schema.table['loginservice'] + + op.execute( + (loginservice.delete() + .where(loginservice.c.name == op.inline_literal('google'))) + ) From 1a230f635af589cf75a8a65aae0fcf8ba2a6348e Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Thu, 4 Sep 2014 19:15:06 -0400 Subject: [PATCH 52/57] Use datetime.min instead of a fixed span for the last login default time. --- endpoints/api/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index 9f9a8c941..a9d2ecdb0 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -282,7 +282,7 @@ def require_fresh_login(func): logger.debug('Checking fresh login for user %s', user.username) - last_login = session.get('login_time', datetime.datetime.now() - datetime.timedelta(minutes=60)) + last_login = session.get('login_time', datetime.datetime.min) valid_span = datetime.datetime.now() - datetime.timedelta(minutes=10) if last_login >= valid_span: From 987177fd7e45800a549e2fed6e410741abf226de Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 4 Sep 2014 19:47:12 -0400 Subject: [PATCH 53/57] Have require_fresh_login not apply if there is no password set for the user --- endpoints/api/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index a9d2ecdb0..2f5e2045e 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -285,7 +285,7 @@ def require_fresh_login(func): last_login = session.get('login_time', datetime.datetime.min) valid_span = datetime.datetime.now() - datetime.timedelta(minutes=10) - if last_login >= valid_span: + if not user.password_hash or last_login >= valid_span: return func(*args, **kwargs) raise FreshLoginRequired() From f746eb33819fcf1ba66bbd75c02991792cdbf114 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 4 Sep 2014 20:04:49 -0400 Subject: [PATCH 54/57] Make the fresh login dialog autofocus the input and make it handle the enter key properly. --- static/js/app.js | 58 +++++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/static/js/app.js b/static/js/app.js index d23235a66..e0b431d7b 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -816,7 +816,31 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading // If the error is a fresh login required, show the dialog. if (resp.status == 401 && resp.data['error_type'] == 'fresh_login_required') { - bootbox.dialog({ + var verifyNow = function() { + if (!$('#freshPassword').val()) { return; } + + var info = { + 'password': $('#freshPassword').val() + }; + + $('#freshPassword').val(''); + + // Conduct the sign in of the user. + apiService.verifyUser(info).then(function() { + // On success, retry the operation. if it succeeds, then resolve the + // deferred promise with the result. Otherwise, reject the same. + apiService[opName].apply(apiService, opArgs).then(function(resp) { + deferred.resolve(resp); + }, function(resp) { + deferred.reject(resp); + }); + }, function(resp) { + // Reject with the sign in error. + deferred.reject({'data': {'message': 'Invalid verification credentials'}}); + }); + }; + + var box = bootbox.dialog({ "message": 'It has been more than a few minutes since you last logged in, ' + 'so please verify your password to perform this sensitive operation:' + '' + @@ -827,38 +851,26 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading "verify": { "label": "Verify", "className": "btn-success", - "callback": function() { - var info = { - 'password': $('#freshPassword').val() - }; - - $('#freshPassword').val(''); - - // Conduct the sign in of the user. - apiService.verifyUser(info).then(function() { - // On success, retry the operation. if it succeeds, then resolve the - // deferred promise with the result. Otherwise, reject the same. - apiService[opName].apply(apiService, opArgs).then(function(resp) { - deferred.resolve(resp); - }, function(resp) { - deferred.reject(resp); - }); - }, function(resp) { - // Reject with the sign in error. - deferred.reject({'data': {'message': 'Invalid verification credentials'}}); - }); - } + "callback": verifyNow }, "close": { "label": "Cancel", "className": "btn-default", "callback": function() { - deferred.reject(resp); + deferred.reject({'data': {'message': 'Verification canceled'}}); } } } }); + box.bind('shown.bs.modal', function(){ + box.find("input").focus(); + box.find("form").submit(function() { + box.modal('hide'); + verifyNow(); + }); + }); + // Return a new promise. We'll accept or reject it based on the result // of the login. return deferred.promise; From 4e04ad5ca7f4192f23040899641178e46b02d02e Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 4 Sep 2014 20:05:21 -0400 Subject: [PATCH 55/57] Move the password check before we hide the modal --- static/js/app.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/static/js/app.js b/static/js/app.js index e0b431d7b..97568e7fb 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -817,8 +817,6 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading // If the error is a fresh login required, show the dialog. if (resp.status == 401 && resp.data['error_type'] == 'fresh_login_required') { var verifyNow = function() { - if (!$('#freshPassword').val()) { return; } - var info = { 'password': $('#freshPassword').val() }; @@ -866,6 +864,8 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading box.bind('shown.bs.modal', function(){ box.find("input").focus(); box.find("form").submit(function() { + if (!$('#freshPassword').val()) { return; } + box.modal('hide'); verifyNow(); }); From 19a589ba54d64944a3e6777cd1cb692d9797291c Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Thu, 4 Sep 2014 20:11:42 -0400 Subject: [PATCH 56/57] Update the test db to have the google login service. --- test/data/test.db | Bin 231424 -> 231424 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/test/data/test.db b/test/data/test.db index 3e631b5d72e5c2c290e95affc3a9794853a6a573..f76110cc23548c74a99fe0ce64bfd130c91fc2aa 100644 GIT binary patch delta 6273 zcmbuCeRvery~lUvOxWxuAqG$k5(pt62~INaFDX)Y*|%hqeMz#(?%D`5yE6pBCSgeo z32Y(1+); ze{8bzJ1?K#_x!$R&RMs4{JPEKH(!SZ?wPs(3p~BZ0NCtZzXuh8LIS*D1D}D9z`Nj$ zT_ra>4{o#*z;^Yrw9;D$fNlEneq}J*qxSSUa9}H4yri$QUv*&Czdb#RmuXI$%|?L3 zHt;bx2@ZoJXowfU@Ov%!l|&Jkj-BldoS+mt-5Ym+Qpf4uJ6jzd`=j_b@eP37b=6i5 zUmN!GUKi&HGE!?}vlyVI03QhQbUPnvr2T%mt)AoA#ujft@Up>1IYtL%)*WKyxIY{d z++HD3=p~FItX&KQTB5ulgxjQ;ONc~VR-2HozO9){NUd>qAQXs(d9HCmlebk2#_HMj zh+Bxn<91)NUX-Kq@aXqDZ;HqHWVBrt>Jwhk=cc10gr916i?N1qG?;WZR2@ zYewN-d~J}h+c3;1Dk%h4D&49zL694)09>YmYsX5MQxJ`Y=SIr|SE?i$49g9MgYy;j z{eL7~wWi3bTHB{K+oyOG|21eBo>BNm=lbMK`{b%B?AtxVgHIfUy=6GLt7&^NzH{U_ zmIl#`R9EANB_)$;=mZQaZ9fyyM;Pxx7V8Wk=))Twj6}Wf-*2d$x z6#%(`u2g*7X8}1b(EY1rpYqkEyz`kqfhnMnuoD=PvH+a^n}?4>za0QKpTNcu&cEAi zf5#rizW}SyqI@*`tsRf;diX#`(HvS5m`FX-=1$O!u7nhF$Hn$gy}Lcm2ZeeMAC)3u z!R3mz4L^5i)^OlZcuw5qrV{Qr6;A}4U5RLm$Rs$qt<4)sMx&e@X^6JP8{*;m_^#&; zohXjV950EoK#{7hFeE3l5~(QYi`EoIOYv$tMR8fJx2H}?XO?vLX)F5rdb+f3hN3x= zl1Yj$r|0rLzQVXJ1?e0Qe|igP0tCWno`mvqo)-y zMbjeBv+#WZqt^PeOKvDf)3s>5<<{)w6jeu4bv#o}Qs`50LwQAC&%(6Qr&ZLISBz~< zU(kBZ*#tN=+UZaY%gU0(8U*IB#W07F=gLo|8Ie&{ie#tg5>Ilvs3AU0C)0wYN-QTc zDOQBrMa)-n!8n{u(WZpGRzO|w*cu)$11`sD zgP7wS?N`H^jo3=CHkGDzJ}shUW_Xe0gfvB>MVCpMPEl+s&1$li26!!=Z^L2^w9h&f zRiRW`q)47gqYf%9krJcu=qJr^T8dG%G>d~;>{JKLhB+lo(kO)zNM6y=!t%OE%A%s8 zFICd)DyIvQNbUJo7z@v-R5ePHRE{TgjZ2YSnnTU_6idpwEXfQi1ujLwO!H(8C{}OHZ7q_gu{cFyHZ1|t>}tG@*JuNPU2}& zVPuV@(u|;@`B^#5z`qS*b1Nl*(m7etNr6EH!%1|SRA~kgFp9*bDUsD^0S1OJS0&HO z5{D!qWlhjYjzNTqEYT$LfXGQ2$BDEKpB%!bR_dy#>yoOIdMbrDDdbi~qct)`X{nT; zYl@;u@c0lGsN@xaQaJ|YVO1(8rxa46Re=;G5#@rUW+)yeR$9O2(6evD-5qO z3dO2>_PvEgr=4>);t)7qpkVn)?41i9qZ1d+!Gg~(dW`lnUf6T;bL=tq+Or!0UnBAq zT7$xAz=e%X_Ds0o>%-2k4?8XnuRXoVof$p<;`oJO`?;`0vh7lf&c!+Dno zccQKOOn2h^I3r)&9ejQGybV+$$M!bo;m^-4<+xCkYvtL7q{kauko3*P#@R5Qbl|5> zlvjm9L4T7g+#L7%d7rl_W_IFe*L@3of_B}PLDeKcVi%8qPXwf6Out_I;cFR9ul4AW zfm*FA5NS=eE=~0H(+!Q?UG1@D14?~{%VflXq1uL6)Emfn25Yj(e)!r(d~ykTcb=A= zu0gLkWI6c7M!cMNw=8f42UFaN=18I?BSf=wQ)5_*2NX3G>e8B{Nl$m8xmJpE-Aniu zaUdhqGYi77Y14%xvJ@|JGVI@k&q7C|D?8AitSnJ`~5GJqh-F2QT9X`r`e8!3EyM?R~wvkjNxE6HE1QXQQ9d zTYYJ@D;Z*kc%Pmb2o5Fnl^(5M>8xeof$!im)}H03YDAf0czOUYvXM5UDBqUvK>TU# z);>jDrp-;ElC})v4hI)EDWZ@H zW}<_=HQ{i)zP_!;zeJ7&Ljy`ni{8j;Qdg>PB|X4I0_|1}*Pc^{s^J;5|A^*y@%OQX zS9%+|2pJJuCp(GZn4y9q8w(G751%xA^ROd%nKz}2&*^86>6e;_m09$DhUQ)Pf^+$y zZC7MDN`%h4kvm$K$7^(^ry=a+vNirD@6vEy#s^Nt^H~!y0pHVV%i!5g1gyY z8|$oX5SrWJ!n^Uw+S%1-YA8YCIe8$z$X06PRpqKTt10QTr8`)Yq~n?MJwq+sJ@lB;l_;4OD04I3jS&@* z9Us8#J@|3^#Ai;77#SQMc>%u>{^(wO#C~4=Y%7#p?({_kKaz@?NtCQeh@!S@IO3enh)XK4r1Q%|I}f2 z8{%_rxPFt784r&fw)lRG_$03H+dg_{PT{Tmwg+Mtoou z=P}AA!2KUsd^-_eu}HKVdI?~9G!joCc4iCig7=7i*29nYIHS4pY7=O zvkgAG8-IwHF!d+~JARHIyNVllWYF+lSo|{Lb51K?WWHkYIrFys0cKxE zeB4`}qef;nJaV7K_iMyg^ZT%2RL_F1Zn5~@Kz!E~>z|s3tsNdmQ1_<{JY>hJi<(vheqx#02h3sG{QTXxqLS{Zd>Rsj8@jE!` z0Nd?XRX#jql(};ja)J(f^|D9j7`iKGp#rdtn0wP-lV%}6coR`_u%iSVvzOd`SDoSW zFU*!Az8MQw{KUvK+;AinD-ZkcUV$8zqdMSP6h_8%r?@sXwfUIa&J#5d!`wGS9&$(%bF#K$C8 zD~6uPxr0S~bDSFAaU(Vvk@S7T(?(y33hZKJ}15B4#T%J=Z;$tAMyU0 zDX@4U;>-V14V#NBKK3^&N?>*g;^SiLvqq*nXOE?buc~x?w^5zR*`o{b3HFVDF_&5T zv1^(hhoc$9H=p`;zEPIV*`pirIX9F4W9Z9s_UJ);Rd+r2so87khdpw?2|Jb}zUjW1 ze>Qx#=IoJ0eEg2BTVe4E#3%XhJZbiUh2!iXm?||gLpfIrA*%8B92zmI2Q633hxe@l zodm(mi<+xJggAGJIde9L$~xe`#N*!?U=E}VK)_?`z?@4oK>emb$pG$4#8BrdxZ41w zmyX>uVeA#d;+O&()`K|{*fAFjc#39ZJ*d7^@PqZBwPeh#!W!^usNM!@FA;n6Hc&VA Y@t)(ifd+61=D;0bc-CcHAS>4VA9FG*y8r+H delta 6218 zcmbuCeRvery~lUvY_i#e5Q1~3Pfuw{fsNXRaDiGK|d|lvZOyQQuU0Va)q7#S|Jb% z^Kr>H*!dvmuBh^OgT8t(QWSGjxezKga-sKI)8$3ZL7l^prwY9GL zs#r`6hr&!nkaTfTd z1LrE`;NL;LO_$1-R?ml@E0uv?sf0f4TqPWzSjTI=y^y)6K{G6E7E2rU3|@{u24McZ zcxaHEG$nh>H4AJr7B9wbrUp+xaRBDrj!lM7=V3G~KZX?yuXyU0*x=tscI_D&18{5; zHe=ZOY=+g=0sa&JEm(|};Y}R=WC+jPbj3?x^=6=@#J+kDnITq zf$aZ6{BhdP{)R5z^THp%Y;xn4rAR`s~y{E!FJ-Ozzs;$ z9oX>B{f)VcJiOp{M?!)a47lT*BjSk&j9A5qbht8FT~p6ji;Tego$4d!u?0 z>xoiygWFkKAL86BE4dqdQpo2Zd75^+!gQ4sp{rvx4b{Uhzwt?4nCC@~A^8-cvJ^$I zoRA`tyv!0QmE~DlQW;v&I@R{J<+9S!+}f>nb$7R|S6gY4VhK_rNV3R;R_WtAr6q$&{tOEL_jNn}zM;RifMZmR83 zmlmPnqH24QIeHN#EvNbAELTL3=qg@XRNURRT9Lce;^jrfmpx6BmppybMH8U7b0IVV zNs@ToirFo7m|f3sWF?ad&kKw~u(N21Ay`@#2vJOl1dq5vsD1k@6Xp&(Cn<+FciTh4h zW7UgF1X^JgMPLXrnG^|DLd9jCqX?QudaDwblr#x;^kJ?NMN=tWMfz)u$Pp~hON7jl zGQny|CB?G>rSK&Dbsxr+@S;MiG#ZdsNHo5dN)e(g2!tj|8oEfHXGKW&W2+X3DHLR7 zf}?pfxF~3ZB&8@qW<*(&Q=*s@1vuD`IZJ3UmC{HB6=Hd$L5g8b(O5!}DN$ons;Dp= z{IVZgQo^EKF^SYt&{L=;&k!n)K`RoSM3ZDxMS=}CU}YteO7pDBGK5Mps0~CJnV%B{Z7rBK(1tc-*gDN*7?hNhB9<3?;()pb?gusarE{Q;>$ViGYX zTE|lkDOl%YMXsJBJ#nfw%*F%ZifGhJ2{rzh!^0$8bgaf*@1vt!B-*@%PV z1x|!TpJT@^xy?$55axU_{x;2V!oJVHz;-z|eZ3J#+3K2O{gq5XE|`TH9dFZTPf{^U8lU!sFr@90yt^|GgRnOx`aYw-pl*weGNt-OX!P)%}s z2QRk3{kP-uO%tByrv#Ccn2mU@g|O(kS(Yq2;#bt(ZaKL@U7~cK{l1&{&^_+l_YKFs zal+*w6R_nJFVO-GH*Ck}as6r)84$#{vv+N?n+qp5bg%7et_~=Q(;rd&{WXFowZYBR z^~I##09n=J4{05hJ!TItyPytJE>V<7lg1tR2iWQ>{R|q1jEK?6jv_@y9LUMFFmV45 z@o9tC4cZgm;VbE)L;8g?`a}bv!zjxz(6|$?xG+Do?V?i#MZ@g7kULye@yLd*s?M7DinAK7z6+nBe!co!ImL?7!qJ=Tt!dgQ`^tHicQr|V!JlX5Dvb1A;tZu+x)7;udjcZ-5oRS1dq>LSSD`w4asapU$@5aY$xnoC{=`A=s{!4r% zeDofC$W|2EzE>~9;Ne&B74V)xe4RCa;J;QI_u>aIYvEI`)xy#H@F#66{{uUs7iPkd zgXk&aM|h(xYr(okUCV%LUPG1Q5Z`T`x!Uo8aX;RGSr>-xsez3@#y_%^KKyB)?#YHj zM^N6j2k;+Q?IX`T4f7tv$E-yw`;*2)c&pvI%FF&5cJ4-eS)awL^p?r+_**95V~CHf z=(<@i%Y}#EHu)Y$d{Yixmu)SYD+@Mk9D z%ZRb8rO;-K;0H3S)T`DT;pi)fac<|xF}-ju9QmiocmOfx-P!glT`PoZzA_nKMU1OX zjTRXPO%0cv{P)SQ@ioLZ1^o3P-BS#QcA&g%qe#OA?=F59<{d(OCH*y@7_Xar*qysR zft_z6z6F12n4-5_4UgYz^8E_&<-T~^B)x1QJba(Y_uq)`>SzCW!g$MEe^zs=Eo-AJBV+}$)-n*qj*EUb>7U6Ddgn55Lc=#V4dd2wIY{A?o9(VzEjv+kz=&9{`3k{DyYvTPG z;pLq9`Dwk3f`^|o@&1DFGT+@c-T1_m&mJh40!KeXc++or>{Gpvhx?y5@jgd*%kR5; zr>=2O-fQB0VJ`6tq|Z2oH)L4BmiPY&8^1z$(%Rqd*FDSO(9cY~(@4IBWb*--cLwq0 zeYz(3JV5G+S4}+%J2SwTZONgY2EAof+98=>$Tod=RzNRXnRbW`th3@=(P<+KAoa*E z_pO7YcJQPv^Omm;=!MR-L$X1m&35?Ia$R$z9g+ifTZ^6=>oX<+gje?D_FUMQ4?eQ3 z*!|P5bdM+Pkg4DY))~)LjKRFCz?hXBrT%10Gx??+IJg*g79hS^+rMz@EtP49%tCxp zXi1q~=1)6hHsa$Cw=Fg1n0&MUmJGwuLX&T+2d!=>?U1>MkGskKsjdan4w;Ae%7bOI zjUv#1*~ky)16_-rllI~HV7IMc*F$@CPXrEqY|2=SIOk(G{~hKnK%7(lZ_P_a2|!Ms z7OG6aP6BaeG=;PEmRQ;)<%mzVu*8L(Jr4Z+gg*zVC z3lnLV(1>%@&WgRd){u4ygE&`e8Sfh`;>~__1e#e4bkLV{CoGau+xF~a-JzXqPHZ|E^(Uc{FAQl>t%AmR;KC%7J zck7;YX_s7Y@@+mi2j;Cte7WDhNif!!eA1j=w9d_lFMs-L$Mlxgv`y9_KH>W9l3v!5 zw#j)tR<}yYzqPEdyzH3?Qoc zE0elj)^ECF6}mamf*Py&F1_@C59Q6kJHhSub+8DMI Date: Thu, 4 Sep 2014 20:58:29 -0400 Subject: [PATCH 57/57] Inject the tables metadata into the upgrade and downgrade functions. Fix a bunch of the downgrades to actually work. --- data/billing.py | 16 +-- data/migrations/env.py | 6 +- data/migrations/script.py.mako | 4 +- ...a_add_metadata_field_to_external_logins.py | 18 +-- ...49_remove_fields_from_image_table_that_.py | 4 +- ...c79d9_prepare_the_database_for_the_new_.py | 56 ++++----- ...9f_add_log_kind_for_regenerating_robot_.py | 18 +-- ...670cbeced_migrate_existing_webhooks_to_.py | 4 +- ...2_add_the_maintenance_notification_type.py | 15 +-- ...add_brute_force_prevention_metadata_to_.py | 4 +- .../5a07499ce53f_set_up_initial_database.py | 108 ++---------------- .../82297d834ad_add_us_west_location.py | 17 +-- ..._add_placements_and_locations_to_the_db.py | 14 +-- ...2b0ea7a4d_remove_the_old_webhooks_table.py | 4 +- util/collections.py | 12 ++ 15 files changed, 80 insertions(+), 220 deletions(-) create mode 100644 util/collections.py diff --git a/data/billing.py b/data/billing.py index 4847dd3f8..8c604aac2 100644 --- a/data/billing.py +++ b/data/billing.py @@ -3,6 +3,8 @@ import stripe from datetime import datetime, timedelta from calendar import timegm +from util.collections import AttrDict + PLANS = [ # Deprecated Plans { @@ -118,20 +120,6 @@ def get_plan(plan_id): return None -class AttrDict(dict): - def __init__(self, *args, **kwargs): - super(AttrDict, self).__init__(*args, **kwargs) - self.__dict__ = self - - @classmethod - def deep_copy(cls, attr_dict): - copy = AttrDict(attr_dict) - for key, value in copy.items(): - if isinstance(value, AttrDict): - copy[key] = cls.deep_copy(value) - return copy - - class FakeStripe(object): class Customer(AttrDict): FAKE_PLAN = AttrDict({ diff --git a/data/migrations/env.py b/data/migrations/env.py index c267c2f50..863e3d98f 100644 --- a/data/migrations/env.py +++ b/data/migrations/env.py @@ -8,6 +8,7 @@ from peewee import SqliteDatabase from data.database import all_models, db from app import app from data.model.sqlalchemybridge import gen_sqlalchemy_metadata +from util.collections import AttrDict # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -23,6 +24,7 @@ fileConfig(config.config_file_name) # from myapp import mymodel # target_metadata = mymodel.Base.metadata target_metadata = gen_sqlalchemy_metadata(all_models) +tables = AttrDict(target_metadata.tables) # other values from the config, defined by the needs of env.py, # can be acquired: @@ -45,7 +47,7 @@ def run_migrations_offline(): context.configure(url=url, target_metadata=target_metadata, transactional_ddl=True) with context.begin_transaction(): - context.run_migrations() + context.run_migrations(tables=tables) def run_migrations_online(): """Run migrations in 'online' mode. @@ -72,7 +74,7 @@ def run_migrations_online(): try: with context.begin_transaction(): - context.run_migrations() + context.run_migrations(tables=tables) finally: connection.close() diff --git a/data/migrations/script.py.mako b/data/migrations/script.py.mako index 95702017e..1b92f9f48 100644 --- a/data/migrations/script.py.mako +++ b/data/migrations/script.py.mako @@ -14,9 +14,9 @@ from alembic import op import sqlalchemy as sa ${imports if imports else ""} -def upgrade(): +def upgrade(tables): ${upgrades if upgrades else "pass"} -def downgrade(): +def downgrade(tables): ${downgrades if downgrades else "pass"} diff --git a/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py b/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py index a59116c7f..2f6c60706 100644 --- a/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py +++ b/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py @@ -13,31 +13,23 @@ down_revision = 'f42b0ea7a4d' from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models - -def upgrade(): +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.add_column('federatedlogin', sa.Column('metadata_json', sa.Text(), nullable=False)) ### end Alembic commands ### - schema = gen_sqlalchemy_metadata(all_models) - - op.bulk_insert(schema.tables['loginservice'], + op.bulk_insert(tables.loginservice, [ {'id':4, 'name':'google'}, ]) -def downgrade(): +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.drop_column('federatedlogin', 'metadata_json') ### end Alembic commands ### - schema = gen_sqlalchemy_metadata(all_models) - loginservice = schema.table['loginservice'] - op.execute( - (loginservice.delete() - .where(loginservice.c.name == op.inline_literal('google'))) + (tables.loginservice.delete() + .where(tables.loginservice.c.name == op.inline_literal('google'))) ) diff --git a/data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py b/data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py index ea36e3f57..d50c3a592 100644 --- a/data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py +++ b/data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py @@ -14,7 +14,7 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(): +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.drop_index('buildtriggerservice_name', table_name='buildtriggerservice') op.create_index('buildtriggerservice_name', 'buildtriggerservice', ['name'], unique=True) @@ -34,7 +34,7 @@ def upgrade(): ### end Alembic commands ### -def downgrade(): +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.drop_index('visibility_name', table_name='visibility') op.create_index('visibility_name', 'visibility', ['name'], unique=False) diff --git a/data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py b/data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py index 18c8bf654..e3be811b6 100644 --- a/data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py +++ b/data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py @@ -13,12 +13,8 @@ down_revision = '4b7ef0c7bdb2' from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models - -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.create_table('externalnotificationmethod', sa.Column('id', sa.Integer(), nullable=False), @@ -26,7 +22,7 @@ def upgrade(): sa.PrimaryKeyConstraint('id') ) op.create_index('externalnotificationmethod_name', 'externalnotificationmethod', ['name'], unique=True) - op.bulk_insert(schema.tables['externalnotificationmethod'], + op.bulk_insert(tables.externalnotificationmethod, [ {'id':1, 'name':'quay_notification'}, {'id':2, 'name':'email'}, @@ -38,7 +34,7 @@ def upgrade(): sa.PrimaryKeyConstraint('id') ) op.create_index('externalnotificationevent_name', 'externalnotificationevent', ['name'], unique=True) - op.bulk_insert(schema.tables['externalnotificationevent'], + op.bulk_insert(tables.externalnotificationevent, [ {'id':1, 'name':'repo_push'}, {'id':2, 'name':'build_queued'}, @@ -77,7 +73,7 @@ def upgrade(): op.add_column(u'notification', sa.Column('dismissed', sa.Boolean(), nullable=False)) # Manually add the new notificationkind types - op.bulk_insert(schema.tables['notificationkind'], + op.bulk_insert(tables.notificationkind, [ {'id':5, 'name':'repo_push'}, {'id':6, 'name':'build_queued'}, @@ -87,7 +83,7 @@ def upgrade(): ]) # Manually add the new logentrykind types - op.bulk_insert(schema.tables['logentrykind'], + op.bulk_insert(tables.logentrykind, [ {'id':39, 'name':'add_repo_notification'}, {'id':40, 'name':'delete_repo_notification'}, @@ -97,61 +93,49 @@ def upgrade(): ### end Alembic commands ### -def downgrade(): - schema = gen_sqlalchemy_metadata(all_models) - +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.drop_column(u'notification', 'dismissed') - op.drop_index('repositorynotification_uuid', table_name='repositorynotification') - op.drop_index('repositorynotification_repository_id', table_name='repositorynotification') - op.drop_index('repositorynotification_method_id', table_name='repositorynotification') - op.drop_index('repositorynotification_event_id', table_name='repositorynotification') op.drop_table('repositorynotification') - op.drop_index('repositoryauthorizedemail_repository_id', table_name='repositoryauthorizedemail') - op.drop_index('repositoryauthorizedemail_email_repository_id', table_name='repositoryauthorizedemail') - op.drop_index('repositoryauthorizedemail_code', table_name='repositoryauthorizedemail') op.drop_table('repositoryauthorizedemail') - op.drop_index('externalnotificationevent_name', table_name='externalnotificationevent') op.drop_table('externalnotificationevent') - op.drop_index('externalnotificationmethod_name', table_name='externalnotificationmethod') op.drop_table('externalnotificationmethod') # Manually remove the notificationkind and logentrykind types - notificationkind = schema.tables['notificationkind'] op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('repo_push'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('repo_push'))) ) op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('build_queued'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('build_queued'))) ) op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('build_start'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('build_start'))) ) op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('build_success'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('build_success'))) ) op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('build_failure'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('build_failure'))) ) op.execute( - (logentrykind.delete() - .where(logentrykind.c.name == op.inline_literal('add_repo_notification'))) + (tables.logentrykind.delete() + .where(tables.logentrykind.c.name == op.inline_literal('add_repo_notification'))) ) op.execute( - (logentrykind.delete() - .where(logentrykind.c.name == op.inline_literal('delete_repo_notification'))) + (tables.logentrykind.delete() + .where(tables.logentrykind.c.name == op.inline_literal('delete_repo_notification'))) ) ### end Alembic commands ### diff --git a/data/migrations/versions/43e943c0639f_add_log_kind_for_regenerating_robot_.py b/data/migrations/versions/43e943c0639f_add_log_kind_for_regenerating_robot_.py index 6ee041e4c..f676bf972 100644 --- a/data/migrations/versions/43e943c0639f_add_log_kind_for_regenerating_robot_.py +++ b/data/migrations/versions/43e943c0639f_add_log_kind_for_regenerating_robot_.py @@ -13,25 +13,17 @@ down_revision = '82297d834ad' from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models - -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) - - op.bulk_insert(schema.tables['logentrykind'], +def upgrade(tables): + op.bulk_insert(tables.logentrykind, [ {'id': 41, 'name':'regenerate_robot_token'}, ]) -def downgrade(): - schema = gen_sqlalchemy_metadata(all_models) - - logentrykind = schema.tables['logentrykind'] +def downgrade(tables): op.execute( - (logentrykind.delete() - .where(logentrykind.c.name == op.inline_literal('regenerate_robot_token'))) + (tables.logentrykind.delete() + .where(tables.logentrykind.c.name == op.inline_literal('regenerate_robot_token'))) ) diff --git a/data/migrations/versions/47670cbeced_migrate_existing_webhooks_to_.py b/data/migrations/versions/47670cbeced_migrate_existing_webhooks_to_.py index 726145167..eaa687c73 100644 --- a/data/migrations/versions/47670cbeced_migrate_existing_webhooks_to_.py +++ b/data/migrations/versions/47670cbeced_migrate_existing_webhooks_to_.py @@ -18,13 +18,13 @@ def get_id(query): conn = op.get_bind() return list(conn.execute(query, ()).fetchall())[0][0] -def upgrade(): +def upgrade(tables): conn = op.get_bind() event_id = get_id('Select id From externalnotificationevent Where name=\'repo_push\' Limit 1') method_id = get_id('Select id From externalnotificationmethod Where name=\'webhook\' Limit 1') conn.execute('Insert Into repositorynotification (uuid, repository_id, event_id, method_id, config_json) Select public_id, repository_id, %s, %s, parameters FROM webhook' % (event_id, method_id)) -def downgrade(): +def downgrade(tables): conn = op.get_bind() event_id = get_id('Select id From externalnotificationevent Where name=\'repo_push\' Limit 1') method_id = get_id('Select id From externalnotificationmethod Where name=\'webhook\' Limit 1') diff --git a/data/migrations/versions/4b7ef0c7bdb2_add_the_maintenance_notification_type.py b/data/migrations/versions/4b7ef0c7bdb2_add_the_maintenance_notification_type.py index 9e5fff425..9f48ca6c6 100644 --- a/data/migrations/versions/4b7ef0c7bdb2_add_the_maintenance_notification_type.py +++ b/data/migrations/versions/4b7ef0c7bdb2_add_the_maintenance_notification_type.py @@ -11,23 +11,18 @@ revision = '4b7ef0c7bdb2' down_revision = 'bcdde200a1b' from alembic import op -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models import sqlalchemy as sa - -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) - op.bulk_insert(schema.tables['notificationkind'], +def upgrade(tables): + op.bulk_insert(tables.notificationkind, [ {'id':4, 'name':'maintenance'}, ]) -def downgrade(): - notificationkind = schema.tables['notificationkind'] +def downgrade(tables): op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('maintenance'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('maintenance'))) ) diff --git a/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py b/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py index a1c8c95dd..1ce802eca 100644 --- a/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py +++ b/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py @@ -14,14 +14,14 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(): +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.add_column('user', sa.Column('invalid_login_attempts', sa.Integer(), nullable=False, server_default="0")) op.add_column('user', sa.Column('last_invalid_login', sa.DateTime(), nullable=False, server_default=sa.func.now())) ### end Alembic commands ### -def downgrade(): +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.drop_column('user', 'last_invalid_login') op.drop_column('user', 'invalid_login_attempts') diff --git a/data/migrations/versions/5a07499ce53f_set_up_initial_database.py b/data/migrations/versions/5a07499ce53f_set_up_initial_database.py index ffc9d28e6..f67224645 100644 --- a/data/migrations/versions/5a07499ce53f_set_up_initial_database.py +++ b/data/migrations/versions/5a07499ce53f_set_up_initial_database.py @@ -11,14 +11,9 @@ revision = '5a07499ce53f' down_revision = None from alembic import op -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models import sqlalchemy as sa - -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) - +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.create_table('loginservice', sa.Column('id', sa.Integer(), nullable=False), @@ -27,7 +22,7 @@ def upgrade(): ) op.create_index('loginservice_name', 'loginservice', ['name'], unique=True) - op.bulk_insert(schema.tables['loginservice'], + op.bulk_insert(tables.loginservice, [ {'id':1, 'name':'github'}, {'id':2, 'name':'quayrobot'}, @@ -66,7 +61,7 @@ def upgrade(): ) op.create_index('role_name', 'role', ['name'], unique=False) - op.bulk_insert(schema.tables['role'], + op.bulk_insert(tables.role, [ {'id':1, 'name':'admin'}, {'id':2, 'name':'write'}, @@ -80,7 +75,7 @@ def upgrade(): ) op.create_index('logentrykind_name', 'logentrykind', ['name'], unique=False) - op.bulk_insert(schema.tables['logentrykind'], + op.bulk_insert(tables.logentrykind, [ {'id':1, 'name':'account_change_plan'}, {'id':2, 'name':'account_change_cc'}, @@ -136,7 +131,7 @@ def upgrade(): ) op.create_index('notificationkind_name', 'notificationkind', ['name'], unique=False) - op.bulk_insert(schema.tables['notificationkind'], + op.bulk_insert(tables.notificationkind, [ {'id':1, 'name':'password_required'}, {'id':2, 'name':'over_private_usage'}, @@ -150,7 +145,7 @@ def upgrade(): ) op.create_index('teamrole_name', 'teamrole', ['name'], unique=False) - op.bulk_insert(schema.tables['teamrole'], + op.bulk_insert(tables.teamrole, [ {'id':1, 'name':'admin'}, {'id':2, 'name':'creator'}, @@ -164,7 +159,7 @@ def upgrade(): ) op.create_index('visibility_name', 'visibility', ['name'], unique=False) - op.bulk_insert(schema.tables['visibility'], + op.bulk_insert(tables.visibility, [ {'id':1, 'name':'public'}, {'id':2, 'name':'private'}, @@ -194,7 +189,7 @@ def upgrade(): ) op.create_index('buildtriggerservice_name', 'buildtriggerservice', ['name'], unique=False) - op.bulk_insert(schema.tables['buildtriggerservice'], + op.bulk_insert(tables.buildtriggerservice, [ {'id':1, 'name':'github'}, ]) @@ -490,119 +485,34 @@ def upgrade(): ### end Alembic commands ### -def downgrade(): +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### - op.drop_index('repositorybuild_uuid', table_name='repositorybuild') - op.drop_index('repositorybuild_trigger_id', table_name='repositorybuild') - op.drop_index('repositorybuild_resource_key', table_name='repositorybuild') - op.drop_index('repositorybuild_repository_id', table_name='repositorybuild') - op.drop_index('repositorybuild_pull_robot_id', table_name='repositorybuild') - op.drop_index('repositorybuild_access_token_id', table_name='repositorybuild') op.drop_table('repositorybuild') - op.drop_index('repositorybuildtrigger_write_token_id', table_name='repositorybuildtrigger') - op.drop_index('repositorybuildtrigger_service_id', table_name='repositorybuildtrigger') - op.drop_index('repositorybuildtrigger_repository_id', table_name='repositorybuildtrigger') - op.drop_index('repositorybuildtrigger_pull_robot_id', table_name='repositorybuildtrigger') - op.drop_index('repositorybuildtrigger_connected_user_id', table_name='repositorybuildtrigger') op.drop_table('repositorybuildtrigger') - op.drop_index('logentry_repository_id', table_name='logentry') - op.drop_index('logentry_performer_id', table_name='logentry') - op.drop_index('logentry_kind_id', table_name='logentry') - op.drop_index('logentry_datetime', table_name='logentry') - op.drop_index('logentry_account_id', table_name='logentry') - op.drop_index('logentry_access_token_id', table_name='logentry') op.drop_table('logentry') - op.drop_index('repositorytag_repository_id_name', table_name='repositorytag') - op.drop_index('repositorytag_repository_id', table_name='repositorytag') - op.drop_index('repositorytag_image_id', table_name='repositorytag') op.drop_table('repositorytag') - op.drop_index('permissionprototype_role_id', table_name='permissionprototype') - op.drop_index('permissionprototype_org_id_activating_user_id', table_name='permissionprototype') - op.drop_index('permissionprototype_org_id', table_name='permissionprototype') - op.drop_index('permissionprototype_delegate_user_id', table_name='permissionprototype') - op.drop_index('permissionprototype_delegate_team_id', table_name='permissionprototype') - op.drop_index('permissionprototype_activating_user_id', table_name='permissionprototype') op.drop_table('permissionprototype') - op.drop_index('image_storage_id', table_name='image') - op.drop_index('image_repository_id_docker_image_id', table_name='image') - op.drop_index('image_repository_id', table_name='image') - op.drop_index('image_ancestors', table_name='image') op.drop_table('image') - op.drop_index('oauthauthorizationcode_code', table_name='oauthauthorizationcode') - op.drop_index('oauthauthorizationcode_application_id', table_name='oauthauthorizationcode') op.drop_table('oauthauthorizationcode') - op.drop_index('webhook_repository_id', table_name='webhook') - op.drop_index('webhook_public_id', table_name='webhook') op.drop_table('webhook') - op.drop_index('teammember_user_id_team_id', table_name='teammember') - op.drop_index('teammember_user_id', table_name='teammember') - op.drop_index('teammember_team_id', table_name='teammember') op.drop_table('teammember') - op.drop_index('oauthaccesstoken_uuid', table_name='oauthaccesstoken') - op.drop_index('oauthaccesstoken_refresh_token', table_name='oauthaccesstoken') - op.drop_index('oauthaccesstoken_authorized_user_id', table_name='oauthaccesstoken') - op.drop_index('oauthaccesstoken_application_id', table_name='oauthaccesstoken') - op.drop_index('oauthaccesstoken_access_token', table_name='oauthaccesstoken') op.drop_table('oauthaccesstoken') - op.drop_index('repositorypermission_user_id_repository_id', table_name='repositorypermission') - op.drop_index('repositorypermission_user_id', table_name='repositorypermission') - op.drop_index('repositorypermission_team_id_repository_id', table_name='repositorypermission') - op.drop_index('repositorypermission_team_id', table_name='repositorypermission') - op.drop_index('repositorypermission_role_id', table_name='repositorypermission') - op.drop_index('repositorypermission_repository_id', table_name='repositorypermission') op.drop_table('repositorypermission') - op.drop_index('accesstoken_role_id', table_name='accesstoken') - op.drop_index('accesstoken_repository_id', table_name='accesstoken') - op.drop_index('accesstoken_code', table_name='accesstoken') op.drop_table('accesstoken') - op.drop_index('repository_visibility_id', table_name='repository') - op.drop_index('repository_namespace_name', table_name='repository') op.drop_table('repository') - op.drop_index('team_role_id', table_name='team') - op.drop_index('team_organization_id', table_name='team') - op.drop_index('team_name_organization_id', table_name='team') - op.drop_index('team_name', table_name='team') op.drop_table('team') - op.drop_index('emailconfirmation_user_id', table_name='emailconfirmation') - op.drop_index('emailconfirmation_code', table_name='emailconfirmation') op.drop_table('emailconfirmation') - op.drop_index('notification_uuid', table_name='notification') - op.drop_index('notification_target_id', table_name='notification') - op.drop_index('notification_kind_id', table_name='notification') - op.drop_index('notification_created', table_name='notification') op.drop_table('notification') - op.drop_index('oauthapplication_organization_id', table_name='oauthapplication') - op.drop_index('oauthapplication_client_id', table_name='oauthapplication') op.drop_table('oauthapplication') - op.drop_index('federatedlogin_user_id', table_name='federatedlogin') - op.drop_index('federatedlogin_service_id_user_id', table_name='federatedlogin') - op.drop_index('federatedlogin_service_id_service_ident', table_name='federatedlogin') - op.drop_index('federatedlogin_service_id', table_name='federatedlogin') op.drop_table('federatedlogin') - op.drop_index('buildtriggerservice_name', table_name='buildtriggerservice') op.drop_table('buildtriggerservice') - op.drop_index('user_username', table_name='user') - op.drop_index('user_stripe_id', table_name='user') - op.drop_index('user_robot', table_name='user') - op.drop_index('user_organization', table_name='user') - op.drop_index('user_email', table_name='user') op.drop_table('user') - op.drop_index('visibility_name', table_name='visibility') op.drop_table('visibility') - op.drop_index('teamrole_name', table_name='teamrole') op.drop_table('teamrole') - op.drop_index('notificationkind_name', table_name='notificationkind') op.drop_table('notificationkind') - op.drop_index('logentrykind_name', table_name='logentrykind') op.drop_table('logentrykind') - op.drop_index('role_name', table_name='role') op.drop_table('role') - op.drop_index('queueitem_queue_name', table_name='queueitem') - op.drop_index('queueitem_processing_expires', table_name='queueitem') - op.drop_index('queueitem_available_after', table_name='queueitem') - op.drop_index('queueitem_available', table_name='queueitem') op.drop_table('queueitem') op.drop_table('imagestorage') - op.drop_index('loginservice_name', table_name='loginservice') op.drop_table('loginservice') ### end Alembic commands ### diff --git a/data/migrations/versions/82297d834ad_add_us_west_location.py b/data/migrations/versions/82297d834ad_add_us_west_location.py index 59eb1f800..b939a939e 100644 --- a/data/migrations/versions/82297d834ad_add_us_west_location.py +++ b/data/migrations/versions/82297d834ad_add_us_west_location.py @@ -13,24 +13,17 @@ down_revision = '47670cbeced' from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models - -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) - - op.bulk_insert(schema.tables['imagestoragelocation'], +def upgrade(tables): + op.bulk_insert(tables.imagestoragelocation, [ {'id':8, 'name':'s3_us_west_1'}, ]) -def downgrade(): - schema = gen_sqlalchemy_metadata(all_models) - +def downgrade(tables): op.execute( - (imagestoragelocation.delete() - .where(imagestoragelocation.c.name == op.inline_literal('s3_us_west_1'))) + (tables.imagestoragelocation.delete() + .where(tables.imagestoragelocation.c.name == op.inline_literal('s3_us_west_1'))) ) diff --git a/data/migrations/versions/bcdde200a1b_add_placements_and_locations_to_the_db.py b/data/migrations/versions/bcdde200a1b_add_placements_and_locations_to_the_db.py index eda4b2840..9fc433126 100644 --- a/data/migrations/versions/bcdde200a1b_add_placements_and_locations_to_the_db.py +++ b/data/migrations/versions/bcdde200a1b_add_placements_and_locations_to_the_db.py @@ -11,14 +11,10 @@ revision = 'bcdde200a1b' down_revision = '201d55b38649' from alembic import op -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models import sqlalchemy as sa -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) - +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.create_table('imagestoragelocation', sa.Column('id', sa.Integer(), nullable=False), @@ -27,7 +23,7 @@ def upgrade(): ) op.create_index('imagestoragelocation_name', 'imagestoragelocation', ['name'], unique=True) - op.bulk_insert(schema.tables['imagestoragelocation'], + op.bulk_insert(tables.imagestoragelocation, [ {'id':1, 'name':'s3_us_east_1'}, {'id':2, 'name':'s3_eu_west_1'}, @@ -52,12 +48,8 @@ def upgrade(): ### end Alembic commands ### -def downgrade(): +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### - op.drop_index('imagestorageplacement_storage_id_location_id', table_name='imagestorageplacement') - op.drop_index('imagestorageplacement_storage_id', table_name='imagestorageplacement') - op.drop_index('imagestorageplacement_location_id', table_name='imagestorageplacement') op.drop_table('imagestorageplacement') - op.drop_index('imagestoragelocation_name', table_name='imagestoragelocation') op.drop_table('imagestoragelocation') ### end Alembic commands ### diff --git a/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py b/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py index 79ea17be0..9ceab4218 100644 --- a/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py +++ b/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py @@ -14,13 +14,13 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(): +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.drop_table('webhook') ### end Alembic commands ### -def downgrade(): +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.create_table('webhook', sa.Column('id', mysql.INTEGER(display_width=11), nullable=False), diff --git a/util/collections.py b/util/collections.py new file mode 100644 index 000000000..b34dc00ed --- /dev/null +++ b/util/collections.py @@ -0,0 +1,12 @@ +class AttrDict(dict): + def __init__(self, *args, **kwargs): + super(AttrDict, self).__init__(*args, **kwargs) + self.__dict__ = self + + @classmethod + def deep_copy(cls, attr_dict): + copy = AttrDict(attr_dict) + for key, value in copy.items(): + if isinstance(value, AttrDict): + copy[key] = cls.deep_copy(value) + return copy \ No newline at end of file