From 2597bcef3fc330457a6818362ac4b99e7dbec8cc Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 11 Aug 2014 15:47:44 -0400 Subject: [PATCH 01/49] Add support for login with Google. Note that this CL is not complete --- config.py | 12 +- endpoints/callbacks.py | 128 ++++++++++++++---- initdb.py | 2 + static/directives/external-login-button.html | 17 +++ static/directives/signin-form.html | 6 +- static/directives/signup-form.html | 6 +- static/js/app.js | 100 +++++++++----- static/js/controllers.js | 5 +- static/partials/user-admin.html | 30 +++- .../{githuberror.html => ologinerror.html} | 8 +- 10 files changed, 231 insertions(+), 83 deletions(-) create mode 100644 static/directives/external-login-button.html rename templates/{githuberror.html => ologinerror.html} (66%) diff --git a/config.py b/config.py index a903fa29a..708449714 100644 --- a/config.py +++ b/config.py @@ -19,7 +19,7 @@ def build_requests_session(): CLIENT_WHITELIST = ['SERVER_HOSTNAME', 'PREFERRED_URL_SCHEME', 'GITHUB_CLIENT_ID', 'GITHUB_LOGIN_CLIENT_ID', 'MIXPANEL_KEY', 'STRIPE_PUBLISHABLE_KEY', 'ENTERPRISE_LOGO_URL', 'SENTRY_PUBLIC_DSN', 'AUTHENTICATION_TYPE', - 'REGISTRY_TITLE', 'REGISTRY_TITLE_SHORT'] + 'REGISTRY_TITLE', 'REGISTRY_TITLE_SHORT', 'GOOGLE_LOGIN_CLIENT_ID'] def getFrontendVisibleConfig(config_dict): @@ -115,6 +115,13 @@ class DefaultConfig(object): GITHUB_LOGIN_CLIENT_ID = '' GITHUB_LOGIN_CLIENT_SECRET = '' + # Google Config. + GOOGLE_TOKEN_URL = 'https://accounts.google.com/o/oauth2/token' + GOOGLE_USER_URL = 'https://www.googleapis.com/oauth2/v1/userinfo' + + GOOGLE_LOGIN_CLIENT_ID = '' + GOOGLE_LOGIN_CLIENT_SECRET = '' + # Requests based HTTP client with a large request pool HTTPCLIENT = build_requests_session() @@ -144,6 +151,9 @@ class DefaultConfig(object): # Feature Flag: Whether GitHub login is supported. FEATURE_GITHUB_LOGIN = False + # Feature Flag: Whether Google login is supported. + FEATURE_GOOGLE_LOGIN = False + # Feature flag, whether to enable olark chat FEATURE_OLARK_CHAT = False diff --git a/endpoints/callbacks.py b/endpoints/callbacks.py index 015f3c3a7..ba53cbc5c 100644 --- a/endpoints/callbacks.py +++ b/endpoints/callbacks.py @@ -7,6 +7,7 @@ from endpoints.common import render_page_template, common_login, route_show_if from app import app, analytics from data import model from util.names import parse_repository_name +from util.validation import generate_valid_usernames from util.http import abort from auth.permissions import AdministerRepositoryPermission from auth.auth import require_session_login @@ -21,19 +22,31 @@ client = app.config['HTTPCLIENT'] callback = Blueprint('callback', __name__) -def exchange_github_code_for_token(code, for_login=True): +def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_encode=False): code = request.args.get('code') + id_config = service_name + '_LOGIN_CLIENT_ID' if for_login else service_name + '_CLIENT_ID' + secret_config = service_name + '_LOGIN_CLIENT_SECRET' if for_login else service_name + '_CLIENT_SECRET' + payload = { - 'client_id': app.config['GITHUB_LOGIN_CLIENT_ID' if for_login else 'GITHUB_CLIENT_ID'], - 'client_secret': app.config['GITHUB_LOGIN_CLIENT_SECRET' if for_login else 'GITHUB_CLIENT_SECRET'], + 'client_id': app.config[id_config], + 'client_secret': app.config[secret_config], 'code': code, + 'grant_type': 'authorization_code', + 'redirect_uri': '%s://%s/oauth2/%s/callback' % (app.config['PREFERRED_URL_SCHEME'], + app.config['SERVER_HOSTNAME'], + service_name.lower()) } + headers = { 'Accept': 'application/json' } - get_access_token = client.post(app.config['GITHUB_TOKEN_URL'], - params=payload, headers=headers) + if form_encode: + get_access_token = client.post(app.config[service_name + '_TOKEN_URL'], + data=payload, headers=headers) + else: + get_access_token = client.post(app.config[service_name + '_TOKEN_URL'], + params=payload, headers=headers) json_data = get_access_token.json() if not json_data: @@ -52,17 +65,76 @@ def get_github_user(token): return get_user.json() +def get_google_user(token): + token_param = { + 'access_token': token, + 'alt': 'json', + } + + get_user = client.get(app.config['GOOGLE_USER_URL'], params=token_param) + return get_user.json() + +def conduct_oauth_login(service_name, user_id, username, email): + to_login = model.verify_federated_login(service_name.lower(), user_id) + if not to_login: + # try to create the user + try: + valid = next(generate_valid_usernames(username)) + to_login = model.create_federated_user(valid, email, service_name.lower(), + user_id, set_password_notification=True) + + # Success, tell analytics + analytics.track(to_login.username, 'register', {'service': service_name.lower()}) + + state = request.args.get('state', None) + if state: + logger.debug('Aliasing with state: %s' % state) + analytics.alias(to_login.username, state) + + except model.DataModelException, ex: + return render_page_template('ologinerror.html', service_name=service_name, + error_message=ex.message) + + if common_login(to_login): + return redirect(url_for('web.index')) + + return render_page_template('ologinerror.html', service_name=service_name, + error_message='Unknown error') + + +@callback.route('/google/callback', methods=['GET']) +@route_show_if(features.GOOGLE_LOGIN) +def google_oauth_callback(): + error = request.args.get('error', None) + if error: + return render_page_template('ologinerror.html', service_name='Google', error_message=error) + + token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE', form_encode=True) + user_data = get_google_user(token) + if not user_data or not user_data.get('id', None) or not user_data.get('email', None): + return render_page_template('ologinerror.html', service_name = 'Google', + error_message='Could not load user data') + + username = user_data['email'] + at = username.find('@') + if at > 0: + username = username[0:at] + + return conduct_oauth_login('Google', user_data['id'], username, user_data['email']) + + @callback.route('/github/callback', methods=['GET']) @route_show_if(features.GITHUB_LOGIN) def github_oauth_callback(): error = request.args.get('error', None) if error: - return render_page_template('githuberror.html', error_message=error) + return render_page_template('ologinerror.html', service_name = 'GitHub', error_message=error) - token = exchange_github_code_for_token(request.args.get('code')) + token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB') user_data = get_github_user(token) if not user_data: - return render_page_template('githuberror.html', error_message='Could not load user data') + return render_page_template('ologinerror.html', service_name = 'GitHub', + error_message='Could not load user data') username = user_data['login'] github_id = user_data['id'] @@ -84,38 +156,34 @@ def github_oauth_callback(): if user_email['primary']: break - to_login = model.verify_federated_login('github', github_id) - if not to_login: - # try to create the user - try: - to_login = model.create_federated_user(username, found_email, 'github', - github_id, set_password_notification=True) + return conduct_oauth_login('github', github_id, username, found_email) - # Success, tell analytics - analytics.track(to_login.username, 'register', {'service': 'github'}) - state = request.args.get('state', None) - if state: - logger.debug('Aliasing with state: %s' % state) - analytics.alias(to_login.username, state) +@callback.route('/google/callback/attach', methods=['GET']) +@route_show_if(features.GOOGLE_LOGIN) +@require_session_login +def google_oauth_attach(): + token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE') + user_data = get_google_user(token) + if not user_data or not user_data.get('id', None): + return render_page_template('ologinerror.html', service_name = 'Google', + error_message='Could not load user data') - except model.DataModelException, ex: - return render_page_template('githuberror.html', error_message=ex.message) - - if common_login(to_login): - return redirect(url_for('web.index')) - - return render_page_template('githuberror.html') + google_id = user_data['id'] + user_obj = current_user.db_user() + model.attach_federated_login(user_obj, 'google', google_id) + return redirect(url_for('web.user')) @callback.route('/github/callback/attach', methods=['GET']) @route_show_if(features.GITHUB_LOGIN) @require_session_login def github_oauth_attach(): - token = exchange_github_code_for_token(request.args.get('code')) + token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB') user_data = get_github_user(token) if not user_data: - return render_page_template('githuberror.html', error_message='Could not load user data') + return render_page_template('ologinerror.html', service_name = 'GitHub', + error_message='Could not load user data') github_id = user_data['id'] user_obj = current_user.db_user() @@ -130,7 +198,7 @@ def github_oauth_attach(): def attach_github_build_trigger(namespace, repository): permission = AdministerRepositoryPermission(namespace, repository) if permission.can(): - token = exchange_github_code_for_token(request.args.get('code'), for_login=False) + token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB', for_login=False) repo = model.get_repository(namespace, repository) if not repo: msg = 'Invalid repository: %s/%s' % (namespace, repository) diff --git a/initdb.py b/initdb.py index 7e48ae3af..5d10a2039 100644 --- a/initdb.py +++ b/initdb.py @@ -179,6 +179,8 @@ def initialize_database(): TeamRole.create(name='member') Visibility.create(name='public') Visibility.create(name='private') + + LoginService.create(name='google') LoginService.create(name='github') LoginService.create(name='quayrobot') LoginService.create(name='ldap') diff --git a/static/directives/external-login-button.html b/static/directives/external-login-button.html new file mode 100644 index 000000000..cc1d39bbd --- /dev/null +++ b/static/directives/external-login-button.html @@ -0,0 +1,17 @@ + + + + + Sign In with GitHub + Attach to GitHub Account + + + + + + + Sign In with Google + Attach to Google Account + + + diff --git a/static/directives/signin-form.html b/static/directives/signin-form.html index f56b8f8db..ec57619a8 100644 --- a/static/directives/signin-form.html +++ b/static/directives/signin-form.html @@ -11,10 +11,8 @@ OR - - Sign In with GitHub - +
+
Invalid username or password.
diff --git a/static/directives/signup-form.html b/static/directives/signup-form.html index fb0ccc6fa..249bff31c 100644 --- a/static/directives/signup-form.html +++ b/static/directives/signup-form.html @@ -18,10 +18,8 @@ OR - - Sign In with GitHub - +
+

No credit card required.

diff --git a/static/js/app.js b/static/js/app.js index 8daec75d4..978012bb6 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1278,10 +1278,41 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading var keyService = {} keyService['stripePublishableKey'] = Config['STRIPE_PUBLISHABLE_KEY']; + keyService['githubClientId'] = Config['GITHUB_CLIENT_ID']; keyService['githubLoginClientId'] = Config['GITHUB_LOGIN_CLIENT_ID']; keyService['githubRedirectUri'] = Config.getUrl('/oauth2/github/callback'); + keyService['googleLoginClientId'] = Config['GOOGLE_LOGIN_CLIENT_ID']; + keyService['googleRedirectUri'] = Config.getUrl('/oauth2/google/callback'); + + keyService['googleLoginUrl'] = 'https://accounts.google.com/o/oauth2/auth?response_type=code&'; + keyService['githubLoginUrl'] = 'https://github.com/login/oauth/authorize?'; + + keyService['googleLoginScope'] = 'openid email'; + keyService['githubLoginScope'] = 'user:email'; + + keyService.getExternalLoginUrl = function(service, action) { + var state_clause = ''; + if (Config.MIXPANEL_KEY && window.mixpanel) { + if (mixpanel.get_distinct_id !== undefined) { + state_clause = "&state=" + encodeURIComponent(mixpanel.get_distinct_id()); + } + } + + var client_id = keyService[service + 'LoginClientId']; + var scope = keyService[service + 'LoginScope']; + var redirect_uri = keyService[service + 'RedirectUri']; + if (action == 'attach') { + redirect_uri += '/attach'; + } + + var url = keyService[service + 'LoginUrl'] + 'client_id=' + client_id + '&scope=' + scope + + '&redirect_uri=' + redirect_uri + state_clause; + + return url; + }; + return keyService; }]); @@ -2150,6 +2181,41 @@ quayApp.directive('userSetup', function () { }); +quayApp.directive('externalLoginButton', function () { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/external-login-button.html', + replace: false, + transclude: true, + restrict: 'C', + scope: { + 'signInStarted': '&signInStarted', + 'redirectUrl': '=redirectUrl', + 'provider': '@provider', + 'action': '@action' + }, + controller: function($scope, $location, $timeout, ApiService, KeyService, UserService, CookieService, Features, Config) { + $scope.startSignin = function(service) { + $scope.signInStarted({'service': service}); + + var url = KeyService.getExternalLoginUrl(service, $scope.action || 'login'); + + // Save the redirect URL in a cookie so that we can redirect back after the service returns to us. + var redirectURL = $scope.redirectUrl || window.location.toString(); + CookieService.putPermanent('quay.redirectAfterLoad', redirectURL); + + // Needed to ensure that UI work done by the started callback is finished before the location + // changes. + $timeout(function() { + document.location = url; + }, 250); + }; + } + }; + return directiveDefinitionObject; +}); + + quayApp.directive('signinForm', function () { var directiveDefinitionObject = { priority: 0, @@ -2163,29 +2229,6 @@ quayApp.directive('signinForm', function () { 'signedIn': '&signedIn' }, controller: function($scope, $location, $timeout, ApiService, KeyService, UserService, CookieService, Features, Config) { - $scope.showGithub = function() { - if (!Features.GITHUB_LOGIN) { return; } - - $scope.markStarted(); - - var mixpanelDistinctIdClause = ''; - if (Config.MIXPANEL_KEY && mixpanel.get_distinct_id !== undefined) { - $scope.mixpanelDistinctIdClause = "&state=" + encodeURIComponent(mixpanel.get_distinct_id()); - } - - // Save the redirect URL in a cookie so that we can redirect back after GitHub returns to us. - var redirectURL = $scope.redirectUrl || window.location.toString(); - CookieService.putPermanent('quay.redirectAfterLoad', redirectURL); - - // Needed to ensure that UI work done by the started callback is finished before the location - // changes. - $timeout(function() { - var url = 'https://github.com/login/oauth/authorize?client_id=' + encodeURIComponent(KeyService.githubLoginClientId) + - '&scope=user:email' + mixpanelDistinctIdClause; - document.location = url; - }, 250); - }; - $scope.markStarted = function() { if ($scope.signInStarted != null) { $scope.signInStarted(); @@ -2235,18 +2278,9 @@ quayApp.directive('signupForm', function () { scope: { }, - controller: function($scope, $location, $timeout, ApiService, KeyService, UserService, Config, UIService) { + controller: function($scope, $location, $timeout, ApiService, KeyService, UserService, Config, UIService) { $('.form-signup').popover(); - if (Config.MIXPANEL_KEY) { - angulartics.waitForVendorApi(mixpanel, 500, function(loadedMixpanel) { - var mixpanelId = loadedMixpanel.get_distinct_id(); - $scope.github_state_clause = '&state=' + mixpanelId; - }); - } - - $scope.githubClientId = KeyService.githubLoginClientId; - $scope.awaitingConfirmation = false; $scope.registering = false; diff --git a/static/js/controllers.js b/static/js/controllers.js index d77ffb298..7690f79f6 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -1681,6 +1681,10 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use $scope.githubLogin = resp.login; }); } + + if ($scope.cuser.logins[i].service == 'google') { + $scope.hasGoogleLogin = true; + } } } }); @@ -1697,7 +1701,6 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use $scope.convertStep = 0; $scope.org = {}; $scope.githubRedirectUri = KeyService.githubRedirectUri; - $scope.githubClientId = KeyService.githubLoginClientId; $scope.authorizedApps = null; $scope.logsShown = 0; diff --git a/static/partials/user-admin.html b/static/partials/user-admin.html index 783c5f87a..fc0acb89e 100644 --- a/static/partials/user-admin.html +++ b/static/partials/user-admin.html @@ -33,7 +33,7 @@
  • Account E-mail
  • Robot Accounts
  • Change Password
    • -
  • GitHub Login
    • +
  • External Logins
  • Authorized Applications
  • Usage Logs @@ -162,12 +162,14 @@ - -
    + +
    -
    + + +
    GitHub Login:
    @@ -175,12 +177,28 @@ {{githubLogin}}
    -
    - Connect with GitHub +
    +
    + + +
    +
    +
    Google Login:
    +
    +
    + Account tied to your Google account. +
    +
    + +
    +
    +
    +
    +
    diff --git a/templates/githuberror.html b/templates/ologinerror.html similarity index 66% rename from templates/githuberror.html rename to templates/ologinerror.html index acb803f57..cd921eec2 100644 --- a/templates/githuberror.html +++ b/templates/ologinerror.html @@ -1,14 +1,14 @@ {% extends "base.html" %} {% block title %} - Error Logging in with GitHub · Quay.io + Error Logging in with {{ service_name }} · Quay.io {% endblock %} {% block body_content %}
    -

    There was an error logging in with GitHub.

    +

    There was an error logging in with {{ service_name }}.

    {% if error_message %}
    {{ error_message }}
    @@ -16,11 +16,11 @@
    Please register using the registration form to continue. - You will be able to connect your github account to your Quay.io account + You will be able to connect your account to your Quay.io account in the user settings.
    -{% endblock %} \ No newline at end of file +{% endblock %} From 389c88a7c4b2b08b879b772a0deca139e004ef3e Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 11 Aug 2014 18:25:01 -0400 Subject: [PATCH 02/49] Update federated login to store metadata and have the UI pull the information from the metadata --- data/database.py | 1 + data/model/legacy.py | 13 ++++-- endpoints/api/user.py | 8 ++++ endpoints/callbacks.py | 81 ++++++++++++++++++++++++++------- static/js/controllers.js | 9 ++-- static/partials/user-admin.html | 17 +++++-- 6 files changed, 99 insertions(+), 30 deletions(-) diff --git a/data/database.py b/data/database.py index 76a0af9df..6099cf5d9 100644 --- a/data/database.py +++ b/data/database.py @@ -116,6 +116,7 @@ class FederatedLogin(BaseModel): user = ForeignKeyField(User, index=True) service = ForeignKeyField(LoginService, index=True) service_ident = CharField() + metadata_json = TextField(default='{}') class Meta: database = db diff --git a/data/model/legacy.py b/data/model/legacy.py index b5afdfeb8..bfa310046 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -346,7 +346,8 @@ def set_team_org_permission(team, team_role_name, set_by_username): return team -def create_federated_user(username, email, service_name, service_id, set_password_notification): +def create_federated_user(username, email, service_name, service_id, + set_password_notification, metadata={}): if not is_create_user_allowed(): raise TooManyUsersException() @@ -356,7 +357,8 @@ def create_federated_user(username, email, service_name, service_id, set_passwor service = LoginService.get(LoginService.name == service_name) FederatedLogin.create(user=new_user, service=service, - service_ident=service_id) + service_ident=service_id, + metadata_json=json.dumps(metadata)) if set_password_notification: create_notification('password_required', new_user) @@ -364,9 +366,10 @@ def create_federated_user(username, email, service_name, service_id, set_passwor return new_user -def attach_federated_login(user, service_name, service_id): +def attach_federated_login(user, service_name, service_id, metadata={}): service = LoginService.get(LoginService.name == service_name) - FederatedLogin.create(user=user, service=service, service_ident=service_id) + FederatedLogin.create(user=user, service=service, service_ident=service_id, + metadata_json=json.dumps(metadata)) return user @@ -385,7 +388,7 @@ def verify_federated_login(service_name, service_id): def list_federated_logins(user): selected = FederatedLogin.select(FederatedLogin.service_ident, - LoginService.name) + LoginService.name, FederatedLogin.metadata_json) joined = selected.join(LoginService) return joined.where(LoginService.name != 'quayrobot', FederatedLogin.user == user) diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 3d79a806d..23bc3137a 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -39,9 +39,16 @@ def user_view(user): organizations = model.get_user_organizations(user.username) def login_view(login): + print login.metadata_json + try: + metadata = json.loads(login.metadata_json) + except: + metadata = None + return { 'service': login.service.name, 'service_identifier': login.service_ident, + 'metadata': metadata } logins = model.list_federated_logins(user) @@ -88,6 +95,7 @@ class User(ApiResource): """ Operations related to users. """ schemas = { 'NewUser': { + 'id': 'NewUser', 'type': 'object', 'description': 'Fields which must be specified for a new user.', diff --git a/endpoints/callbacks.py b/endpoints/callbacks.py index ba53cbc5c..49fa1e8a6 100644 --- a/endpoints/callbacks.py +++ b/endpoints/callbacks.py @@ -11,6 +11,7 @@ from util.validation import generate_valid_usernames from util.http import abort from auth.permissions import AdministerRepositoryPermission from auth.auth import require_session_login +from peewee import IntegrityError import features @@ -22,7 +23,8 @@ client = app.config['HTTPCLIENT'] callback = Blueprint('callback', __name__) -def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_encode=False): +def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_encode=False, + redirect_suffix=''): code = request.args.get('code') id_config = service_name + '_LOGIN_CLIENT_ID' if for_login else service_name + '_CLIENT_ID' secret_config = service_name + '_LOGIN_CLIENT_SECRET' if for_login else service_name + '_CLIENT_SECRET' @@ -32,9 +34,10 @@ def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_en 'client_secret': app.config[secret_config], 'code': code, 'grant_type': 'authorization_code', - 'redirect_uri': '%s://%s/oauth2/%s/callback' % (app.config['PREFERRED_URL_SCHEME'], - app.config['SERVER_HOSTNAME'], - service_name.lower()) + 'redirect_uri': '%s://%s/oauth2/%s/callback%s' % (app.config['PREFERRED_URL_SCHEME'], + app.config['SERVER_HOSTNAME'], + service_name.lower(), + redirect_suffix) } headers = { @@ -74,14 +77,15 @@ def get_google_user(token): get_user = client.get(app.config['GOOGLE_USER_URL'], params=token_param) return get_user.json() -def conduct_oauth_login(service_name, user_id, username, email): +def conduct_oauth_login(service_name, user_id, username, email, metadata={}): to_login = model.verify_federated_login(service_name.lower(), user_id) if not to_login: # try to create the user try: valid = next(generate_valid_usernames(username)) to_login = model.create_federated_user(valid, email, service_name.lower(), - user_id, set_password_notification=True) + user_id, set_password_notification=True, + metadata=metadata) # Success, tell analytics analytics.track(to_login.username, 'register', {'service': service_name.lower()}) @@ -102,6 +106,15 @@ def conduct_oauth_login(service_name, user_id, username, email): error_message='Unknown error') +def get_google_username(user_data): + username = user_data['email'] + at = username.find('@') + if at > 0: + username = username[0:at] + + return username + + @callback.route('/google/callback', methods=['GET']) @route_show_if(features.GOOGLE_LOGIN) def google_oauth_callback(): @@ -115,12 +128,13 @@ def google_oauth_callback(): return render_page_template('ologinerror.html', service_name = 'Google', error_message='Could not load user data') - username = user_data['email'] - at = username.find('@') - if at > 0: - username = username[0:at] + username = get_google_username(user_data) + metadata = { + 'service_username': username + } - return conduct_oauth_login('Google', user_data['id'], username, user_data['email']) + return conduct_oauth_login('Google', user_data['id'], username, user_data['email'], + metadata=metadata) @callback.route('/github/callback', methods=['GET']) @@ -156,14 +170,20 @@ def github_oauth_callback(): if user_email['primary']: break - return conduct_oauth_login('github', github_id, username, found_email) + metadata = { + 'service_username': username + } + + return conduct_oauth_login('github', github_id, username, found_email, metadata=metadata) @callback.route('/google/callback/attach', methods=['GET']) @route_show_if(features.GOOGLE_LOGIN) @require_session_login def google_oauth_attach(): - token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE') + token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE', + redirect_suffix='/attach', form_encode=True) + user_data = get_google_user(token) if not user_data or not user_data.get('id', None): return render_page_template('ologinerror.html', service_name = 'Google', @@ -171,7 +191,21 @@ def google_oauth_attach(): google_id = user_data['id'] user_obj = current_user.db_user() - model.attach_federated_login(user_obj, 'google', google_id) + + username = get_google_username(user_data) + metadata = { + 'service_username': username + } + + try: + model.attach_federated_login(user_obj, 'google', google_id, metadata=metadata) + except IntegrityError: + err = 'Google account %s is already attached to a %s account' % ( + username, app.config['REGISTRY_TITLE_SHORT']) + + return render_page_template('ologinerror.html', service_name = 'Google', + error_message=err) + return redirect(url_for('web.user')) @@ -187,7 +221,21 @@ def github_oauth_attach(): github_id = user_data['id'] user_obj = current_user.db_user() - model.attach_federated_login(user_obj, 'github', github_id) + + username = user_data['login'] + metadata = { + 'service_username': username + } + + try: + model.attach_federated_login(user_obj, 'github', github_id, metadata=metadata) + except IntegrityError: + err = 'Github account %s is already attached to a %s account' % ( + username, app.config['REGISTRY_TITLE_SHORT']) + + return render_page_template('ologinerror.html', service_name = 'Github', + error_message=err) + return redirect(url_for('web.user')) @@ -198,7 +246,8 @@ def github_oauth_attach(): def attach_github_build_trigger(namespace, repository): permission = AdministerRepositoryPermission(namespace, repository) if permission.can(): - token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB', for_login=False) + token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB', + for_login=False) repo = model.get_repository(namespace, repository) if not repo: msg = 'Invalid repository: %s/%s' % (namespace, repository) diff --git a/static/js/controllers.js b/static/js/controllers.js index 7690f79f6..5278e4efe 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -1673,17 +1673,16 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use UserService.updateUserIn($scope, function(user) { $scope.cuser = jQuery.extend({}, user); - if (Features.GITHUB_LOGIN && $scope.cuser.logins) { + if ($scope.cuser.logins) { for (var i = 0; i < $scope.cuser.logins.length; i++) { if ($scope.cuser.logins[i].service == 'github') { - var githubId = $scope.cuser.logins[i].service_identifier; - $http.get('https://api.github.com/user/' + githubId).success(function(resp) { - $scope.githubLogin = resp.login; - }); + $scope.hasGithubLogin = true; + $scope.githubLogin = $scope.cuser.logins[i].metadata['service_username']; } if ($scope.cuser.logins[i].service == 'google') { $scope.hasGoogleLogin = true; + $scope.googleLogin = $scope.cuser.logins[i].metadata['service_username']; } } } diff --git a/static/partials/user-admin.html b/static/partials/user-admin.html index fc0acb89e..ca76342dd 100644 --- a/static/partials/user-admin.html +++ b/static/partials/user-admin.html @@ -173,11 +173,15 @@
    GitHub Login:
    -
    +
    {{githubLogin}}
    -
    +
    + + Account attached to Github Account +
    +
    @@ -189,8 +193,13 @@
    Google Login:
    -
    - Account tied to your Google account. +
    + + {{ googleLogin }} +
    +
    + + Account attached to Google Account
    From 11176215e10a069045892420a2c41c6492fe11af Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 11 Aug 2014 18:35:26 -0400 Subject: [PATCH 03/49] Commit new DB changes and make sure the metadata is always present in some form --- endpoints/api/user.py | 2 +- test/data/test.db | Bin 614400 -> 614400 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 23bc3137a..d0d089dcd 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -43,7 +43,7 @@ def user_view(user): try: metadata = json.loads(login.metadata_json) except: - metadata = None + metadata = {} return { 'service': login.service.name, diff --git a/test/data/test.db b/test/data/test.db index 4d04283311e6feb845dff3fde9a4d370858119be..b64829db209fca64a5f122a3c96d74f8f7d66845 100644 GIT binary patch delta 6425 zcmds*dwdjCmdCrgs#9HgbucId2x&qhB&5>yeq<6zr;|?7-RVb?&Z7lMRdrPYA!*2y zfEdad{mjhnXI5y}yY4WGpd-=XBWN=0h~m1aJ32Zus581cj2{P=QQ|oJQFdKX_J)Xp zLC4SV-{y}_cb$8__uSt-=hXe&okt6I9xc3eF;cs0i3h2Dq~q*nEa%-fQj;B9_F%5U z=eO*k$G_aQA*YmwbC*94qX; z)1OQZ-j9ZVASWw__FzkozPLL1hoO^L`TJyg^52J#Vfem>Hzap&oz6;Id;QC1kNr!E zc>q$L9vR2hl=bdS_HP@*O1r`Oq;vZotU^IHC68`Di7kBUog0&5Hypzk+*#o=dr?qw z6;Y9#*l`eFgSZcwJywvL4Ik{kaU5Tz{`W$&7y%jciSk3cNBefBV~*n%$48EL9d9^} zADZWU+EIaHsO9QSabJDcoQQ2%>hp2A@%|+1Ig4A7aIn4=y!Q#>tfayLMyhY~ zxC2cQxq*uLLSkKWb)A>-Mrvqxv(M{ou4xa|1gfH~?ez_UJLahs!!em|YWDF`wB7^! zXOR{zB6A5rZEIH>BEhJ;w0a{~9jwu}4b&LJK_92a^hm6?YinFE+$zo0ceS>9MjF5? zXOSYlv77Zbck3+!Ra>>VNDsGjgYrnJU#>QWs{BLEMwgGHShr_$f8Fp_mpI%LYV4;* z0Q)IY!h2+Ggi1751vbZ+E#7*cXWMWi8>(vQ>Z~22wrtv@M+Q6f`r*!iNV&uPyJ8`%JG&NW|*rNvG+v44^YPFwDZ0(n;{On+3tI=L6dMS{74iPG=WTu60iP24U zF_=@WO>QpO=8pQsP(9_1`Poo3)+kaQt~Suv5DZgI6e~8x+8MsN$sLx&04IVUoI{rI zO-)rkPpq`lU*8|@-XyU;$=%#qI>Pq1^%i_!)PK6H zdCt@qERI7C!;zEu0_;nVXO8+Y+=3ux#=La&N=4hg!;gLrOGCeh6?{W zbF|h5ttw-ojZT}Qv#aM8~}SE{MiJF_t25;-rU=CZ$9{gby^-*GHL(=i-F zpqy#w?0=gz1^y;b>rX2YYVudsPcm0FOOn66p)S@G^$GqcTUG0;WhJ?l^E7)Km~b2I zuN4|4p^X7gp0E~zx)avWN~yM`p`8yia?mG)-BlsFIxL1LpWN0U1gqN`o1;=)buAO9 zoq{eOxFWKy$3>NsNlGzvl8ZA6DNCA4a)u6@C&d_)LHFsK6J<)gXG8CR-ajyq=+=7~ zisnd4B5B4+(Pa!(M)Rct&j>6vnT5XLDEKzmTb$^m6HCcTdD!9@$$S_l&pnyo&DB_X z0mpDGEoeHNL5!1}u1TaSN)$=QIYpv%!-&fo$XJ8%qcrVw4|Y0d!f`T88OxTjw3DRZ zUw(-*e;~0wt_Z=&5sSuoqnn)O=2a)?Cz0Tz&!P9TV82j)rd| zs&xe#CBCx8%Pw(pC$`lwO2KWGB|{{6Ru@Sq4NM+Jlwd9~l1Qr*BPjxEby@8eBuc=p zn@UTxtV#kL!6^#KNg72eicFDlgM(Ww@VX?kR@8-@cR(yisuquP0;@`7JT5~AR%1vx zZp2BMk_8^;SWc7~Vd}Ggz(RWpL`9?Gl%SIgEyyIt=y6gKMT3+CN)=T_RpOcmwv^ba zmm3BJCz1?O5aCvG46TwX&kCe2NdhN{hHmiScO^EVU?xnsqdeSR8ZHoyBo&ocNI~ME zt1halqyUDrRTLNk1+Ah&Qi1>j)l{97SXw3pT~p$+pz)HZf+T6HDiCGSkaWo8DVm0p z^BgHFyh>^!C+U*L>+sMAJ|=Ce3OJgkG?_6-M$_PeWC_M0$~?(xA_Em;SxuKfU8&7o zz|xAqiVVy}L4qSCUM3ZW5lAYoC~$SMs2B=3SZZ5R04s=5R1p@h0gIl4@spBFF=Sja z;+mjAnHU93m)hzIC`B?@9abGLD-x6*f>lFeNtUMcIIYtRN2{RSWm~nJRfIUlL5pT^ zB&Wc(po(#dlr&aVIF^CCDuVl5HoichXibqc4eAqzlPfb}sv^t*O5_X}sw{}{Ny>K5 zx$C^X`Y2x)Xb;7_;WkdL^3+7A7B<>K$w5(StFLQqiO2%$W@FJvh;3}*DLG1e+i7o; z+)&pfR<+sZRs&zkvy?>fQ|DYMPdO^DbcV^wlz(#uq&#(s*(&t z#Wh)xc~zV`l1K?%KmX#og`r5i#0bFoV#;s7Y7u!x=0Vy^b1ecdNc7a1mr{ndU*aT{ z@(j<*6xiv+C@rISd#O!N2xss!-r%TM?F27D!;8gLAp4B&;A z9|OG?0jU9Bb9pmF6oE$@@N2%-yc$|4@j_3&II2a!7sRifF$;6d=R+G0;wvw2J{80x zGZuc1BaHWIpoZ|0%bWLwaPhL1sp$}Y|MjC6@A8FM)0d)dWNtoQFsFU9IrW>(#3km@ zFJ287==MwObIka~=D9MbTw~q9MzuA1rTW#)EO+Ji2{ZK_P z7KzX`5gkL3om|ZtT~(5`BG@aeZh&Q8($OTB4`g$Ja}(g1b7#P_i!; zn(VOuhokVT2x~S<5_5)bwy$0G|ML*JT=P-fKUNRlHWc3v}FVzdz;ur6p!=0r<6@C7acMa>f%W&;r!n;K&mRHddm%CXkq=LS z%AeZD@RCpdRW);$f=4nSvgAI9OxyL{3Dd|0kxYpE<$Vx&RsAa;ncgK}d@UUF*a65a zx^~Arux1Q0R~~DDz3G0)oELZ?2AmH-=KNDTaWi)zKn~fAeO77&y!A2kkdwwzl9j$a`>xdFHlkkoA!LG>38b9nXN*UfGs}#o@^G zW~@s;lAe|3Omz{4eFy#~PNh`ZLf9?nPpqF>!$=1lR=LZaUE+#DCyRwpIe4;^IGq$Z z8J@KTgC=E}hi4m_(>Pv+r(Qt^TOPJg<10@Ld|>uS;L`;#o_&wl$MI!%txh$IMKHY( zu6ox5+&X)!?*Z`HgnbW|@j$l^+$}pEbSQ%sR@!Gmk=T<|_x^2I0rxItth2 zl>_(W%nttQ+V=rHWgo*<9grO0_fz&gxL|$hv}w4&qC7a`6F-BRlqS#An%>gn>ArGs z?_r2?>I-ea^)C?RI=EyzaQ+;kYzyDK9%MZZWnNk_-C*VxgTJ6~>a&kSnYrUPbel#I z*o8slPmavUe8>HVOz+C%3&Z8$<|iPM9dGFajwc~tngaSN71FCGCM2mv!{%Do&!tON|}=*H)O8zsiD zMZ@jI;C+xmd)nzkOMLtg}v>!XAw;+q`0cc9Bq&s z1<#%w4^P+%JfxDk&S(myQ39(grm=pu&Ze@6d$C2|e|r*i%p;Crmcv&)4~njWfiQ>b zdcmS>7|5DCpFR#!=M!VN@c2#7n7Ogp0y{IGScex=yZ>k!?Xv|2u7<4n2a2mf$OC7UU5pG3OndyC$8)+Id*+De)-ipow?T+xrGUJwqJB`B$&`uGyOarx(K{Yh;Oo zIR8(*3(THg@a7Wu#KRHeIJflK2D7*a{4y7&Cc_gOF{kb0F0hw}`%(H(+XtXhfYYRh zK8=7P5r(?#iR1|QFOite%3fFWBeSRL%Uk)GyM3@+CdM)6vhLI1xC}SfWuWuG&N2wg ze*W1Tf%7{Mw)l+D1c-77<5sOG1-~hWuqAD;JZSc)U&3at_yM|_IEZ?VnBdTA$jW|Z z(J9ce2C|Sp-ZU{;NsQ*;C68`zGkdn3zsm>UK@Gwozz9TX|&j^93I_6*E6nElt!i1hcB Zz%1^cZ7@42q7wXOJyDF6-1R-{{{h72SJ(gm delta 6334 zcmds*dwf%6n#XfaPA;dkO@S&>C=F1ALQiro@3}#Nq)pQ#ZEj7{rUiwQe+z-9cxRy3XwAdWC&U2S;qjb(a6O zfAo^)eZKGW{@&+(-ly-@BlET%nYVog=Dl%d4dy+t`twah*{lO4xz(LJh!VpyhMTgd zh5A{=4t9+c70j1!*V_sw84`L0H(Or>@7!$#x&P=MB&J-sq$HQ? zx!2e@H;{An?jQ=X>z3r+>ODmW=_gm_p1$fRQF8ljQ*$?M9x0mdcSw4h^{*6tFQPov zH%Js_rs=t^{(VI8o#E$l_ANVzs-W}D+>tG(h~hM7hhCKEW`luig|{)$NadZ(9&l4 zIcXqu@{_y=CdKmi7tjTd*!pa1upYx3hU3_BY)NjiJu=*R1mi5^lDU>?6&2WZ4BY>( z*e~GKzrtv^^`yB7I!>6T55IQ|JHom+DH&?1t!tGVYEvV(%oDS zWTBe&&@BN$6hJbXh`T*>YkjN7@AK7(HSv1*-e;KI6^qmd8@)_(4PTRVvrWEeq+Sj% zk`RoAg+M?O6OEETDf$`$kpPoys^t<~qa;V_8pTkfKgK*7Z*IB&f^Fto$-#WBh@3({jJ%USRHAV}we^Ccb^!r$UtpDd(z!n5?O|(L zWtOkw>KojxYaEiA?&)%F%*0~Jeznu%4u|7FX!6E3_oSVT&c2oy{PC!(ytbyVwuf~@ zvRbw`l1eo6MN(pTV>ILJ?`1U6)8q+tc{Z+zM>i&fPy^E+YTr!vzz@H`m;u%=ib4|; z@V0uBJRJ~de@)!$X^OOjYnwnQ7>L(32c+6SlnXWSv1X}0=@ugHL{ek|b&|x%u`7s4 zhBDnWpEOvlRxPV`t?lVjv*oN(&amZjYoNB>r#RO+*0#0B+Wg(^tlt@SWYX2~mWZZq zSW8b00Xace2f#Y6GsMRF7?4unp)=Tg(8hAHx;05bxh~pK!};LUvzXwLM6R0gNBxbgH&pFO#KK&Rjs+XSHA%6V_O$X+ z5Jesevm)Q>s|CCqa(e?oCR|$=Y6yZxpAX(}7MsZ{iJrvz_5ja!b8Q_$%jR&B@7kcM z&0#OSrZ3sv5o`(9U*&Cxgi@QL8`4cZeoz0J@bK$rF^Ao1qY9VXb`@SfTx7>V;lRbF zc783P7hwPA)V7?@?PnbLTGz#HILtb6{-QM=e#eQI+4GJYY(utAVfZBD*iRE~Mt_qyezyywhhD%xD_jtiXdferNq+#{GA%q; zTjK{bsF9i@Y$z%;#l>i>iD?p|@QIW7Jm@=#hZjU+3E3O3kF(KcZ)?yW^|plSnt`A8 z#yzOSv9-bahIl0AZ5&1}A2_3eAkj*O6)2wJG8E8Qj#AVVpg1W-i;|RPc#+Mjn>rRN znYMN9-D+2NcgF^`ouwH-(IUmlcA8nt(u--}06Y*RFjQoG!&d%nu-oj$Av<9%GQVlC z-Cg+Rx#WQlE+g{G(=rg#EYDGbAfzcE3MncDQZkiMfSgG&ESr%t@U3M8Kfp@%K+i_| zXgGG3UCePP7CS|wuQ`?WIo%zrGD^2PXR&=wu%o@we!*z&g3;G}xgyEVNMj}k12`ZE zNz7_UV5HM+RoM(+BuSx!3@=h3qb^fWV4_Cxn#5~7r)mnr=WUtO)zNlt(q3g_k2+^! zG$%^@@Y&_Wb50h?IBrtB)aWKBjA_iVgLv2${5Cq_^Nk5|T;LPLJ+zhBY#We4ie*!b zLNPp-qJS)mlp@ePC9!mx6=_Cg0BgjZxYd9qNK`HJQVcDj?nt3DhCzFk(EydwXh5;Z zK?W=mgiGN@C(;l(UqXlXB%#Vohj`EBq z^C?;4sf?ON*_la8R7zweiqX=%MvIKh2omh9GctTfkG)_3J58nuq@zUjxtr3bBdBqBZVkNQfLY&yhh0=G>TyuAtR|7nUh%f31wPT zuA!*_t%#JWo(ogpDTP)Tij&a?tc)U&Xy|j8+~qQdmZOS*QdlGo04hx>+-T}?GK=Kn z0Ve~v-(i|r&H})3Da2G1TBZOBn?gfU6eCKsB*-EFX%UV%Oulk~0U8ntMVCSo0|n6~ z6$zkZG8`+YEDcxzwm40T=F*}fr&7qGr3DH|j7CX70aS*gH98}pR6u>>Jx&u}o@QtU zsjE>Lc2p#l2b7F_DV7n?RFVJ`Mj4__cb#34jJj(=;Ygf`OOfV4q&46XVqssMB-g|w z5Q{`&LQ<~vxtn4_qrWch3pTd$VSja$=e<<*QJs=SKiRM6UK7X6l1Sqg1(3ZP77(%SIh4s-a*;;)`t6pDn@l2DQHXC^Q3P!=Y6x8yudv0<<)SWZoR-h&TP5BJWgiw1?F)wo4#(|Fm@T9 zpPabBK2B`oH=8fK)y8=bJ!pCAfu)gVxjtImPzQX8dT%`JT{POa@)U`$?@F)j$Yxy` zwYOVIZB!lUjy9gIX>O?owY-Pn>m;USXtnjXws~Ji;8TqhId14C>&n^xKc8ygP5*3N zPt0BM=$~)dX>BM+g^NiEDo@c0J=)}P=n_LEiBeK&QOl@2tH}!7w9|T;z;7$R0j|E& zdarp(Ymr`B4qvpARj^@~^$JT-^G|-EYja^INiKyWyR3UGQ&zlwSohA!4PIRZU)gQl zMu^*gI{;VRWgQ_Vd#>t%kv)jGr2XA1q3dqzK1;>k@BNEjItv~sK*-8_5YqP2W=Ypd zVYCn-pS=emOKv`NK=;msgDcUPNA@D-6zZu#xP0Fz^NRhiLHb_AT+p)32JQD*_Yn*? z;{<&6KI;x*TJej2f^CD=Q-EI!DU0^PvilJOFZ=!{u;>Bn zX;5_dH%}L!m#L|SPJ_)>=qgoSbiG8(e+Bdb+5L z${x@w#JsULzxl@86I~qq(Lw8aWAXP7!Y>Y@W!N6xeJi~EA+(I*${jOxFBn}$6+HSd zl7ej6wFrhELCDL0_q!*cYY0h^_vf9*AU%wvuu*$IhHnmAcTArA^y2|tb)J6|EH&l^ z`Z;*`kadrF%V+QCUPtb9whHd~DMHS9uJ|M9d=w$gzg&9)+8;wm@TqYK7X1wE*rhAx z@6$^c!ao{O2%rB9?bwWwclYaB1-y|!+TMN`?O4%2y`R&)3v$1@stR8JI6_t|xwRSE zoPLM8hQ!!aX^qMT2pyil{7)-WZl~^8rRLQlG}&{ zKd(6hS7Rg!>yCkWFp86>iRtls2uhYHW zb72gb5JF!5o!5K~|K z$IEr^;<5DYwIRmRU4aj8DI#O@dGrsB!k`9CR8Yr@GDo1KR7OHAm8Jqo(J~Sv>e{NY z<277F?jfp1RuORZWb!D1-%#a-6;n_k3(Ftd21}-*K)Cl#JPq@T$$gd?zifL+FKrz= zS05FV=%^mLzFXH?#?BSI1hI%SGtAI+DPk3bx3G|$hFHX`>m2aYX^2(k`tL^g?q!H& z%KRX#m&V}>_gSmptJ4unzWVLEbS(xq4O*ANt4a{7bkCvNpnV2nP5;ep+u$E&AQsiO z6@#zOM69A%@iF*JDPk@BqaB_15PWMGv4&ta~Wb4efN$PLw0iI zWCFZuZh@;A@@?Z!Zm>f?OCGfl_B~JN?H%y+43un*9J${zSuZ`Vx3$AJXQEGP06A#k zr|tYeU)TnpEk!BH^5lAg%l}{j+{vSTSwtTB6uJcR6aiW;bHWM{gK{4G^hm*4sNUU*~$iXrdi?Ki@G zE0IV`#cOu!?L9Dm8j|GT6-cC6|0(tBZQbyT(NA_?iJ0`u*S6>jyWq%WNTjVa>4Gn< MA{P?No{u#D4?#ajMgRZ+ From daa43c3bb959b5a8d637c85d90f17e63a64f14e4 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 18 Aug 2014 20:34:39 -0400 Subject: [PATCH 04/49] Add better messaging around pulling of base images when they fail due to invalid or missing credentials --- data/buildlogs.py | 7 +++- static/css/quay.css | 4 +- static/directives/build-log-error.html | 27 ++++++++++++-- static/js/app.js | 51 +++++++++++++++++++++++++- static/partials/repo-build.html | 2 +- workers/dockerfilebuild.py | 14 ++++++- 6 files changed, 93 insertions(+), 12 deletions(-) diff --git a/data/buildlogs.py b/data/buildlogs.py index 2ccd03899..8f184de27 100644 --- a/data/buildlogs.py +++ b/data/buildlogs.py @@ -25,7 +25,7 @@ class RedisBuildLogs(object): """ return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj)) - def append_log_message(self, build_id, log_message, log_type=None): + def append_log_message(self, build_id, log_message, log_type=None, log_data=None): """ Wraps the message in an envelope and push it to the end of the log entry list and returns the index at which it was inserted. @@ -37,6 +37,9 @@ class RedisBuildLogs(object): if log_type: log_obj['type'] = log_type + if log_data: + log_obj['data'] = log_data + return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj)) - 1 def get_log_entries(self, build_id, start_index): @@ -106,4 +109,4 @@ class BuildLogs(object): return buildlogs def __getattr__(self, name): - return getattr(self.state, name, None) \ No newline at end of file + return getattr(self.state, name, None) diff --git a/static/css/quay.css b/static/css/quay.css index 01fe84e60..68834f282 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -2535,7 +2535,7 @@ p.editable:hover i { margin-top: 10px; } -.repo-build .build-log-error-element { +.repo-build .build-log-error-element .error-message-container { position: relative; display: inline-block; margin: 10px; @@ -2545,7 +2545,7 @@ p.editable:hover i { margin-left: 22px; } -.repo-build .build-log-error-element i.fa { +.repo-build .build-log-error-element .error-message-container i.fa { color: red; position: absolute; top: 13px; diff --git a/static/directives/build-log-error.html b/static/directives/build-log-error.html index 095f8edd0..cf03fa7b2 100644 --- a/static/directives/build-log-error.html +++ b/static/directives/build-log-error.html @@ -1,4 +1,23 @@ - - - - +
    + + + + + caused by attempting to pull private repository {{ getLocalPullInfo().repo }} + with inaccessible crdentials + without credentials + + + + +
    +
    + Note: The credentials {{ getLocalPullInfo().login.username }} for registry {{ getLocalPullInfo().login.registry }} cannot + access repository {{ getLocalPullInfo().repo }}. +
    +
    + Note: No robot account is specified for this build. Without such credentials, this pull will always fail. Please setup a new + build trigger with a robot account that has access to {{ getLocalPullInfo().repo }} or make the repository public. +
    +
    +
    diff --git a/static/js/app.js b/static/js/app.js index ad6527fc1..00c1a27ce 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -113,6 +113,14 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading this.currentIndex_ = 0; } + _ViewArray.prototype.length = function() { + return this.entries.length; + }; + + _ViewArray.prototype.get = function(index) { + return this.entries[index]; + }; + _ViewArray.prototype.push = function(elem) { this.entries.push(elem); this.hasEntries = true; @@ -4021,9 +4029,48 @@ quayApp.directive('buildLogError', function () { transclude: false, restrict: 'C', scope: { - 'error': '=error' + 'error': '=error', + 'entries': '=entries' }, - controller: function($scope, $element) { + controller: function($scope, $element, Config) { + $scope.getLocalPullInfo = function() { + if ($scope.entries.__localpull !== undefined) { + return $scope.entries.__localpull; + } + + var localInfo = { + 'isLocal': false + }; + + // Find the 'pulling' phase entry, and then extra any metadata found under + // it. + for (var i = 0; i < $scope.entries.length; ++i) { + var entry = $scope.entries[i]; + if (entry.type == 'phase' && entry.message == 'pulling') { + for (var j = 0; j < entry.logs.length(); ++j) { + var log = entry.logs.get(j); + if (log.data && log.data.phasestep == 'login') { + localInfo['login'] = log.data; + } + + if (log.data && log.data.phasestep == 'pull') { + var repo_url = log.data['repo_url']; + var repo_and_tag = repo_url.substring(Config.SERVER_HOSTNAME.length + 1); + var tagIndex = repo_and_tag.lastIndexOf(':'); + var repo = repo_and_tag.substring(0, tagIndex); + + localInfo['repo_url'] = repo_url; + localInfo['repo'] = repo; + + localInfo['isLocal'] = repo_url.indexOf(Config.SERVER_HOSTNAME + '/') == 0; + } + } + break; + } + } + + return $scope.entries.__localpull = localInfo; + }; } }; return directiveDefinitionObject; diff --git a/static/partials/repo-build.html b/static/partials/repo-build.html index 3afe87508..225f58701 100644 --- a/static/partials/repo-build.html +++ b/static/partials/repo-build.html @@ -77,7 +77,7 @@
    - +
    diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index a4de1cc47..d200a336e 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -223,6 +223,13 @@ class DockerfileBuildContext(object): if self._pull_credentials: logger.debug('Logging in with pull credentials: %s@%s', self._pull_credentials['username'], self._pull_credentials['registry']) + + self._build_logger('Pulling base image: %s' % image_and_tag, { + 'phasestep': 'login', + 'username': self._pull_credentials['username'], + 'registry': self._pull_credentials['registry'] + }) + self._build_cl.login(self._pull_credentials['username'], self._pull_credentials['password'], registry=self._pull_credentials['registry'], reauth=True) @@ -233,7 +240,12 @@ class DockerfileBuildContext(object): raise JobException('Missing FROM command in Dockerfile') image_and_tag = ':'.join(image_and_tag_tuple) - self._build_logger('Pulling base image: %s' % image_and_tag) + + self._build_logger('Pulling base image: %s' % image_and_tag, { + 'phasestep': 'pull', + 'repo_url': image_and_tag + }) + pull_status = self._build_cl.pull(image_and_tag, stream=True) self.__monitor_completion(pull_status, 'Downloading', self._status, 'pull_completion') From 35bd28a77e1b407a31487b8ca242c34d385b678c Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 19 Aug 2014 14:33:33 -0400 Subject: [PATCH 05/49] Add support for the Flowdock Team chat API: https://www.flowdock.com/api/push --- endpoints/notificationevent.py | 2 + endpoints/notificationmethod.py | 55 +++++++++++++++++- initdb.py | 2 + static/css/quay.css | 7 +++ .../create-external-notification-dialog.html | 8 ++- static/img/flowdock.ico | Bin 0 -> 5558 bytes static/js/app.js | 13 +++++ test/data/test.db | Bin 614400 -> 614400 bytes 8 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 static/img/flowdock.ico diff --git a/endpoints/notificationevent.py b/endpoints/notificationevent.py index f1cbec42c..e393dc134 100644 --- a/endpoints/notificationevent.py +++ b/endpoints/notificationevent.py @@ -184,6 +184,8 @@ class BuildFailureEvent(NotificationEvent): return 'build_failure' def get_sample_data(self, repository): + build_uuid = 'fake-build-id' + return build_event_data(repository, { 'build_id': build_uuid, 'build_name': 'some-fake-build', diff --git a/endpoints/notificationmethod.py b/endpoints/notificationmethod.py index b49055157..56adcc0a5 100644 --- a/endpoints/notificationmethod.py +++ b/endpoints/notificationmethod.py @@ -4,9 +4,10 @@ import os.path import tarfile import base64 import json +import requests from flask.ext.mail import Message -from app import mail, app +from app import mail, app, get_app_url from data import model logger = logging.getLogger(__name__) @@ -187,3 +188,55 @@ class WebhookMethod(NotificationMethod): return False return True + + +class FlowdockMethod(NotificationMethod): + """ Method for sending notifications to Flowdock via the Team Inbox API: + https://www.flowdock.com/api/team-inbox + """ + @classmethod + def method_name(cls): + return 'flowdock' + + def validate(self, repository, config_data): + token = config_data.get('flow_api_token', '') + if not token: + raise CannotValidateNotificationMethodException('Missing Flowdock API Token') + + def perform(self, notification, event_handler, notification_data): + config_data = json.loads(notification.config_json) + token = config_data.get('flow_api_token', '') + if not token: + return False + + owner = model.get_user(notification.repository.namespace) + if not owner: + # Something went wrong. + return False + + url = 'https://api.flowdock.com/v1/messages/team_inbox/%s' % token + headers = {'Content-type': 'application/json'} + payload = { + 'source': 'Quay', + 'from_address': 'support@quay.io', + 'subject': event_handler.get_summary(notification_data['event_data'], notification_data), + 'content': event_handler.get_message(notification_data['event_data'], notification_data), + 'from_name': owner.username, + 'project': notification.repository.namespace + ' ' + notification.repository.name, + 'tags': ['#' + event_handler.event_name()], + 'link': notification_data['event_data']['homepage'] + } + + try: + resp = requests.post(url, data=json.dumps(payload), headers=headers) + if resp.status_code/100 != 2: + logger.error('%s response for flowdock to url: %s' % (resp.status_code, + url)) + logger.error(resp.content) + return False + + except requests.exceptions.RequestException as ex: + logger.exception('Flowdock method was unable to be sent: %s' % ex.message) + return False + + return True diff --git a/initdb.py b/initdb.py index 7e48ae3af..cb56d987e 100644 --- a/initdb.py +++ b/initdb.py @@ -251,6 +251,8 @@ def initialize_database(): ExternalNotificationMethod.create(name='email') ExternalNotificationMethod.create(name='webhook') + ExternalNotificationMethod.create(name='flowdock') + NotificationKind.create(name='repo_push') NotificationKind.create(name='build_queued') NotificationKind.create(name='build_start') diff --git a/static/css/quay.css b/static/css/quay.css index 01fe84e60..a5cdf019b 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -4559,6 +4559,13 @@ i.quay-icon { height: 16px; } +i.flowdock-icon { + background-image: url(/static/img/flowdock.ico); + background-size: 16px; + width: 16px; + height: 16px; +} + .external-notification-view-element { margin: 10px; padding: 6px; diff --git a/static/directives/create-external-notification-dialog.html b/static/directives/create-external-notification-dialog.html index d384f3f59..ba78a4ac9 100644 --- a/static/directives/create-external-notification-dialog.html +++ b/static/directives/create-external-notification-dialog.html @@ -73,7 +73,7 @@
    - {{ field.title }}: + {{ field.title }}:
    @@ -86,7 +86,11 @@ current-entity="currentConfig[field.name]" ng-model="currentConfig[field.name]" allowed-entities="['user', 'team', 'org']" - ng-switch-when="entity"> + ng-switch-when="entity">
    + +
    diff --git a/static/img/flowdock.ico b/static/img/flowdock.ico new file mode 100644 index 0000000000000000000000000000000000000000..e3d92799b7bfe01a0ae208fb9576467411a505d6 GIT binary patch literal 5558 zcmd^@c~Dc=9>?=Ko!4n=D=LdDN;E(Mgv|h930p`6lpP@;D99>HWR+EA$F?f1KGD(E zu}*;{12N2uKO)v5(J8K#EQ zv6?uW%s_dDHZE+@!PyK0xY-%W`3~l~ur+0Zw@l}M!f&swVIT5X+EMDQfPaUnqilmF z&ZTRiB1Z?8i}X;vO&=GFIB>H!mh<_RBoABz*qZ78!RO36|2;iD3Z-kfwWV%v;XCr2 ze@tAA3!50IEMnqnu|9q-F~Y5njd5)^7w&)X1w0P;Rwl5sFp~3aIJ4(pE>14_()l%< ziB`e6jq13VLr3*?ZNk+<%>hIFa@ZJkrOQxr$P(@j0?_s)XphC9jx-Qy~ybICyi#rDD{qUqI2t#dQ819b36X`~H zE;oalqZyn9_Hc5v!*b%sINy85@mKA(YnFbo2!p2<}7Bcq(0wk={5w?b!@ZCv!R9WjP-%LI=4Y6hFv&=6t2?kGlB#E+gZ!y*T+z9PXEU zV6a+%vD-q7)w@CVK!oAebr|l9hO951@N?ni#D$-$16H~SN!*kKas>s=od4#&v933c z0RKD&%sm0dACDfUW3(d~=t%^+v*GP*A?JI#t-y0pKE(L? z&Y5aYn`-=>gTsvqM~9(!Y6L1}GA#TFP_87p258&@^zX>Pzt@K~P0h$?tHno>>u`6m z!CHS$_0vDVVFVf% zMzG|H3|c?S!1`qrdUwa5_dCf;6ToT(bRGiEE9?=v+8e7yz7q=c_e0QX5!O%&^hbDT zFvQ{*@8_JUT7~1Xj}=Z5{u#nAAHkvu!mpA+oA7mS%fPN1h0(oHENdKvX*0lS9Ki}# zK4KyQu{LP+Bn4n?V1S$-wk{a)adGZbai5RRD}(>sZ)H&Yk?_xx_%DrsUOgh`vnYP8 z3`V~Z{{2yK9)PqXaCLJ;LTo4~#$PEkWG&W5h9fB{{-yjmyDF64Ju%p`;QT05Dr8Wv zltJ^F42+slFzd#kdw&f24P#&v88iY0buzfQ3*>x?@mGq92!}W}2I=W3UeDv4uAQ6s z$^NPU&@^G*~rkm!a*oU^+3Z%3dV{y=&h8%$V&pV00}JCwZb~89af=j@Niy^ zl%(~Dj~D+*ImyY2({ZMoQ!{_`zS5#bEHY>S!>SQ1VH4OM%`ooMF#l>dF#dgbk`cb&2Oq3w_jD|J^)?N1g#Yhq3hWU z13w9j10~=Dx56x}6~;kbSQDIy%~?rEPfMN9rmRdV7q{7LwiU%8tteg`cRL~?R4^U) z#X0?;J}p&?_7-)*Uh2~fZBYw!37Z|#3Zu|gm_)Xc>uz}X7b78N4YD#bkwv(2Wo95J zI~zGU*~rYyJeHT2i-P<-s((`~7QgW#XS!>nut5_w7iqIvKns?HN}v_d3T9L*^rG9q zj%$N{oD`hUCWxcskeic*EnBv{pv}n6-XixQTRy%S`NW40K6tNiI_Bg#^Q3+q&9&WC zDiQ5ioY01)N$t=|?tpem2bifHpvU*XGw~AA;=_@jmyO)q+`mwce5|mr0L8_{6_Xh+ z>l~=Z>o9Ql4rW?tpj^ib9<#q1>I2E+6|TWyP&eI8}kYVz{&Xy zv0INKAt4bP6E|S{jt@~-SU5nqM@vddcvG!A({cNQj~DVj8Cq>vGO(ZVQGey4;=V@J z?R}l|w>|DuE*NTH<#befrI&uM?XxSu-#>K9@8GG<1^!>gsdEx?bRs_?naw!~SAD+4 z{$QQ!`64!A-8u3aoT|fL_B_kBrM&&23w_ULp$y6KNIH%uuwL1|y}kYI<59e`#1@pQ zw=i&Z+pF_07su~EFIESoiWE&$ojRv^E|i^;t+?Pips<*mr%&cu3h&8oU{T zXLnPGHe$3P|CQ{2J?l06C;Pf&4A*Bsc5m}5*}r~Zu#0ix5!ffC5T1Ji-^)+2u6hU& zSD(NlGIPTJ)TAT}kx0at{2nNr7<-_2lGIm3mgO>dSB@f_)EShxVQCZgQ{`b^Y3FxJ zM@W54?3v%oU|Ti}ze_Usks9Si`+!wmpfs7kAgtJ2eS0r7vq()5(E{zTCOBpH!nUXn zJg0;Sep*@@Pb`l7FaN3a7V$dwFd{mjvq}P8vNxMYc7Piy{gXY1!v8*5NwtguCAktn!V*YG1F8 K=eB?M|L{K?9?(|+ literal 0 HcmV?d00001 diff --git a/static/js/app.js b/static/js/app.js index ad6527fc1..8676d7d99 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1076,6 +1076,19 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading 'title': 'Webhook URL' } ] + }, + { + 'id': 'flowdock', + 'title': 'Flowdock Team Notification', + 'icon': 'flowdock-icon', + 'fields': [ + { + 'name': 'flow_api_token', + 'type': 'string', + 'title': 'Flow API Token', + 'help_url': 'https://www.flowdock.com/account/tokens' + } + ] } ]; diff --git a/test/data/test.db b/test/data/test.db index 4d04283311e6feb845dff3fde9a4d370858119be..a947e59643b1155787f4970e641fb95bf1b9aef7 100644 GIT binary patch delta 6225 zcmds*d302Dmd9&(m8zG83bGj>P=sV4F|YFW1yE8+Rb{JGr7Ec;6+n3J)q5c!3)x8! z!m~xXXU^#ns^!;%xUs0w&>(0cwyiicc8{nGwg~NU!Q)|OROmCO&(M7k!MP#ru(sOG ze^Y;?Uf%uvKEHdv_x_gpZ8(&(;ZV*)Gt4EMXBL}Fo?o#ml!`fjV{(4sJnekX`I`e1 zU8Byl6wGSM)LYv7;_d3H#Cyw0X!Bc>^Hb+4(@^I1XiO)NaU`ZD`h7gcMWM$YtgjccCYD zI@rW#-NTsWU$_1^k?0x1a&zbRCJKAEW7G1&mlCIY&tP}X{PSapf9re0y6;qSV#}Iy z>C?9jxQtfERf_QpP!9JGW7Y%PHYd6UhOnGRT0T#>)^5iN{-d`kacC`2F7PiWhVDCJ zpYh(_nMRux<==tdo!Gx_k3DJCYNzlr|Y(^rJ0nSfC5uj9it0lpy?DT(k*IagBYk5OT=sw(7-)ldP3jEPYnRaqSIdzolOfT|Bx z*9Z8@x{4|ieRu;zn4HYvLfsn5(8u5pU>8kpgki^!-D#Y?qk1xp7 zdSi8dHd-2^gW)o%JVXk$oR>ruUz=;WqNV{p+FaJ%S07=@8bW1_vBs`IPcPda=<-!{ zHz+zb{Dx3bIF{~$twqH0~M8IMMSQld_ptX=Rx|T++ zzpP7(HP^1LMlW49&ts!)LNrj(9#iYxo>-^S+1%e%&bfP3t*Ei1e03ko)D48&oBCV0 zQn%jXYl+7@+$+(fZ_Iq5UyMb(L0{BURbO6R$CXH4iuQ>j7l0&}mQ_V0Zy-|Kz%lia zk_wtGDWe<8#NtRpxPfQ=;f4|t{p1_-Os;sf$gYa|$f&=ihp&sSig!0Kawt?0u86iW zqA(y(p5~T7tlrbnP%;qXgXNwYy<_x)Z_Km{o3PQjV>I1mVJ8gUZe6GPFeAkcKaxpmHA1MuZi+uhPiDIisJ`yTn86hl1d}UrA8K9}EhCrl@<*U(a zpICBG=_i)Ze6l(cCd1)KO$kf$f%^J7UMljG$D#o?RLl6hK0ocLspjg1QRwpD1z|>t ztD;781Stp+0@qU!}FtB*hY>NKlfC zrWR1r0)}!+BrkH@NV@eyXZ8=lKIpQJxUi)3qrIat%9<-5QMPfw$3%qJHmE@0>dtl%O@_+;g~ zvbx)s$K`GY^?Pq{ykNp1d}zU1^3<*JjD^mjX`|_@>vvgWTB(k4xYko$q3n!JDr3 z)+-+N$}K%w)~)>jE>9+#OeTx-yvcdqdC_DVoluCqRZ;hINi13zq)+l~eg1@f7$OdeUb5Le@R^~{^Wt@Xt>Jm?gEYA_FEOB&P zQ%F{iTP+0^hshkpVF^rTApj|`+OUhO)(P9tFlbU zvH}h>5=BT1ABVc&X@;dWK@cQF6SljvNlj7WlE4rYErVU>I2c`s6FMU(B81Bbl!Ow5 zttcCGQsVKkx>95jv=V4SQdJUS(pibsw79~u=mKH8D_bFXF3yoUp-D1C&wvh)5(WxV z*IAW`i=w2VQn$^MO$(YX$Z=??i3*HP#R*xVMF@)$SSBuLlE$MwZrjXkM&=||0q+E! zBneg&Swf<~4^@;@p3xKrnxb=VTWPi!CnZTx1Ojr#Ld3cXdg(gBLmDK7*5ew~;}yhcN?5;c}qUK9yYW}zb}!cz)Ou^M`+z{bywb1=0yiX&t`4k=PN@Q?w`Xh{P( zRVglzqu&(Rc7L-Z5RF#x6i4|2o(3-|!f`s#5GMUK!J5j_8m^r8RWS@#R6$pj1|l)m z7Y=&qVu`KvlVt&pEb^B-Zkiq(2+t}?T#=~n6#}RTjLl-;Gi~jMFEj? znS@e>WOBU9(5gT(8Y_>!+MX1=chWU88GI2Li6hbMcaly`zQI6?lE9&qV@W4(v`_-a z@uPn_mejlUIuq;WKy!vc8!GHuLO;Z%taUXh$uJ{LW|PO%h$r8?nsEOIl>>I~t%v)# z(mo%Nm39oB3D|RP-3EG3M`ERY@vY5($VD$y+Nb_i^F3o0&NaRSUgyJM*&>kMMFgJAsf2SZNWY52~c}K`D+|n|7E@Xf9-oa~g`K^Mf zv#cKT&Gq=LIpw>}$=_|puQLxyHwNUm&VG~Gex3OynUk(F-z2l`M)Ruc(|}!Pze#NC zcbl)@=5Fdevd?~aZy6tvxGG<`q_R5dgIuCa41N8py`E_G#lXJ=tt>+P167R{}; zw-$M8N~(*iyv3x)6A4uZ$9LS5Cg(}#S=cF$I&&vm3G^bb3b1Iy5>L`&M+%B$cv?fNpKzST7VmlJ zeYE09$B=#A(R~?4W;QzRz!#yw4#zV4#Nmg}8~R+-f#Y|hb2}Wn?b7ar?S?-qF?`=5 zbYiFDK@98P!=UB69OtkFDX+YZs&)g?5^Vhx6+Q(>?BMLJM&>N^d>SD0p9Z9O|JHUx z&qUz~fPDLDKvI*47Y+YRG`tkry!Z?-GoSjr3@si4rt6(uUm)^XU~&WUuaWCHVA}Tn zJk`j&6PXV<7NOK(U}hnfG4yHZ$penN(Vk&oazj5JH2hOhfQC`F?s06y7Ehl2D4P8X zU}k)HIRmBdb)04Ky`Ndr;QThpwA$o!PVlEKOMNwUV#=Ik5^r>@w|{J>l03F5Y?Jjd z%VkU0EJHKTX3ymOf+%z{nQ$F;YDI#*TOQjuS)OGHk%A{xRs>04<2szMP{%&UIs0P% zSiaFJqD#}jpB>LThV8SG+*6D^0iC-OqTak8a+kN_{*Tb*{f_O}#O_`LJ$Arx#-6m{ zUt)%z9gAZTdgBGq#&YuGi>T_wF};8El^1zNKpT8r?IuKxI)z69073 z(jAfX)5gra04%<~>9dhTj;_LaG)?L}DZxP_&cn|dT7`3m!m&i0B^gN;RY8c0hCeXo z-NWzW4`L|=e^`iCnDKMiBy@NW3S02A77w<=Zu;SY>;zaX5qMu^; z8N2keRlhR){wrRL*Z^UQlWG#%fKWrQF33ayxkAHh;mPaQ?McYq`Fa|8@cp9GGir5>Dzk|*Os_M8)) zZyA{lc}j78J6Gv)M+$rHV*g8?wpDd7rt~Nw)kN9 zC|W^{y_*$!0hQ7C5xaB2R=d&Gj?T`2B8xEi9y_<<_mxI#8~SJ_yeVe!Vf$3EA8vYE z(c76YH)sxT#;}R0dFTla5|Z=Qe#BXe;KI6e6Y=Jw04iLGdu4E<~W4kU_P0u2NSkm|0>sT%}HC!J1rCw)tIC!KJu?ylmR1abp{ z8fr)IVQvfb=y6zd5d>j2sDP6&Ix4QKE4p5Xo9u|r?y$3?#C2w8Tt`RTH)L>tFzOE9 zHXrijsehf{dC$Mz_nbFp)A4DWj!)Y*-sssrq0;C%y!7Evj@fq7V0+*8p6%q(QT7*Y znUhOscy`KQ`-SZT+bMYV&$gqdTJJFulO`S>msz@Y7ctK8vSCNt1!CG$;q94Ux4l4& z{&c~2GDq46^7E$459rM!M;PMz3~sVM3qQ2iiZcJ+(Myb8JZD@c(|M2>Q#aM0Deu}v zT)R629;ju$M&rGN8@-eSh3qsh6MBX=IM%rfr(H@(?} ziwfYu-n)7&(|+I~^qEGSS4ft9Ye!H2jvnLKJIXmJ5p1lgZjx)|n8<_@^>L5S7mNFA z8W>-~)lh?Ep@MePjebECP$CkKx!iP9O_ST_^;V0Ou^RlrXGVK@G+Yy?^DqsSd}YGL z)_Wu28rjcCLLeFv{C-J{*GayF=&kjK{Y;|1ii>k~k{qtC6N7cW5EH1vHGeeL^UK=2 z+WNL&^@dgccxc%MUzMi1WJh;bWp~}G&bk#2r<0?*kkl1%3v1VFb*WV=gDutg^dF5S zyq~S>u61@bw8pw)+=`gsM%8XdDB+B?wmFiW65ZX@78NVPO>(R*u%fC)P}FuLVD58c zDW7nz40<}*$|jlRtGSw5SJN_wq^3ICovYK)XrfzP>vn}gF(lM`qU$?T&N}CY#wh;5 z;81ya<%X(G))8*g+PcEYcSeLW>bMZBWx9hc>*-GX z`_GL`59<>}p`P)3nmh@f_6xMHGUjpDhZ{px^(Ywd$Eq9rQk6f#1?%}}gH)4n31L?} zAu|4IN#f+_B4UJLlAbl4G+3=xtxauT-r26U6|+h)!xqa;{;C$Q;#}ca-rN#x_I0$d zK4-|0PF2Jj!5s`otKD%56b!hbN92B7Sd5u8y=;a$a2Ug;6M%M6QDIMSOLvCs^T*M?+kc zjs|K&l?kzdb~o`-03r{DSdnk?Rv}&vx;*{>6RN5X)&@|W*NeA)VVuA#@y__l7C+B- zaLuhk>kB!@$CjO@V?f;e2^qJ$Po$;Ep4MwA$ zH^TH+xzlr1*?Ubxs4UaXI^;DDkxZtq%R>9trD5MvQ=Z-Y6Wa_p1l}CkQ=J8CHJWZV zpNKzk{;!_(T)$PC{{*^O$4s8Vg^^$9*W|A)+rDk4&w?cZiS{z0nDF`GSZL(As!AWC zQKi%nW`hx-J|;$@^-R4G!Oy;HnufjannER!Xk7NhYGQ1p!P6A*MLdnc>IUSaJux?I zNo-Z1rZyIidg}UN$VbkIAV{>5W(A68xHN?{mZKCki6~A=(xN1#7+z%C)HSWMm2~rp zmJYSOqoZ||+QQNdqG*v~WjoExX6e~9av&ZF66(u0|J+vmO|V<+<~}=-ouB=V!M1yot6p-!1?pECSy?wYsn{Bl{6osotl2ZJ~y2nnOr z5I2%emsMrch>;|P64Ja|q)=K_AUaW_cunFpo>Mi2;d3@lX>V=5JZYD*x>KEUV>Bm9 zywq1tyy~0}UJjeoRAhFMH;n1>z#N0YV6r`Cusvow3-0Z;MM}(Z<2NzSG~FC0uU^Ps zT1Vd|V!f?LMoE@UG781;T#`bvEK-U<^OVHWDORKzl|ii83NcEWQe>o1ikL(~5-BW;IGmxfDvCKpNu|IdijfqWLJF@@GQ>tP z3?rl^H7#=zi~m4n%_`PlDnu(HrK*?X6nILZ6^7zuc)-dKi9}d!OLNSXF$0dQOD6~u=2%Caek`yCKv?RzPLMagsII_IO z0)sSg3ZhHG#7F_Uq=ErTI?b_y%F>7xaHBJ8=2V#$6*-v%NlOV7k{FGWkbHor+weO1siHNH*7z)Rjm=tdC zhnxIvAsX^lOLApYLeX$IDkS78ud6;P)cLAo-auUwAM#a1c-|vag+s2Gb$EFo2glKn zVK77XbGGuEouRKhd#AIQSJQ&1@+#P=z|v_J{z|DV1s^$yAXS-^ruIM6njKtRaCId^ zEYhHTlU~a{TX>xyB7tS_h}W~vT(1ZSc9{N;U(fE^c#R}FSVoo@32&;g>mI zz;f$fl;;g9=*qr(byVZ9KWMpqQ09iomqY6fT1sv#|7Xw=8C2L|10mj9a57{my|KI} zWD#$u^bdqAKU&;#^|!ndYwkp|%Q!qAugD|5TF&`unY>2s`SPr|LbqO{50fp|$ir;T zzD6EqbJq3ps-e^I%H+f~`Y^N2UoBsI%ncje_qgTx$9&-ixh7IkTaCQ&8c!_bnKjs( z@)T)W*`8Y7+SXQ{R=YZsd)zFJ}$hfc~f2HQp388|6lvCSwn zQ^xC&rUElXUO%*NjdkJV|DX32c*oycR}xbfJo&r(c3W$UVIyObg3430LJzir92{z} zMNvvJC2DDvXEj;DYj#^N5~iKSTk+C|tOqTnqnq;eqGJ4}l`O-xd#sBr`3*mOQP-y8 zwIn$g5A3n-vy5JF?rGgKCDVIH89uewx`hz8{c8_i^00M)80B8qiNpHL$#!LhiF?csnl72@j7fN{V!P{>QfC|n84M(+KxYIU1f{YX z9N;PvD49bNC9oU=eH5?4fr-~V4s&we-OX4zY`vI2iE2Bn&k%ElZo;OUGtahjc-s-{ zN^{|NkKjv3th)%?6MOH+4<5B%uoRZ=nxK2o;4;eai%){lWGdhm;IOVm%k1_ou5VK!dejZPL3W(@aa~~e{Q!xA11=A1eMKkel%n-rnKLxYL54?X! z*Jj}D1X%mP(_nV~-+h?TJtdicSyzTPKV!YeJm>xfY^p94FP|}=<|>uQ?#b?v?^(W%*nc^b}+1aPm^1Sl7FncgqIpgi0Z+fX*gmcFA`&8 zj}X{pCJz$aqxZ-0hh}mY;V|v_5PzQ_FIdLiy?%x6@m!9gFAEUz{O8`l<=KF+-F{&r zW-Ndxc(+Z!ldXWrH_ttXU$X*&rOIu%g#-lg^5*aBp7P7deP0eBinfeBjqM`i5sAPUZW>vr8UdnlQ^Y`~bi$N&5L^2z9Q z9{vkb80gsz&VBFLCg48n%~lknueYHmEjm

    &5}AXy37&*ghUuW8dDn1^;?Hu&CxuMtp7pu=1mYFX5MqfHm_scBuX!{zX4jYTra) zP5S8Pg02OyehezLVG^+Nzy08XK0CQ^6oF1>H{zuXd4ZVu!>xAgW62jRgni$$dP^(5 zI3AL%jw26QM(IUo_2w4*&IEW;iO60HKW6tw`pjnhY7wL;%abb!F88Anyqkv)GmAX_ zDJ~a)i5hQn;u#{uTvqc8<@WdJKtA9+b{PW~A-csY&D^8hx(WH^D3 z&I4HCS>1`3&Ief0)+KNBm6JVVE!l;4BE7l$@-;Mmz7bX=n{uba>&+eb^T8*3 h7Xy<%c~_%8vmFoI1}<%)$#VSG5^^Rn@AYuQ{{qGD78?Kn From 32ea1d194f1d97e68649b1848fdcf7787412c466 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 19 Aug 2014 17:40:36 -0400 Subject: [PATCH 06/49] Add support for the Hipchat room notification API --- endpoints/notificationevent.py | 22 ++++++ endpoints/notificationmethod.py | 62 +++++++++++++++++ initdb.py | 1 + static/css/quay.css | 7 ++ .../create-external-notification-dialog.html | 4 +- static/img/hipchat.png | Bin 0 -> 2828 bytes static/js/app.js | 63 +++++++++++++++++- 7 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 static/img/hipchat.png diff --git a/endpoints/notificationevent.py b/endpoints/notificationevent.py index e393dc134..f3f4d6a77 100644 --- a/endpoints/notificationevent.py +++ b/endpoints/notificationevent.py @@ -15,6 +15,13 @@ class NotificationEvent(object): def __init__(self): pass + def get_level(self, event_data, notification_data): + """ + Returns a 'level' representing the severity of the event. + Valid values are: 'info', 'warning', 'error', 'primary' + """ + raise NotImplementedError + def get_summary(self, event_data, notification_data): """ Returns a human readable one-line summary for the given notification data. @@ -55,6 +62,9 @@ class RepoPushEvent(NotificationEvent): def event_name(cls): return 'repo_push' + def get_level(self, event_data, notification_data): + return 'info' + def get_summary(self, event_data, notification_data): return 'Repository %s updated' % (event_data['repository']) @@ -87,6 +97,9 @@ class BuildQueueEvent(NotificationEvent): @classmethod def event_name(cls): return 'build_queued' + + def get_level(self, event_data, notification_data): + return 'info' def get_sample_data(self, repository): build_uuid = 'fake-build-id' @@ -127,6 +140,9 @@ class BuildStartEvent(NotificationEvent): def event_name(cls): return 'build_start' + def get_level(self, event_data, notification_data): + return 'info' + def get_sample_data(self, repository): build_uuid = 'fake-build-id' @@ -155,6 +171,9 @@ class BuildSuccessEvent(NotificationEvent): def event_name(cls): return 'build_success' + def get_level(self, event_data, notification_data): + return 'primary' + def get_sample_data(self, repository): build_uuid = 'fake-build-id' @@ -183,6 +202,9 @@ class BuildFailureEvent(NotificationEvent): def event_name(cls): return 'build_failure' + def get_level(self, event_data, notification_data): + return 'error' + def get_sample_data(self, repository): build_uuid = 'fake-build-id' diff --git a/endpoints/notificationmethod.py b/endpoints/notificationmethod.py index 56adcc0a5..a6d958037 100644 --- a/endpoints/notificationmethod.py +++ b/endpoints/notificationmethod.py @@ -240,3 +240,65 @@ class FlowdockMethod(NotificationMethod): return False return True + + +class HipchatMethod(NotificationMethod): + """ Method for sending notifications to Hipchat via the API: + https://www.hipchat.com/docs/apiv2/method/send_room_notification + """ + @classmethod + def method_name(cls): + return 'hipchat' + + def validate(self, repository, config_data): + if not config_data.get('notification_token', ''): + raise CannotValidateNotificationMethodException('Missing Hipchat Room Notification Token') + + if not config_data.get('room_id', ''): + raise CannotValidateNotificationMethodException('Missing Hipchat Room ID') + + def perform(self, notification, event_handler, notification_data): + config_data = json.loads(notification.config_json) + + token = config_data.get('notification_token', '') + room_id = config_data.get('room_id', '') + + if not token or not room_id: + return False + + owner = model.get_user(notification.repository.namespace) + if not owner: + # Something went wrong. + return False + + url = 'https://api.hipchat.com/v2/room/%s/notification?auth_token=%s' % (room_id, token) + + level = event_handler.get_level(notification_data['event_data'], notification_data) + color = { + 'info': 'gray', + 'warning': 'yellow', + 'error': 'red', + 'primary': 'purple' + }.get(level, 'gray') + + headers = {'Content-type': 'application/json'} + payload = { + 'color': color, + 'message': event_handler.get_message(notification_data['event_data'], notification_data), + 'notify': level == 'error', + 'message_format': 'html', + } + + try: + resp = requests.post(url, data=json.dumps(payload), headers=headers) + if resp.status_code/100 != 2: + logger.error('%s response for hipchat to url: %s' % (resp.status_code, + url)) + logger.error(resp.content) + return False + + except requests.exceptions.RequestException as ex: + logger.exception('Hipchat method was unable to be sent: %s' % ex.message) + return False + + return True diff --git a/initdb.py b/initdb.py index cb56d987e..6b68cee85 100644 --- a/initdb.py +++ b/initdb.py @@ -252,6 +252,7 @@ def initialize_database(): ExternalNotificationMethod.create(name='webhook') ExternalNotificationMethod.create(name='flowdock') + ExternalNotificationMethod.create(name='hipchat') NotificationKind.create(name='repo_push') NotificationKind.create(name='build_queued') diff --git a/static/css/quay.css b/static/css/quay.css index a5cdf019b..bd8f571e8 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -4566,6 +4566,13 @@ i.flowdock-icon { height: 16px; } +i.hipchat-icon { + background-image: url(/static/img/hipchat.png); + background-size: 16px; + width: 16px; + height: 16px; +} + .external-notification-view-element { margin: 10px; padding: 6px; diff --git a/static/directives/create-external-notification-dialog.html b/static/directives/create-external-notification-dialog.html index ba78a4ac9..bf0c5da03 100644 --- a/static/directives/create-external-notification-dialog.html +++ b/static/directives/create-external-notification-dialog.html @@ -88,8 +88,8 @@ allowed-entities="['user', 'team', 'org']" ng-switch-when="entity">

    - diff --git a/static/img/hipchat.png b/static/img/hipchat.png new file mode 100644 index 0000000000000000000000000000000000000000..4b0500763a2546cb6a4f41f94d3b4cc0c14f27a3 GIT binary patch literal 2828 zcmV+n3-k1eP) zKOSs*0qtXRxLtxMcqIZ!!E-8}Gw9UqEoz5fqvdi7$Io6lnsfPNeo;|*&2^jI7Ig|A`l5Yhae6# zT|Y3Xnl$czqDOAXG%(h`Ce8&UzxbjUAfNzE<}6!snN!pZzfxEDNm? z3=J8OBzQ&md>{Y|5E>K(N`?%`1d zZoMK>9~$yVa^k?-Vj21dnE}W-SIT!~{=6w8`|`O$a5R)kurwqes%$+6#sBxCSEITb zT3y+?mc`~snzFp~Y$3w*plNtTIGo}Ax-Wfx!c*giurw36 zjA|zg8c;N9?R1#o!j?CCMRjcv*^1VHCzqZ(nhR4f09fr^dWP>?|MDGuqR|{B_w^=f2N{Q4blg*ai%VFWSAbvz!1#gax1f z`mM2(hT*yuGJ#1`eX(rJjI}MonrZDlp7-g-445>aDb&~!hV;uxU#wI%dmo*`-gUDd zd~^gX4iFgzZPvb%A8yHP8qn+_Mn5#a^l~Zm8rWRH(V?Zgmupl(0XaL3{*Ns=f9e7Z zTDa1mO25*cyA`OYSf8PsYuoJB_Ho@ zm}=BooqiCCAkY+??q{bwa8FX7)>3sQF%ib)vnRvl0wGYU(spJ!J)(cAUhCkG`NjDq zP$}RMD5GJ;tVwNW>rCcP8xs~Ei@G|fc<;5E%^5%V$LqDWXP$y6fdSm!XC~Yo7OZVM zRVVZEqr>5mAOkGD?NE+?uwLswKNZy9VR-hbaf8}U()m0wb|4HIh(2)avAm*My9a)) zVsp)vvg_a(ctvA3GVw z{+g2Yw~*L37NQ86h9Kr%sf1svvbq*t5flX?^@%jKm!$KFk2ckRorqj%t%m2|DvLdE{HMGUa4ZM`&&79-fL|*%JhVqt2z(M0?6s?w(*?yX zmZiS6G3LHmpX@jin6cwzX6JVs{o05Z-fJ;lWp$jmR058HPf{4P{bIYJ;a7gxfZp(m zU>JDaoBn;W#mX?F4(6H{URnR>oR2E4PTV9iF#p3HBd4vvRdY;NeT&)c2T!7^7CZ;R zHz+>FsMnxDYjX1KpcxPuyy73bj|hUV>B_XZQ^JS$2gr`ChoeT%d1LMNN{bC0Koq2p zwjYjoaNf#yw!rRT;$ps7_Lmkh?|+jCf5q&R{&au+$-A)(3?DnU;%WsnL9jR$&UL1%Q?Z%9w&(2IkhzcH&)~GE9RvYx{W?HMh+&gd4MuY}K^g+R1 z_;y*J$grj>0AmoA9ZDnR?Rj@ka zlj8EeS&RVYtlPF`<9^gSp;CZlArlY+AF1y|P!w1iG!2@DB*X22l7Bko-VF<$R`4y4 zGoxm&JGAFVgy`V$B8actx2B6p=bvf}rGX7^q`)IVlweTjXXicq>UspQdiLbY87rQ9 zW{k$9hszCz7hVZ81&#s7fTbW1xIJ*XST#R6W%$qAmVEKIX@O2NSEe7@c@)7K5Q19g zvgs2VoDg_UeB|}Com;aI77P$n+9pm({&vlDw1(4FxA%BK)~SNi1tn$X8mHR}89_=# zgh_ix-{@fpy~hvl-_{*==9(?**Kdbj3!a6=k$BH-r#CHZ8W4DVmAg+|Se#RcP(37q z>$X7;Bpq1ys$Sdpr4B@tB*_QJ;91b@jamUj#v^k!>`FfblO75V)egRkzC2@cH4aT=*k_BT{(1CW_-&lJP%>`Tx~hw12+@J1QR9M+Kk?P# zf$=ddB3o~bTq!drrLM4-RKTDEO~dJiMD81rv|`4jp#ypZ#&?jYadXOYi(%4(W?*w5 zSe=#j+P!zh1xB{nNGh+kjeP#S3)y)v>cO)ReQTR1m#}P~+iGUhChpm39rKo7g-m^!9%ROq$p!IJ))jPrj;K@awr*A$4O%N!$VE59v?%i75uA^E2B!cw4 z$G_N$18~ zC)_@`@6VeTwYt|$Xf;q~wq>0tIDGcfnS$anbFJCxl0*R#0DzNenhy?xx+XiE z5CNJ1Eb5$0R5$mLbv)O2n*aT@8n}ARdOE-KTv2Ipd5yWoMhG?M>5=!}*|%xT|3RxG ev@C=EAp8%NkGu3nbEE_S0000 Date: Tue, 26 Aug 2014 22:09:56 -0400 Subject: [PATCH 07/49] Add Slack notification support --- endpoints/notificationmethod.py | 78 ++++++++++++++++++++++++++++++++ initdb.py | 1 + static/css/quay.css | 7 +++ static/img/slack.ico | Bin 0 -> 22486 bytes static/js/app.js | 18 ++++++++ test/data/test.db | Bin 614400 -> 614400 bytes 6 files changed, 104 insertions(+) create mode 100644 static/img/slack.ico diff --git a/endpoints/notificationmethod.py b/endpoints/notificationmethod.py index a6d958037..9650e79f6 100644 --- a/endpoints/notificationmethod.py +++ b/endpoints/notificationmethod.py @@ -5,6 +5,7 @@ import tarfile import base64 import json import requests +import re from flask.ext.mail import Message from app import mail, app, get_app_url @@ -302,3 +303,80 @@ class HipchatMethod(NotificationMethod): return False return True + + +class SlackMethod(NotificationMethod): + """ Method for sending notifications to Slack via the API: + https://api.slack.com/docs/attachments + """ + @classmethod + def method_name(cls): + return 'slack' + + def validate(self, repository, config_data): + if not config_data.get('token', ''): + raise CannotValidateNotificationMethodException('Missing Slack Token') + + if not config_data.get('subdomain', '').isalnum(): + raise CannotValidateNotificationMethodException('Missing Slack Subdomain Name') + + def formatForSlack(self, message): + message = message.replace('\n', '') + message = re.sub(r'\s+', ' ', message) + message = message.replace('
    ', '\n') + message = re.sub(r'(.+)', '<\\1|\\2>', message) + return message + + def perform(self, notification, event_handler, notification_data): + config_data = json.loads(notification.config_json) + + token = config_data.get('token', '') + subdomain = config_data.get('subdomain', '') + + if not token or not subdomain: + return False + + owner = model.get_user(notification.repository.namespace) + if not owner: + # Something went wrong. + return False + + url = 'https://%s.slack.com/services/hooks/incoming-webhook?token=%s' % (subdomain, token) + + level = event_handler.get_level(notification_data['event_data'], notification_data) + color = { + 'info': '#ffffff', + 'warning': 'warning', + 'error': 'danger', + 'primary': 'good' + }.get(level, '#ffffff') + + summary = event_handler.get_summary(notification_data['event_data'], notification_data) + message = event_handler.get_message(notification_data['event_data'], notification_data) + + headers = {'Content-type': 'application/json'} + payload = { + 'text': summary, + 'username': 'quayiobot', + 'attachments': [ + { + 'fallback': summary, + 'text': self.formatForSlack(message), + 'color': color + } + ] + } + + try: + resp = requests.post(url, data=json.dumps(payload), headers=headers) + if resp.status_code/100 != 2: + logger.error('%s response for Slack to url: %s' % (resp.status_code, + url)) + logger.error(resp.content) + return False + + except requests.exceptions.RequestException as ex: + logger.exception('Slack method was unable to be sent: %s' % ex.message) + return False + + return True diff --git a/initdb.py b/initdb.py index 6b68cee85..7cadbee87 100644 --- a/initdb.py +++ b/initdb.py @@ -253,6 +253,7 @@ def initialize_database(): ExternalNotificationMethod.create(name='flowdock') ExternalNotificationMethod.create(name='hipchat') + ExternalNotificationMethod.create(name='slack') NotificationKind.create(name='repo_push') NotificationKind.create(name='build_queued') diff --git a/static/css/quay.css b/static/css/quay.css index bd8f571e8..2a4696551 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -4573,6 +4573,13 @@ i.hipchat-icon { height: 16px; } +i.slack-icon { + background-image: url(/static/img/slack.ico); + background-size: 16px; + width: 16px; + height: 16px; +} + .external-notification-view-element { margin: 10px; padding: 6px; diff --git a/static/img/slack.ico b/static/img/slack.ico new file mode 100644 index 0000000000000000000000000000000000000000..d32c2528026b0950f2efe73a9b96e2092f013000 GIT binary patch literal 22486 zcmeHv2V9lM()SPxb~+Xij@@W1SU{y5)L4>e)L3E?J4K3Mk1-O#hAq-(UGR4!k1BOJ0h!=YJB8@mo`9+FC&lIM82bmI-*T< zK~oS#KOzJ)kbv@+o;KavpfmMu(v^DD?m<24*itWjTe35-q2Bd-lU-w5>SNHG`Wo~l z$L4nA+^QQ5>1;_Gt$Eou1%87$KEI8kVVBZVe8QP?p@ib!#y^{4yM zhEq2qlRr3G~8+8jE3DGBK%DUi7Sr_L}cE&==x#UL$8NO6}c_!s$%%Ml$&!Y#|7SZq~ z;WW}XoW_~#rVmXI(BcVOsBWEfYEb(k)vb@%B!e2%&!9#kHn~KG2A8R6qfBaQaD}=V zTqmdcH>s`RHR@*c134JqB7;^}$)NpJYTW4>^=o&7#y37g6HHIjJ0^E%n8{rlYnn}y zf%lPV4ox=8rKx7O$+ulDO*2!Fn^_*Yn%yT)qdf9J+NC+lOiO5H`+WMWLq5&zUO=BB zF0wD6&->gXf2Tq+c<&~ejk`nLC*+X(_&f^iS4aW!2ejnv99lB?KCKvfpH_`1qTq2Q zv}t-NZSgFn9kU+L*RvncH*?DKhKjQysWc~wN)>ykEdL;t6&$A6z`L|( z`6Jr5>JjZ(_KcQZeoCvZJf(Hlo{_Uj+T-5$l!1zg!J|ba+sBP=qhg8`Q5~w<$Uc3fdG$l#(G4TvPX_g8w*k2I~NUr$i12oddrX5pOq0o#((F6m(DCR-Ud zu2);XZd0I(e_<;ZELiaAoKKf-*G1B%v--a+?_t#BjTsTj*25Oh1Hpm?bNxcUdZb0u zq~4Xwo%;{$Kj`5%;S#~x&n5H)3swb#zi%~o>ckK%1T3HN$UpunC_F4s+5EgO!X);> z)p#m+S74(HRd)B+i)POk9nA9=`1sjVk>CTYO@yJE(qr3}%%Ah=g3s0{haViPswpXz z5)!uD`C|=!M#kBr+XCi)0q(`WpuJK}feV6r8Ra++5sTkhgMKqD?fm(Svq?KPgp2V9 zEBP;Tam7Cu*C{r=`Z^C8;;d!oqO=swpFjgK8-s#2D;si0>%jLMWo2d6vsXVIc2rC4 zdrgN@!cDLP{5~x~VAa(+PhI_k3SXU9!6Jd}>N>4EkOjdM&jMvlx)T3PC43SLxp)q1 z*%kO*TiSUF?(PBUjCJTI5 zm+6zyk1a?Fy zt4N^wRPJg@k7p((?C;RJvkJX(`y|FI*Y2{{)6y7 zuAK|1C@FTEDlHFfWfdva(tLI8RDp}4XH_rds*y#h6907d2$gECNCZzRmDZ3I$50i= zT=f)BZ&hFS57yugBwd5F@QEs)s)qjJn#3GX5oQMX7l?8_;*$d6Cjh@*>OiTG$XW$! zNiY@im_RO9!51p{9D|gjTtLWw{9K<1KU{90pjr}qF5oZa1`<@rtjk3`Zn)(_UKG#( z?E)&SC8#e!1(5{N9m!D+{v^l&Du}$G!;g|LQb?JXdSFc83ugUQ5J|2jMu?x4FZHBx zGcZJ+U(`~LKLQx{yzs^q}4i zgluX{juzd?$+8!{)4dOkhdesnvmeU zeM?#p=s=dsCXqpa3mGDsEp?$5E8NHu(Ri6RnXmAnW-Deu4t1xu!>7}*4by4adTUAc zT)MRlE#J|e)_&cI0%N<Ip?lpZ zY@Y`mOYxy?sqfSFi^D1M)W@_d-G}yO%%y!7=h30dpHot%KPBD#g3g`tp_}Kt=niDe zn-^v(<;-+nx^rPJ$$;$95 z^)1&uAE zh|$HgVLaqX_fp7Wy%S|L`e+#qJo1!A z9eqY#XC9N^#V52l;|V2PT}p{pmp}$wiMWbR+z6tRH^WJBEl`pHAKVI}q8kyA1J~32 zTbrrq&Nh0Oy%iC%VD`845c1#76_;q&(ucHX#Ut9k`Vj@6d_obaPiPHfz{^=DiDzFW z4$UGyP)K~WTqEbbMcwE<8cCisi&jETjDnna26AE^g(`=@nLz#X# z!1UWyt(<59`YxcCgZ=~1yMX>v(60jhW+TX!HcW>{F`b&tlo`sDb%5!?RV{rz&^H0S z8R-9#oSN{RW_zeIVc$o45vT&e2I0Lb%2h>2P;KW>W&K%k`5SBpgm+c-`!DDq2KxE= ztv>x`58EDgs?zP-x5sD@(opTUbedlMhHnj07RIA~yQcoEsP}7JyUrVW*wIB0)!KRr z@bg>jrdPLaT|Jkjerr)5iRx_wOOYL1V)=X}un`C(p@#wDJjLw1;4dnNX+ z9g&+>&BK`e{MHa9jz$>Op<~w`b_bR9TQT@ZB)?p-4wr3FJ=w{~)U=&AQbZ7Qr=6US zM14eiZO4Vb11)b7Y=mqF2 z3dlgg0ku*&6*rx7#OG2Z`SU)wJju!WxrmEvTk}g?Sza0_BU_`>6c&rh5K)GxzNa|r z+)YrZ30>cKle*XHPD5&w z&)XEc%b6m-^Q68(yZj8hlTNIIH`&y-jWg3Yuo3kj`UQvwZS2QNHzh zLymScD<*g2$26x~9{KfpL zo=?p-6q0pvF`44+wPoBh>5P@(ZMA)D8QJVDlHOR~JYGtlq&%V#C;vrLPdz5zj2|g3 zb{oY;M&tZkM#(rob20r14wv-I{xD!RLkZr}ZyigTjr zNnSLq4tzkHzkEphS3RU1nb#=p`VX``^9fz!19XpZW{L|bLi=vGkB%hzcoxyT5Tc;{ zL~)r!r^<=0vJu^48_MBPRKT;TghS~u=zr3o@7A7ZWPhSrpI{f9N3=hPC==UoIpi(A zW<-3;hB$i^@x9r^rJ=avA0Ynes;2*Lrl4;P`tG220zLNAOte+*3HpUb#A|Gbw~Zp+ zHygr7DB3(gTzFMW4>@5Z?$ER9g1$cJq4&vb1o|eRml+c~*$__{Mf~|};*FuiM-C8Q zzpAB|PU)sNrCZ^Y?~GI47N@*3PWg9;jXox}nMFJ*5ciC&xMv*1J>xv?8M!+2GMw_Z zpdSYMX`ufM^g*Ei7W9Wee**O9{ukd{5L?VFYjjgcKUN_j(In3}@1f>SsEG1UMOb>~p`xM>Qr9MNC7&tlor6hu9e;WzY_?IiGjzy6obKuD64 zS8#$;QfQJxT5ys>Tv(!`?SIQqp+nw?$Q9zDzW)kMc98$4j{$RSSAlPxafBo|UJN}l zRR2GD3?Ye5TdLz>yDD`^CtQ}Bkw6{2Z&*)-$4?_j$ydRfGWTVV6S8%{fM_^~R+k?1(< z_j;1WmO5F)7CZs}tQ%X!l&Fux-)o}Edhq;d1-^C0fjcpOmpW{#lNdr0mFWWVV}iz20b+VtH%L5FSc#ii(v!AuBOF2%}7y#x!CAxo-<;a z$<4;<{w(A2>s!Q^1ft#Jq~%zKJ4tnpy6a=}oc+^_ySRAGSY?cU_jnSNwLBYp3|)pW|FhHO4Z( zzKcoj+-b&&%!y`-U&UD5OiLN|a23{$cJ6H%c6bciE_OG2P?2|?@882Dk2^QJUJg4d zwpfr{3 z^kUcuGHk~fHeprr?}~9??`7Ddas4_MxncbbHfVj7VGqU@OY)^X)cR2APgZ+MoBwsL zUu|Kts%o!kzvCq@h|otF_zvEVWY^n`!H-C8Mc=UTTE#` zg>4|#PHB~MFV>$Y_=nveLsz5WUp)>1VL!;Q4`gpcY5cc9v%k|*JHYNzb^bc?!mxfny1khiJk_u$qw;qz@?!14{*YnESYiIv{5J&ujS*q5 zh&ibA_cK+bO*2>grF%tKg3||T-d}+j%ldOY)09)hdf?X{d(9Huo$jc|BGwLUD!E=Y zb1%wam-#C1sc*$-9ep>G{NAocxuM>T z6`5}~y8at%O{MQvJ@0A=yHkceDZ>u5+VzKTA1`5lJcJ&xZL%)OzMuyFe+fO(zfDao z_i@QLHuukeWcJ}5hP|sahK6Df65ns*OAI?%hK(!3rnOrBaVAAp)kkj1w2{jOY@+fa?WF2P}J4ZI84@ptb#Ef1Cqmf^e-d%Y~Ggkj6dW@9myR#&+m z#?YqWRdzS8fp@WvOml_Os6x1N-ls`c`izr!CuZGv>zr3!PWsH1*UwMyv?00w*3hGJ z4oj9-lylYVCsFQGq0uTbRfW`hSK^s z%N0*U6#{Ig19kb}#qg&|GZFSI#-5pID!zb&tf>B6vQ=TQ7Tf)X7LRA{h@j4ZBnT$Q}5#)137aXjAY z>5_{3p4c;8P4gUUI?KRsnJ&+|X+_2y-WF}gpMihDyNG_FEtF%4uw>Z(^ShV~| zl&Cb0x}*MM>8x|U?XuE+;_s$;-&gYw9TmfO18c{;O8!MTd@)$ysc9<1`x#yjKaTvo z$ZS(G%f7vXyZ&RuCl$s5{-IU*dqQsJOha)FSgi&iZnPRa3e7(n)L?=)v%HVE8#Od>Xh_d^wxNKK{8y%)^#{ z5BC*Yi99ycWz}vQhEEMIUf)vt-GX=VJ&V7MSYwsvT%Esv*_YuP!=_8!7`9Z1u57t% zGU7y>T|Nxo4YvFG(5=X`8_X;A{P*O7k{qI>eI9!~Y_E9jo}RpHdwcfZV8!r3V)*jZ zxOVPc@@4oLvGEc$|GLnH;m^dYqg`H>U722rGk^Ep^D{^0UhtCiI)8Np;mQ44geE%u zq`5O-pI46sK1>|AyUT+sXWhTOoj&W4dcJibY`FL`)((88xcPEjzLn_~D1%=W!w-q` zvCh=AchkKd;yhiOpEes>!#^KFk{rfscvs&M)KKTlQe2$XT+AE#kIOpmTdL**dTNGm z6~h-ywRS3w!5C{w{A~*HH&WB+!aM0R_Cl3g;XL~zqru!KR?9!e5tQIGu5x>Mr#+3b z(|t$hUhv-a=(1b>#}QLs!rzT8m%F?&21ASs{$mW^t}6K#V-fRGxsTV)haPeGzB>Lx z4%wG)IP73lQ(H@q?y|AicH~$1RY~8f^1H@_*Rb08WN6Mmd7=B>V-U)v~hMytBkB|#4&1C$&s^vfH!i*`u z*H3lz;mJ;2#2akoF|1AM^Bi(;kLu{u<+7NPed_a7I(M<6p1u z6aD^OZCc|lUmgD~A`8o{cK$POgMSzQo#IOx;cfqBHDg%!clGm6wqqS#Y6E@TW!*6d z-JJD~Tw$m3UlCfyl(bRmSc-SZ;)>lUo@@PAF#dl`zSa!DRu7>w3^mnIp~|QFE6W4q z6(QVS1b!`&p9Fjfua*E6B!4N-DkxXMx+(~xInW8N1XQTtb3o9+h!hSqfaQSt0>bdF zmVo*J5>)7y3#gc4Ai-49K!v~;{yO@gN4dfP?ZY-TRSAVLE9lUyfT9}#1s`fy3wew~ z;_D>^Nc_Bnm7S7rF+$YqRfJp%RCrMza#R(k4He^zav2;wNNOR zWr)s*8apjRWd33hZ*d`sj_YvG+bZ7N|6+50m>m7j4^8a*;L9YZAMs|M5OG-E><>5c zCkgPL*QfHG5&3<8vbjG@2K%ehj*YNJ*$YF{7?p4)>C`11l75j(H!)v^QLI<uP8<)9CRUblfQ^t-mu zQFrALoqS8(n%!G8vC;hsZ<@^yKIkZmEO_}!|F7|u=5CVP(ZxXVvziC^cR}xxfd0Bw z(C6DW>B8_clXT`1u1W4`7bAu1)Moe1+DDcwkR5{VewVK9zTn^B=e2AOgiZ_d5RG;p z!8@eE2OQ5wQa&Mq`YHf4t8GOzdeq)jiL96|x+;VqO zW_5q7>dGe@JT~;{eR|wSBL@5k|3!vhqhv?Yt8o{G-{LE}TzBv>4mv8KHy64q_}DRg z>bNO*>9pqlit75QsXX+AL(h;C_AlY1$nb?^_)jwYDH;BK(1jIO7`xEz!H-X?KZf26 zK7S1VKOS2@>#1iG#dokhnKrfA9bJ3pn!2aZM2D@?cXkvy)RRhm@Y?;hrJir4c}y`V zJUr3h&X4N(7kXXz5;A-YHTr9jhL2$dz1>8!yAOrUiqK!y)H!tNje?S#9#!6R!cW_=7ZTjYb2ovPFoby;4x1b&VTA4-OACByepn*T9X>Ac07HO*Zmdb@pLW_R%>09pIl zoeRF#@b0nfhqGSR>yHkIgYT)NN7CWrVRf>y)!yQZVy(B@|A65y$%2)R#k)29 zY8n2(I(oG>92nP?zdiQu6m75C6!e#$YWmi^3l#jj<$F@+hPDUrulFE`&=@GJaXp+7;4 zh29o(U=>&Tip@ZEJc##E^gsUIrMdBm31c4X&b{D4%tcDlSiC7#)G3I!VE8>Ve6jI{ z7tGdMkFzW$pDiQHpLL4Muhzb%y$U z%6-Gnn=PO}5mBi}abx(YGyK3AzS)A;C-|;+;r+a4O=DKq1t&U;)cMw>J5C7R42f8NTtFzTq1W-GMv9ubhRRLCdfA)7N>o!<{;;a{tIb zTy?akftp7FOQ1_RnYO~k_N%bx@W030bF|E{#=OStpCw2mh8u=V9F)!}<%6)*<}wS**)f`oG}gs!Ly0TAVp}OFg0C zA;JDeaDrS`6|Qw`aj5c}$M>fC_RM$WK8s6Po z^LwcK|EHmKOmSR={$q+Bw~j3ONA#6djYZ4>cnFil-Y>L{DLO5SE>ioISB3tMl(mX0 z?t(d3ZWUYnNo5<6*XfYU@ob>-*=2>CpHEo~e_cO3g1kWNgGl%fO~@Lr6xMj#s3qPv zUgD+hQ7r`pAd-Hn9^wLcU&F1JI2952{-8rUcucWGbW+9_es$&iLOflF2>)F+sN}aR zBN3cyOW?JB!Ok)eZxK7icl%ldI!GZ$2lj^Fu4<2Ue64k>w6)GnoLu+erwe@NMZ)Ko zH>UOaE&N1pjqU~Kt_$Azlkt{6Sf@KFIn^ZRjPMbF&o0B~nBjxWUS9Htvj%-Ys~O3=Ha$^q3(@x#Zyv+ zApAzcPuYDBKW7d)AiwS2`pO4x4GJSB8fIt1KbhgL%&1-?XGN4@vF0M%}pW-1`&Zf6VYx=6;s9euOWxB;P=0*(vN##C|U9 zNp{9MJon$;hL^?kco}!D*`OS&- z^5*OJ{|rBFhMziY27GXj6lbnaq8djHLypQFLXsS}g(Wz=tuC{OeKg53?is`1o81l_ zURQS6)#XmQ?-uC#ln$(F#A={bbg8e{hpnQ@eiU**HE7iq3%vzw$kqzoUUf(>{M*xW p#8el=frz!*lR^&>|C6R(fHLAgN~HjOWnCcZ(-2=;Bg)&;{{WdaZv_AV literal 0 HcmV?d00001 diff --git a/static/js/app.js b/static/js/app.js index 3394543e7..c272b7662 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1141,6 +1141,24 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading 'title': 'Notification Token' } ] + }, + { + 'id': 'slack', + 'title': 'Slack Room Notification', + 'icon': 'slack-icon', + 'fields': [ + { + 'name': 'subdomain', + 'type': 'string', + 'title': 'Slack Subdomain' + }, + { + 'name': 'token', + 'type': 'string', + 'title': 'Token', + 'help_url': 'https://{subdomain}.slack.com/services/new/outgoing-webhook' + } + ] } ]; diff --git a/test/data/test.db b/test/data/test.db index a947e59643b1155787f4970e641fb95bf1b9aef7..ddfe7166b2af30d01966351fdc308ff0061f4e25 100644 GIT binary patch delta 6169 zcmds*eSB2anaAhl&dW_mCO{+z1O^NZ1a9WOzr+MHlVm3InwdM6Rk>7tF~IT)vBwXT5!9xRY|wqwf(fKlzjrC@=~?s zzs)CqB=F>xZr;_dQNF_shXCZ2GT;OLDoPQ7o^U`6ySrVFy-o z^O0k@V;hcR_L@_7=6rg9T@jU+aGdAt~-tul*|d`Mz4R){y=fL(PKr+XW%utgIo4F3hR5*Mz;l( z&V+%V?2A~E-6qr!{|it*~NZy%Z1K4QN7`q~5|@uHvM1VLhBUalqD?r#=+DY{+= zw)&IxOfwhpCxyC@)Fy^o$(DG4Nk)Circ@*rYYAtXGw6el%~r>4s-EPD&Un+_+xV$H#khH zEJLnt-5~bR8#ec}Z{9rM*QDs65bGnQex)u|KQK%Ussovp-cVVNr8k7)iRh4?iKltQoHQ@z z(ox^$rtrGpq*=eWDN3cJ z)?lMVrCXzI*JAmm1%|C0H#wb7yCGN;s3G0ijUN4rxt!~lJWRGVtJL>*uMcj> z^lxY#Oss992G_>|LL)z{N{RKUja0j^X}#)QOAn_SGm5YV6`wM5wNzMWXIheMo3AdG zVdxZ>iZ;<4SK{cy0tEl2>Shz`Z_8_dz)F3LEk!Mp2w|k z(_8#XC6ny$N_MhJaAUgNv$|5>w6Q%D=Z1V;UM9Fs3kNtQ(XcikgnX%nV1k}_`;^(^ zcI+;^$wd@iHBscYu!SR+TH4bZFr&cycbB%~Y;C_#Y3Z!J)CEWV9?PXmJn?If<)g|= zR;JNvH<`^wLB8ehQfK6Io!@9VgR)sJRiQ@n8Ir~FWrgVevPvB9u;6aTW3HvH^@Z=k z5%JT)k$?^IHCryVrm#;qf2(dI=kJ)-Pe7Ndn#Dhr82)9NE&S5%9lzD`QEW+)V#2}Z zOqvVPOgIu?rKWVypJD@ib2OQbH??t`L^5>(hrWN*vKTcUwKOeh2nV7h-P)9CYK&y+ zQh`j|+Z;}?NhX~Nhhl7ds*Q;Usf0KIF2D06SdwFDT~-NM6I6ob6&C*DC_<*QGAU9b z&$4vC*4JAltKDmQ2DO2~!QORR4^2`mL5c(|xJkB(oBFMkcv_UC@gnQbU1eVdyUlGK zcVkb^zd}+;QieHsT2%;`PbWm0)qu^2I>*Qo#qsFa3XB{{Y3gz}Oz9n3?VgIjP104Q zSVfU;f`p$%%iW6xdsnIQpth*Wz39Tx%sHdCe3}IBrau8y3baJ>oPn8fQ>z&_Y+k1> zGc2uY62Z@*M4Dh3MI=NOB9(Y8%Ze&52%L;YufiH;(C2^{mgB~2u~$4b=YF4>z*1`U z;uoKP$}P`iGFe>vO|Jc}e}{&-$A$KfV7ACt5Xw?+ZN)ze`L9Pjz7-pGjnIP5i=aM1 z@hVTS97z%qCFq1A(~>4hVpbJ2tHon+n#`>@Y*KlgNQ*+2*9cLUSrCKL2q`O);7?0* zR^((=V=Y#X`Lu)CWX{u}zIdJ&37yj@f|Y57kYxcJRF+fiu(ld4O`QB^EWV>m)hL$L zS(Xq;L7plEnoxL7CIp^Rvb@S>Wl2Kom)q*gWRaFRRRKFILDk9>MTlBfB4|m}A!f=1 zM4gI`R(OOIh9dJRYChI zZS%@F7L=5wsZP-~SOJcQW>qCZ5uk$^Seh)YqsdBJV;K(`ORB^XDn*L~t3$-HB5Q<9 zGOQ+0Ixmq~l=j#v%6K)a>6*+CjG!^VWCd7Xo*_h10ZkZL5kv{?^?;r{uke~AfS!WP zLGD?OkU2>Qm1IU1HBDA1ew?)3d+O>$iirwjB-RpZkC9ZouD&JZyz;lXVCQ79rg3E%t4`; zW62Dfy1;lkps|>v;$p_%#2m>PGld5PJ;*Nb9)HO3%>H1osf}Xc4#OsW2_{h&S~hjJK_IC3PTw^HUlO)y`<%E{>nX~Xslk0@*O*ji*bS*8m66W)fmYG%pKi_q{&w0(l z|DR6|X!~8xwOILYR}S8>%NZ$?RFxA&I93^nhHY8nIXI0dnxNq&LX$Whc4Zp%?Q)*L zD!%!T9jN1O=ct2U%l8_kW$33)yc$JzJFj(+<&Ut2eg#^O<27h#Nc0_c*sc0&CEEJ((mOJ)2Z^9}H z4tJpi4+7Kf&K^ZY`<*A){0H{DSpXNSnWkQo%T?$vxOUo;(~9!lc^+JMZgIThAnmob zCTyE^r{$z2Ztj3#wYPa^F88#8lgWh3l*Wi~u?3AKxSo)T0+$Y2B^aF&$!t~@1ztpb z55b)DT`R9c@&U+Wn*U!qP~xC-6e|!W{|Ei$pmPUS8vfZ%^xZ?w;|^}c2Qv&mI~8{| zdhTJ+qV)1t^HBT|U|6Of{SNYu1B1Hq_qQTy0vI#z82vT+#RM=ee|qlY=!QpuK|J_# zhvBcxo#?Md_dN!PlGo7ni2NQP_!)QIj21o)h{9@9J(~Um$XRo=rrap4K)<)blqa76 zIltCI+-c}b(QO!rxAVy{(aj9l!z2K^I;A9pPLkK;MR?>`;JxD62Ys_Jd1)((iO z*7rX`lmie&T}2IOp%V~uzOyBVUUmY)={tNi>cIg~aAN04!(V&4e(%Tw#JoRkIgH%- zfFP{;$7uRAKumw$Jqwu&0O358_b2pV0U(HV?{%VBAs}qaUcAfjSDmTX4K85JncMn} zJBsjRIituFC+HMGGb-HKrtT05MX3bEDgvo6f=*M4p|3jAP7_7=9&G7Ob{FcHjz5bD z^V(OSr86Lqs}4Uffac7^C#?_LZ$x>;_^6}6H2Z0zG;^kp{#cA(>savq*e!;hKGR3& zGC)z2zdVL&X8~&db}NnWlBt}E%5O&hTLL*PZ{GbbdVe;sDgu>FMrq5L{(0kaVAag) zJYeX_GySu14zOnJQVt;ZTwtvzf95cHZ!WO-2a^@(?RmheKK${2q8Cbml~*+Qp-~z` z?@mB3j?V{H^;1u+GxR7j9)(^^EdW+=N<2C4#_dHFPn%pFg@+1$FzvNz`T2|TZp2@Q za{bV;%#%?0 zPy3CYUUXtE6lgPp?{k!P?b>g2_n>#?!IwG~A9GaYJ=bGYbfcF_p{!{RUyHGIA1p?@ zI9SVy$DUn_YI$JV7VnsWmI{z(bwl@j^xpzLS!7*3an$JU{Op-{>a`hNA>m_K@!smk z(Q6VE#>|C3`5oF?1+e0)zwSWpEAit+SMGTvYBZ>MXP&87S#!003~H>#w^?hPI(oGl zpPXL4l~*qd6*e(Ipulh-A+PfG1wm3tEmf*ZE-s;cs$IE3w>rBZ4W9sza*lvwA#&| zlfRPtd-rp{zwiCsC3oB5W!nxfyLo{%xN~8#HTXcozHmC`ddlK@-*v+ElIy8MGu;!e zp{Lfbf!3!ju6JBdyFPFoht`K(hn{Y~#foLWc;kVghK@a$wfqBpSLaE5?yW^nn1M{> z5sB49E4p^zDW^L|%r+PLR3z36ebhCYDJCDZo2@g_EO9fJU1F|8zq;GW4ZYhvg4_OW z_fO5d4D_lDeP8Tp#&UC4^bQsF?!o5eMb8YK=sk&DzVOpu4*mP)Co|{Y|7xn)Mx3RP zyLo~eda-XbbN+1uZnM>SHn0+z_W}M${|IJ3blc9Mu7P1}+3hXw4Y{w|gBAQoZ`07> z>i~ONcyws^`ah<;G`7%ev!nc3Aok#vQ9NbimwU}t8_J#y4R>!H!IvDli8k}BC}R$> z`X{>vCw2{5v#u}nu_3W8P%A{MLqdHb7K{mO86T_&B++i%*muiN>x_ zPcPRW>hedr>s7y^$>EKPRL~f2?cTtLy;3M#g`;Gp7*;Bc}UsfHXyNg;G zj`1}0^+`R=n}rUoY;&d3e- zBj7a>J@M{DxVO5cq^qtjQq)q_RZ!Xy_O&$C@*UB}4UOpIGgfAh3Q-ZRh6#mBSuqjz zMq(9pCBCpA6&1&%Dv_&~YC?3hgb8`e zEKaA>=+wJ5^mOT+IgFA+GdX>7JRmEj~3Wh1Rqt?&%^~ed&Mzt>}t1-H9wz0Xd z-xI2BY2^D$yYxhJ&88~!@R!!*e7sGJhsxU%TAjz6=u|tK`@70`PmiV-HFlJ3+RU-F z1JU-T{uaK(W3>2NlF1IwhH^CLD=S|ZkPm*j+g*DnGTkVO}fyu;(AM$L;53b(vn{A(0t_Z|RLcSt@ASlyZ6(z*NL5>xpa?D@q z^HU*)j?{-@rJPWOe*d;@87g_(7G6PB#iCR+8mkU+j1a1;s}Z)#tqBwL%>y7jK^ z?)Hs(8$;0?Nl7FvyBT^FEw5r}k4y;?&yQ!?-*Dyp80-$WecX*bzUT@~;**@jlANX~ zBquTkDN(FQ@{C5S8qMe|tDzHDVboxCkDl*_A-z3~?nwo1T3W?2t2oh3Qt-1h-@UZE zeSK2t)|akwFTL<fwXTy5D=*GLhM+hv18dO+;Q+NuzPx9csAq%9$2|URu zGS4J+mEw$~-Bw_8TC8ybHmOt&W`$Cc3QT?yKI1fC@9^cYy2x;pM3a(08zd)-1}QTVlnj(DFUSThX$*QT-@)h5tjq|S zqL7NBf?igpNtqRrFpUDka*QsDqKp{Qu_lMoRW&J#EJ-s8#0{Q@(ZwWbu%aqKwya3Y zXoz$a<$z6UGC5fWsv?0^B16iWMnOsjCv&=kWQ8z>-318}x3Zv6WQc)QR(xOFVl1b>8O`J!?rQWA_Z+X}AGc~%v9s7*sv z!6ieXpj2Ttc|l_tO{7?zn|QoEC3@|*%NvA9tjvRZ?&nipJGY&YWRXW{$Ifr3MV=QX z{(3B>@4D}F;5G{i(d-4CK;vAbhH=luf_uZbe3ABvQ(=7M+QDx&}3(R#~7;b`N@SDuBbK)1+Z2$i7 zxd+(=y~ppzkBpTHF`19}qrr-*xKE(y@|BZ!OP(Zb&0X4t_Rh}2q~6=Ds4cojYi}*` zRR^nzBfetF>y3r0DyPrFS1hhqTpz+&c+{0U$4*+$N7`oFN#gvj1@VU{PvBbyYF&_;7CYnNe!+sx-xlqXqpJyH|+Klqr&BardWZ|(WbkcA7X1qZ+;my z{K`3uFF$&JmYJP{UT_ktQE0Do9X@m9)=y013e-UmYtX5^&U;e!U)gR3mJW?vzZxCC z+qnb7`bSx`exLIcwkqwh=TKxn5N(yM@1VkafQUV^c(<9o2tAMi#Qb}KDBZBT-88aM zbOsQgyVtoNr)QH7nSq69xlCt&JX`|&-&IS9IEcK4d-jzge3W!t|d%m6p3do_CULCD?q%3GhK@I#Y+{^TtWA62PA$QNz`y1N-2rz_ecm50&JPHh6T4_P< z-vUEidd(*&^D*aerZbj#!d#e_I{g}Fu@7z9%AzkGbFNES=ig!)xo9T_Cf@NlOpP$O zyVVS=7`~g*ZhCk8u(PXhIm1wfK*?~MNDAuPuTuP(q2FWx!ttlQCs5@KKzNGWH=2P}(}mjW0>+XvzZlz{NmMQ8Xxf1LFI-bO_;(5Z zl1K&ae566kGTf~-iWdyiSUyetH4@bq=u(w>p=-WEzR-1%&1TYSA;=1S-4?P>Y`oj3Qzl zplmy4jiJxxK~5g$eR=5p%Yh~6x%Wab^VxEf|0$~cn#VmguLPTfZQ+4(O15uTe;=(r2}U(9Zw zWh=EdSPol?c6#Sd1Sw*k6<_;E^r&fcOl#XqPx2OpA4KI0@g(kAwHr6v+R=v#pi-+@ zVie~aUaK%$+t8Z}p~cS;Blu-fKRnyEqUW+mN!jAJv?^UUm)Y0dBOpn2J|b7q;w z#%awr2r!1Ntt5_CijcqUipN6eV-fOSpcFiePD+q}`l6Z|bXI@T?I91cis5{YFG<3nKkhTY`l;d%)+zS63uKKn0l>7 zM-q_t+&e3B&Ak4p=W2AYo@mA{8_-T5_tnrZFMXYY4qr{2#Hf9@ccS5Iz{>?U-oM0b x>p|)Bz`29#z)SY^+dO7#H##%fa`&~syeez`Rx_^)ow^*d-L)Q=IoU64{Xb!6G%f%D From 7014e0b6622f47c980e9048adf3c6f4de5fb82ae Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 28 Aug 2014 16:14:19 -0400 Subject: [PATCH 08/49] Fix hipchat icon --- static/img/hipchat.png | Bin 2828 -> 3213 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/static/img/hipchat.png b/static/img/hipchat.png index 4b0500763a2546cb6a4f41f94d3b4cc0c14f27a3..afdef551e5717002495f3717ada5bb6aaa9ef079 100644 GIT binary patch literal 3213 zcmV;8407{{P)aa6mwv2O}tsBhDZ^Q4wL_A&WdgSpovGB>^EJAt4K$rMF72)$>PnA{>%8 zFo5RGJ*Q5eQ(gV5-{)7i?(g33ZB;lN=3l`yw`*XBOVWA#0nzUZ>10BJUZ?eoI;~`h ziT1Wx5Rc>+4xBjOci-`I0}mIL^eU;S?|9kaw0WdJs^9NJMulFZMKoy3<1Hq8N`j^A zuJp7c19G~2-7l-_L91DmZgvi;n{ln3`2_=4Z#pn>d;Y1RbrrSA_=2EQgCKxP1%gWX z1PF(fLzW>R10hgrG1|;Ed3R>)9{*7PwPT0mZ23V!)_#8I;RWw*nO$_eq#JTbM5l!) ztpd=V$Lf{KPFbHzQY30b1QH7Oj!ZL)yAOIQ-=(V6xE7!;}NIvlRq}1%r zWq+JGZt94ES-WmI$n^jEbl%ccoBsp9UnIr|YL&90-T-(TL5w0f#YRp>D%t7DbWUkU zdy55=Q3JAo!i5C&dKxT94#!P@3uw;vuzoJnyb8IUCKhv?8XiSZBg<>9=Z^!iCB)Pm|d*D%cS^A&8`{Vi*@ z=5y|3F=z!a7(gRHCcm2NP&sgEACe3#E@vM`)1ATJG*yIyl}%H zF3GFO81u@R%EGeFSWOTXz~us+o|m5;$?u;Yd0Y0Mm7nb4*~M$2)(sXTs03(mBG@eM zGwWZ@=$evn>4reEe?33{T;BuW3?c$Bpo-L0XH0iZ%*9EfeK*6AQS8siic%z7*f_ev8eYkqe1{| zeParkqMU{3mvo}#BESy?!340t?B zNH&P9+_>i%w>KbM1G4jA;lR?uvQ$Jpc>SoX78Xq%)n@9AnG^5Fme~m!8$s0JyHp>$ zcI(&UuL0Sx`^ablArUno`FMOpe{8X)HdQ=5H#!%*AtjP(1U}hyeDpOSUwl)P*PMRn zG(0-IcblsM9~st5*(3u&&}!IUP}0A_>D4uZl-AVS&X-^AicSr_0M-sx24r_>b0D3P zZ1l`bRzNgrq*`ZuK}kjTW{`6w)m_7lei4-l{6Vt1rJ>bmXp7dfTPFwvluX$l;MDo5 z9?c+CwG9dQ0+G5PM0#?3n-v){Q{%sXBjhkub+rl2AoY#@Si+IQO%SkJjBOUgVu|^_ z>I{R^9W*zC2q9VKipo^2YQqvySdOeXSC!HNfHB(OM$>dorq1DOvmEu#D+Cu6n2iQU zGe`%ku?j&GaV<5)<<)H#q@=>Gs4)?c_VHp>Gf3A|+c^vdiJ%M`H7ANMwK|Z?B~ez@ zaN8zYWp`3-ceeWdHw9&&)hMZoPD^ITq@re!PKoWRI;X@H6AXb~!=(#$juw@+IMDc4 zR+DzmRNndYt6L7SJHL<)LtkL~*6 z7XY;yd=hK^=WvUI*{m^8Q_rN|u3+@^x2URf-~9Uv%zX1xhD=-p7ivgLFt>Q!hMk9@ zx&d@r@CWIgnRdu(HcD54jLG}y#|UZ(ISe{If7<*Net+=VntH}K5_5Y1GM|35mz1H? z`Q5S&RM{Oj3=r^#cx%I6l79XYufDb(oL@H@%^h*{@zBU8CSlvM8*tvL0Yrv!fPY~iwJkT$Tk$F8C-181n z6U^xK`m28D4+g2WyU8ys;ct77vuXD+9A(v!6%IjcH2G_0(Yr^dYwo#v>pq^CvJ&D% z@CFc!(Qf;WCG9L`MHIgZWbt3NO?zq1>P57R6$u1Eui?yR3rJ74UF)y4P6ki@1BLko z5EBKc!5azNF(UDCF(g^dM4R-eWEGM(Kuv>#$|?u4%dM2I25m&0X(TQqi7(%Nk*qF> z*NWsGAMJ+Eps~6Rj0UKz=QmG}c=O%ac-ytVx^KQ8q1EhM@et2CPyk!kZ8|LSc}+V9@c{*t=OZ>oN4Y7PS!Wo3@-? zTMj~8jIvl!I;raOr5)PG$2eORA6ETt>?FJ)i9iS}(G=zv@Zj?+S_N4$Z4BqOF5)+j z-;Xig3?2zwKJbMU8Aq#D4vh-@vNCzM2Tin&F=KQ2_LF(M|DO|Eg_!c{I(BW`2eC$F zaT?sxg2^LZz7nEUUF+c&SFYXo(VkJ*VnvXl%E8ESd3?5X^8fkrx;>3-J5a=)BSjo7 zDxD&DHI|URN|{n1A^xIq*XZ#4^Lmk zrggg%fljZ5S{J(Z=7x%`i<9ClM(Mgi%B$+)y8mikp{vwx!)jF0f_fLmWE+2;Gl5?W zz3aB@>_^U&F?{NBs?U}y?}SExdJn|tq(dLf?b9nW?bHp6*yqb@5_85cI^ry=Ou!PO zyrgakLNfOZ&SBBBqv)62`IdsD44+Qs|y9(~w+K=dpzXej|}NqYWbnI@-mG7dj=|iUkCD|IwoUc)TJ;Q3)VEFKSB+ zd40Nbtf&mT-Jx`iYn{@_v4b{$wsi6%w-ux%yHx9lbvQk-F0bDh1|dct?j1ME9?F6X(|KU{WX-{~CGY7sHgH}v_zC@QV%rau|fxuu@DS>JA1_U!P)ICH}f z2697fgMFv+7XEqb%>9Q>_s1PjLVT1~De{_JG%Q191J=xxa{Cu=q_jzpCOdtuvh(^K z`M=oo)$x%hzq^zjcDY3YfDriyLZcgXN(XoF;7U`Q2D#EMT&lAjExed>vgBfRX+?cn zO})b=%PL{|V?!1Xy0hD%|2H5vY6AZv_V)n*$iOIFp?n^s00000NkvXXu0mjfByuHl literal 2828 zcmV+n3-k1eP) zKOSs*0qtXRxLtxMcqIZ!!E-8}Gw9UqEoz5fqvdi7$Io6lnsfPNeo;|*&2^jI7Ig|A`l5Yhae6# zT|Y3Xnl$czqDOAXG%(h`Ce8&UzxbjUAfNzE<}6!snN!pZzfxEDNm? z3=J8OBzQ&md>{Y|5E>K(N`?%`1d zZoMK>9~$yVa^k?-Vj21dnE}W-SIT!~{=6w8`|`O$a5R)kurwqes%$+6#sBxCSEITb zT3y+?mc`~snzFp~Y$3w*plNtTIGo}Ax-Wfx!c*giurw36 zjA|zg8c;N9?R1#o!j?CCMRjcv*^1VHCzqZ(nhR4f09fr^dWP>?|MDGuqR|{B_w^=f2N{Q4blg*ai%VFWSAbvz!1#gax1f z`mM2(hT*yuGJ#1`eX(rJjI}MonrZDlp7-g-445>aDb&~!hV;uxU#wI%dmo*`-gUDd zd~^gX4iFgzZPvb%A8yHP8qn+_Mn5#a^l~Zm8rWRH(V?Zgmupl(0XaL3{*Ns=f9e7Z zTDa1mO25*cyA`OYSf8PsYuoJB_Ho@ zm}=BooqiCCAkY+??q{bwa8FX7)>3sQF%ib)vnRvl0wGYU(spJ!J)(cAUhCkG`NjDq zP$}RMD5GJ;tVwNW>rCcP8xs~Ei@G|fc<;5E%^5%V$LqDWXP$y6fdSm!XC~Yo7OZVM zRVVZEqr>5mAOkGD?NE+?uwLswKNZy9VR-hbaf8}U()m0wb|4HIh(2)avAm*My9a)) zVsp)vvg_a(ctvA3GVw z{+g2Yw~*L37NQ86h9Kr%sf1svvbq*t5flX?^@%jKm!$KFk2ckRorqj%t%m2|DvLdE{HMGUa4ZM`&&79-fL|*%JhVqt2z(M0?6s?w(*?yX zmZiS6G3LHmpX@jin6cwzX6JVs{o05Z-fJ;lWp$jmR058HPf{4P{bIYJ;a7gxfZp(m zU>JDaoBn;W#mX?F4(6H{URnR>oR2E4PTV9iF#p3HBd4vvRdY;NeT&)c2T!7^7CZ;R zHz+>FsMnxDYjX1KpcxPuyy73bj|hUV>B_XZQ^JS$2gr`ChoeT%d1LMNN{bC0Koq2p zwjYjoaNf#yw!rRT;$ps7_Lmkh?|+jCf5q&R{&au+$-A)(3?DnU;%WsnL9jR$&UL1%Q?Z%9w&(2IkhzcH&)~GE9RvYx{W?HMh+&gd4MuY}K^g+R1 z_;y*J$grj>0AmoA9ZDnR?Rj@ka zlj8EeS&RVYtlPF`<9^gSp;CZlArlY+AF1y|P!w1iG!2@DB*X22l7Bko-VF<$R`4y4 zGoxm&JGAFVgy`V$B8actx2B6p=bvf}rGX7^q`)IVlweTjXXicq>UspQdiLbY87rQ9 zW{k$9hszCz7hVZ81&#s7fTbW1xIJ*XST#R6W%$qAmVEKIX@O2NSEe7@c@)7K5Q19g zvgs2VoDg_UeB|}Com;aI77P$n+9pm({&vlDw1(4FxA%BK)~SNi1tn$X8mHR}89_=# zgh_ix-{@fpy~hvl-_{*==9(?**Kdbj3!a6=k$BH-r#CHZ8W4DVmAg+|Se#RcP(37q z>$X7;Bpq1ys$Sdpr4B@tB*_QJ;91b@jamUj#v^k!>`FfblO75V)egRkzC2@cH4aT=*k_BT{(1CW_-&lJP%>`Tx~hw12+@J1QR9M+Kk?P# zf$=ddB3o~bTq!drrLM4-RKTDEO~dJiMD81rv|`4jp#ypZ#&?jYadXOYi(%4(W?*w5 zSe=#j+P!zh1xB{nNGh+kjeP#S3)y)v>cO)ReQTR1m#}P~+iGUhChpm39rKo7g-m^!9%ROq$p!IJ))jPrj;K@awr*A$4O%N!$VE59v?%i75uA^E2B!cw4 z$G_N$18~ zC)_@`@6VeTwYt|$Xf;q~wq>0tIDGcfnS$anbFJCxl0*R#0DzNenhy?xx+XiE z5CNJ1Eb5$0R5$mLbv)O2n*aT@8n}ARdOE-KTv2Ipd5yWoMhG?M>5=!}*|%xT|3RxG ev@C=EAp8%NkGu3nbEE_S0000 Date: Thu, 28 Aug 2014 18:39:35 -0400 Subject: [PATCH 09/49] Add migration for new notification kinds --- ...4a0c94399f38_add_new_notification_kinds.py | 50 +++++++++++++++++++ .../82297d834ad_add_us_west_location.py | 2 +- 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 data/migrations/versions/4a0c94399f38_add_new_notification_kinds.py diff --git a/data/migrations/versions/4a0c94399f38_add_new_notification_kinds.py b/data/migrations/versions/4a0c94399f38_add_new_notification_kinds.py new file mode 100644 index 000000000..91ed2dd08 --- /dev/null +++ b/data/migrations/versions/4a0c94399f38_add_new_notification_kinds.py @@ -0,0 +1,50 @@ +"""add new notification kinds + +Revision ID: 4a0c94399f38 +Revises: 82297d834ad +Create Date: 2014-08-28 16:17:01.898269 + +""" + +# revision identifiers, used by Alembic. +revision = '4a0c94399f38' +down_revision = '82297d834ad' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql +from data.model.sqlalchemybridge import gen_sqlalchemy_metadata +from data.database import all_models + + +def upgrade(): + schema = gen_sqlalchemy_metadata(all_models) + + op.bulk_insert(schema.tables['externalnotificationmethod'], + [ + {'id':4, 'name':'flowdock'}, + {'id':5, 'name':'hipchat'}, + {'id':6, 'name':'slack'}, + ]) + +def downgrade(): + schema = gen_sqlalchemy_metadata(all_models) + externalnotificationmethod = schema.tables['externalnotificationmethod'] + + op.execute( + (externalnotificationmethod.delete() + .where(externalnotificationmethod.c.name == op.inline_literal('flowdock'))) + + ) + + op.execute( + (externalnotificationmethod.delete() + .where(externalnotificationmethod.c.name == op.inline_literal('hipchat'))) + + ) + + op.execute( + (externalnotificationmethod.delete() + .where(externalnotificationmethod.c.name == op.inline_literal('slack'))) + + ) diff --git a/data/migrations/versions/82297d834ad_add_us_west_location.py b/data/migrations/versions/82297d834ad_add_us_west_location.py index 59eb1f800..1eb2e12ff 100644 --- a/data/migrations/versions/82297d834ad_add_us_west_location.py +++ b/data/migrations/versions/82297d834ad_add_us_west_location.py @@ -28,9 +28,9 @@ def upgrade(): def downgrade(): schema = gen_sqlalchemy_metadata(all_models) + imagestoragelocation = schema.tables['imagestoragelocation'] op.execute( (imagestoragelocation.delete() .where(imagestoragelocation.c.name == op.inline_literal('s3_us_west_1'))) - ) From 07aab4274c1a54543e93bbf1a1c176fd002d9831 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 28 Aug 2014 19:19:20 -0400 Subject: [PATCH 10/49] Fix parameters for logging the extra data needed --- workers/dockerfilebuild.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index d200a336e..8442237bd 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -224,7 +224,7 @@ class DockerfileBuildContext(object): logger.debug('Logging in with pull credentials: %s@%s', self._pull_credentials['username'], self._pull_credentials['registry']) - self._build_logger('Pulling base image: %s' % image_and_tag, { + self._build_logger('Pulling base image: %s' % image_and_tag, log_data = { 'phasestep': 'login', 'username': self._pull_credentials['username'], 'registry': self._pull_credentials['registry'] @@ -241,7 +241,7 @@ class DockerfileBuildContext(object): image_and_tag = ':'.join(image_and_tag_tuple) - self._build_logger('Pulling base image: %s' % image_and_tag, { + self._build_logger('Pulling base image: %s' % image_and_tag, log_data = { 'phasestep': 'pull', 'repo_url': image_and_tag }) From ce7e3a8733037a38fbb4e84cca76c98e57768a88 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Fri, 29 Aug 2014 13:16:32 -0400 Subject: [PATCH 11/49] Do not link against layers that are still marked as uploading, there is no guarantee that they will ever be completed and their ancestry may be incomplete. --- data/model/legacy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/model/legacy.py b/data/model/legacy.py index cc32b8979..52723bd11 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -1037,7 +1037,8 @@ def find_create_or_link_image(docker_image_id, repository, username, translation .join(Repository) .join(Visibility) .switch(Repository) - .join(RepositoryPermission, JOIN_LEFT_OUTER)) + .join(RepositoryPermission, JOIN_LEFT_OUTER) + .where(ImageStorage.uploading == False)) query = (_filter_to_repos_for_user(query, username) .where(Image.docker_image_id == docker_image_id)) From 584f6b9635c8d073292747ffcb2a16a8dd00f1df Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 29 Aug 2014 13:59:54 -0400 Subject: [PATCH 12/49] Add a spinner when a tag is being deleted --- static/js/controllers.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/static/js/controllers.js b/static/js/controllers.js index 41e1443ea..4d1c8484f 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -523,16 +523,24 @@ function RepoCtrl($scope, $sanitize, Restangular, ImageMetadataService, ApiServi $scope.deleteTag = function(tagName) { if (!$scope.repo.can_admin) { return; } - $('#confirmdeleteTagModal').modal('hide'); var params = { 'repository': namespace + '/' + name, 'tag': tagName }; + var errorHandler = ApiService.errorDisplay('Cannot delete tag', function() { + $('#confirmdeleteTagModal').modal('hide'); + $scope.deletingTag = false; + }); + + $scope.deletingTag = true; + ApiService.deleteFullTag(null, params).then(function() { loadViewInfo(); - }, ApiService.errorDisplay('Cannot delete tag')); + $('#confirmdeleteTagModal').modal('hide'); + $scope.deletingTag = false; + }, errorHandler); }; $scope.getImagesForTagBySize = function(tag) { From d1b2ff588af1beb02910b0f247c2ad30cd9e913b Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 29 Aug 2014 14:00:07 -0400 Subject: [PATCH 13/49] Add a spinner when a tag is being deleted --- static/partials/view-repo.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/static/partials/view-repo.html b/static/partials/view-repo.html index 4f588ccf2..e5f2cecc6 100644 --- a/static/partials/view-repo.html +++ b/static/partials/view-repo.html @@ -391,7 +391,10 @@ ?
    -
    +
    + +
    +
    Are you sure you want to delete tag {{ tagToDelete }} @@ -401,7 +404,7 @@ The following images and any other images not referenced by a tag will be deleted:
    -
    +
    From 2c20fca37e49720afbec7840023d84a31a47842d Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Fri, 29 Aug 2014 14:30:49 -0400 Subject: [PATCH 14/49] Fix sharing tests and add a test to ensure that uploading images are not shared. --- test/test_image_sharing.py | 86 ++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 37 deletions(-) diff --git a/test/test_image_sharing.py b/test/test_image_sharing.py index ab278f9f4..ef2458ccc 100644 --- a/test/test_image_sharing.py +++ b/test/test_image_sharing.py @@ -46,25 +46,30 @@ class TestImageSharing(unittest.TestCase): preferred = storage.preferred_locations[0] image = model.find_create_or_link_image(docker_image_id, repository_obj, username, {}, preferred) - return image.storage.id + image.storage.uploading = False + image.storage.save() + return image.storage - def assertSameStorage(self, docker_image_id, storage_id, repository=REPO, username=ADMIN_ACCESS_USER): - new_storage_id = self.createStorage(docker_image_id, repository, username) - self.assertEquals(storage_id, new_storage_id) + def assertSameStorage(self, docker_image_id, existing_storage, repository=REPO, + username=ADMIN_ACCESS_USER): + new_storage = self.createStorage(docker_image_id, repository, username) + self.assertEquals(existing_storage.id, new_storage.id) - def assertDifferentStorage(self, docker_image_id, storage_id, repository=REPO, username=ADMIN_ACCESS_USER): - new_storage_id = self.createStorage(docker_image_id, repository, username) - self.assertNotEquals(storage_id, new_storage_id) + def assertDifferentStorage(self, docker_image_id, existing_storage, repository=REPO, + username=ADMIN_ACCESS_USER): + new_storage = self.createStorage(docker_image_id, repository, username) + self.assertNotEquals(existing_storage.id, new_storage.id) def test_same_user(self): - """ The same user creates two images, each which should be shared in the same repo. This is a sanity check. """ + """ The same user creates two images, each which should be shared in the same repo. This is a + sanity check. """ # Create a reference to a new docker ID => new image. - first_storage_id = self.createStorage('first-image') + first_storage = self.createStorage('first-image') # Create a reference to the same docker ID => same image. - self.assertSameStorage('first-image', first_storage_id) + self.assertSameStorage('first-image', first_storage) # Create a reference to another new docker ID => new image. second_storage_id = self.createStorage('second-image') @@ -73,68 +78,68 @@ class TestImageSharing(unittest.TestCase): self.assertSameStorage('second-image', second_storage_id) # Make sure the images are different. - self.assertNotEquals(first_storage_id, second_storage_id) + self.assertNotEquals(first_storage, second_storage_id) def test_no_user_private_repo(self): """ If no user is specified (token case usually), then no sharing can occur on a private repo. """ # Create a reference to a new docker ID => new image. - first_storage_id = self.createStorage('the-image', username=None, repository=SHARED_REPO) + first_storage = self.createStorage('the-image', username=None, repository=SHARED_REPO) # Create a areference to the same docker ID, but since no username => new image. - self.assertDifferentStorage('the-image', first_storage_id, username=None, repository=RANDOM_REPO) + self.assertDifferentStorage('the-image', first_storage, username=None, repository=RANDOM_REPO) def test_no_user_public_repo(self): """ If no user is specified (token case usually), then no sharing can occur on a private repo except when the image is first public. """ # Create a reference to a new docker ID => new image. - first_storage_id = self.createStorage('the-image', username=None, repository=PUBLIC_REPO) + first_storage = self.createStorage('the-image', username=None, repository=PUBLIC_REPO) # Create a areference to the same docker ID. Since no username, we'd expect different but the first image is public so => shaed image. - self.assertSameStorage('the-image', first_storage_id, username=None, repository=RANDOM_REPO) + self.assertSameStorage('the-image', first_storage, username=None, repository=RANDOM_REPO) def test_different_user_same_repo(self): """ Two different users create the same image in the same repo. """ # Create a reference to a new docker ID under the first user => new image. - first_storage_id = self.createStorage('the-image', username=PUBLIC_USER, repository=SHARED_REPO) + first_storage = self.createStorage('the-image', username=PUBLIC_USER, repository=SHARED_REPO) # Create a reference to the *same* docker ID under the second user => same image. - self.assertSameStorage('the-image', first_storage_id, username=ADMIN_ACCESS_USER, repository=SHARED_REPO) + self.assertSameStorage('the-image', first_storage, username=ADMIN_ACCESS_USER, repository=SHARED_REPO) def test_different_repo_no_shared_access(self): """ Neither user has access to the other user's repository. """ # Create a reference to a new docker ID under the first user => new image. - first_storage_id = self.createStorage('the-image', username=RANDOM_USER, repository=RANDOM_REPO) + first_storage = self.createStorage('the-image', username=RANDOM_USER, repository=RANDOM_REPO) # Create a reference to the *same* docker ID under the second user => new image. second_storage_id = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=REPO) # Verify that the users do not share storage. - self.assertNotEquals(first_storage_id, second_storage_id) + self.assertNotEquals(first_storage, second_storage_id) def test_public_than_private(self): """ An image is created publicly then used privately, so it should be shared. """ # Create a reference to a new docker ID under the first user => new image. - first_storage_id = self.createStorage('the-image', username=PUBLIC_USER, repository=PUBLIC_REPO) + first_storage = self.createStorage('the-image', username=PUBLIC_USER, repository=PUBLIC_REPO) # Create a reference to the *same* docker ID under the second user => same image, since the first was public. - self.assertSameStorage('the-image', first_storage_id, username=ADMIN_ACCESS_USER, repository=REPO) + self.assertSameStorage('the-image', first_storage, username=ADMIN_ACCESS_USER, repository=REPO) def test_private_than_public(self): """ An image is created privately then used publicly, so it should *not* be shared. """ # Create a reference to a new docker ID under the first user => new image. - first_storage_id = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=REPO) + first_storage = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=REPO) # Create a reference to the *same* docker ID under the second user => new image, since the first was private. - self.assertDifferentStorage('the-image', first_storage_id, username=PUBLIC_USER, repository=PUBLIC_REPO) + self.assertDifferentStorage('the-image', first_storage, username=PUBLIC_USER, repository=PUBLIC_REPO) def test_different_repo_with_access(self): @@ -143,64 +148,71 @@ class TestImageSharing(unittest.TestCase): be shared since the user has access. """ # Create the image in the shared repo => new image. - first_storage_id = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=SHARED_REPO) + first_storage = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=SHARED_REPO) # Create the image in the other user's repo, but since the user (PUBLIC) still has access to the shared # repository, they should reuse the storage. - self.assertSameStorage('the-image', first_storage_id, username=PUBLIC_USER, repository=PUBLIC_REPO) + self.assertSameStorage('the-image', first_storage, username=PUBLIC_USER, repository=PUBLIC_REPO) def test_org_access(self): """ An image is accessible by being a member of the organization. """ # Create the new image under the org's repo => new image. - first_storage_id = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO) + first_storage = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO) # Create an image under the user's repo, but since the user has access to the organization => shared image. - self.assertSameStorage('the-image', first_storage_id, username=ADMIN_ACCESS_USER, repository=REPO) + self.assertSameStorage('the-image', first_storage, username=ADMIN_ACCESS_USER, repository=REPO) # Ensure that the user's robot does not have access, since it is not on the permissions list for the repo. - self.assertDifferentStorage('the-image', first_storage_id, username=ADMIN_ROBOT_USER, repository=SHARED_REPO) + self.assertDifferentStorage('the-image', first_storage, username=ADMIN_ROBOT_USER, repository=SHARED_REPO) def test_org_access_different_user(self): """ An image is accessible by being a member of the organization. """ # Create the new image under the org's repo => new image. - first_storage_id = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO) + first_storage = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO) # Create an image under a user's repo, but since the user has access to the organization => shared image. - self.assertSameStorage('the-image', first_storage_id, username=PUBLIC_USER, repository=PUBLIC_REPO) + self.assertSameStorage('the-image', first_storage, username=PUBLIC_USER, repository=PUBLIC_REPO) # Also verify for reader. - self.assertSameStorage('the-image', first_storage_id, username=READ_ACCESS_USER, repository=PUBLIC_REPO) + self.assertSameStorage('the-image', first_storage, username=READ_ACCESS_USER, repository=PUBLIC_REPO) def test_org_no_access(self): """ An image is not accessible if not a member of the organization. """ # Create the new image under the org's repo => new image. - first_storage_id = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO) + first_storage = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO) # Create an image under a user's repo. Since the user is not a member of the organization => new image. - self.assertDifferentStorage('the-image', first_storage_id, username=RANDOM_USER, repository=RANDOM_REPO) + self.assertDifferentStorage('the-image', first_storage, username=RANDOM_USER, repository=RANDOM_REPO) def test_org_not_team_member_with_access(self): """ An image is accessible to a user specifically listed as having permission on the org repo. """ # Create the new image under the org's repo => new image. - first_storage_id = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO) + first_storage = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO) # Create an image under a user's repo. Since the user has read access on that repo, they can see the image => shared image. - self.assertSameStorage('the-image', first_storage_id, username=OUTSIDE_ORG_USER, repository=OUTSIDE_ORG_REPO) + self.assertSameStorage('the-image', first_storage, username=OUTSIDE_ORG_USER, repository=OUTSIDE_ORG_REPO) def test_org_not_team_member_with_no_access(self): """ A user that has access to one org repo but not another and is not a team member. """ # Create the new image under the org's repo => new image. - first_storage_id = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ANOTHER_ORG_REPO) + first_storage = self.createStorage('the-image', username=ADMIN_ACCESS_USER, repository=ANOTHER_ORG_REPO) # Create an image under a user's repo. The user doesn't have access to the repo (ANOTHER_ORG_REPO) so => new image. - self.assertDifferentStorage('the-image', first_storage_id, username=OUTSIDE_ORG_USER, repository=OUTSIDE_ORG_REPO) + self.assertDifferentStorage('the-image', first_storage, username=OUTSIDE_ORG_USER, repository=OUTSIDE_ORG_REPO) + + def test_no_link_to_uploading(self): + still_uploading = self.createStorage('an-image', repository=PUBLIC_REPO) + still_uploading.uploading = True + still_uploading.save() + + self.assertDifferentStorage('an-image', still_uploading) From 417fec0b68617f3c93b224a6ddbcf30a55d2f5da Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 29 Aug 2014 15:46:43 -0400 Subject: [PATCH 15/49] Fix namespace selector bug from the landing page and make the namespace selector update the URL if need be --- static/js/app.js | 8 ++++++-- static/partials/landing-login.html | 2 +- static/partials/landing-normal.html | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/static/js/app.js b/static/js/app.js index 51434ce39..dfdb9a879 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1749,7 +1749,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading when('/repository/:namespace/:name/build', {templateUrl: '/static/partials/repo-build.html', controller:RepoBuildCtrl, reloadOnSearch: false}). when('/repository/:namespace/:name/build/:buildid/buildpack', {templateUrl: '/static/partials/build-package.html', controller:BuildPackageCtrl, reloadOnSearch: false}). when('/repository/', {title: 'Repositories', description: 'Public and private docker repositories list', - templateUrl: '/static/partials/repo-list.html', controller: RepoListCtrl}). + templateUrl: '/static/partials/repo-list.html', controller: RepoListCtrl, reloadOnSearch: false}). when('/user/', {title: 'Account Settings', description:'Account settings for ' + title, templateUrl: '/static/partials/user-admin.html', reloadOnSearch: false, controller: UserAdminCtrl}). when('/superuser/', {title: 'Superuser Admin Panel', description:'Admin panel for ' + title, templateUrl: '/static/partials/super-user.html', @@ -4037,7 +4037,7 @@ quayApp.directive('namespaceSelector', function () { 'namespace': '=namespace', 'requireCreate': '=requireCreate' }, - controller: function($scope, $element, $routeParams, CookieService) { + controller: function($scope, $element, $routeParams, $location, CookieService) { $scope.namespaces = {}; $scope.initialize = function(user) { @@ -4074,6 +4074,10 @@ quayApp.directive('namespaceSelector', function () { if (newNamespace) { CookieService.putPermanent('quay.namespace', newNamespace); + + if ($routeParams['namespace'] && $routeParams['namespace'] != newNamespace) { + $location.search({'namespace': newNamespace}); + } } }; diff --git a/static/partials/landing-login.html b/static/partials/landing-login.html index e2500815a..0a3046d2a 100644 --- a/static/partials/landing-login.html +++ b/static/partials/landing-login.html @@ -24,7 +24,7 @@ {{repository.namespace}}/{{repository.name}}
    - See All Repositories + See All Repositories
    diff --git a/static/partials/landing-normal.html b/static/partials/landing-normal.html index 8a0badad1..6b9b6e42e 100644 --- a/static/partials/landing-normal.html +++ b/static/partials/landing-normal.html @@ -34,7 +34,7 @@ {{repository.namespace}}/{{repository.name}}
    - See All Repositories + See All Repositories
    From 07c7cdd51d53a3b5009192bae3cd69b927f6f823 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 29 Aug 2014 16:25:11 -0400 Subject: [PATCH 16/49] Fix PingService when loading results from cache --- static/js/app.js | 53 +++++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/static/js/app.js b/static/js/app.js index dfdb9a879..f5c612c5f 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -441,6 +441,29 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading var pingService = {}; var pingCache = {}; + var invokeCallback = function($scope, pings, callback) { + if (pings[0] == -1) { + setTimeout(function() { + $scope.$apply(function() { + callback(-1, false, -1); + }); + }, 0); + return; + } + + var sum = 0; + for (var i = 0; i < pings.length; ++i) { + sum += pings[i]; + } + + // Report the average ping. + setTimeout(function() { + $scope.$apply(function() { + callback(Math.floor(sum / pings.length), true, pings.length); + }); + }, 0); + }; + var reportPingResult = function($scope, url, ping, callback) { // Lookup the cached ping data, if any. var cached = pingCache[url]; @@ -453,28 +476,15 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading // If an error occurred, report it and done. if (ping < 0) { cached['pings'] = [-1]; - setTimeout(function() { - $scope.$apply(function() { - callback(-1, false, -1); - }); - }, 0); + invokeCallback($scope, pings, callback); return; } // Otherwise, add the current ping and determine the average. cached['pings'].push(ping); - var sum = 0; - for (var i = 0; i < cached['pings'].length; ++i) { - sum += cached['pings'][i]; - } - - // Report the average ping. - setTimeout(function() { - $scope.$apply(function() { - callback(Math.floor(sum / cached['pings'].length), true, cached['pings'].length); - }); - }, 0); + // Invoke the callback. + invokeCallback($scope, cached['pings'], callback); // Schedule another check if we've done less than three. if (cached['pings'].length < 3) { @@ -510,12 +520,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading pingService.pingUrl = function($scope, url, callback) { if (pingCache[url]) { - cached = pingCache[url]; - setTimeout(function() { - $scope.$apply(function() { - callback(cached.result, cached.success); - }); - }, 0); + invokeCallback($scope, pingCache[url]['pings'], callback); return; } @@ -5401,7 +5406,9 @@ quayApp.directive('locationView', function () { $scope.getLocationTooltip = function(location, ping) { var tip = $scope.getLocationTitle(location) + '
    '; - if (ping < 0) { + if (ping == null) { + tip += '(Loading)'; + } else if (ping < 0) { tip += '
    Note: Could not contact server'; } else { tip += 'Estimated Ping: ' + (ping ? ping + 'ms' : '(Loading)'); From 066b3ed8f042701a0a5cbd394021f9f37b4dc7af Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 2 Sep 2014 14:26:35 -0400 Subject: [PATCH 17/49] Add client side handling of user login throttling --- static/directives/signin-form.html | 29 +++++++++++++++---------- static/js/app.js | 35 +++++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/static/directives/signin-form.html b/static/directives/signin-form.html index f56b8f8db..d8f822968 100644 --- a/static/directives/signin-form.html +++ b/static/directives/signin-form.html @@ -4,18 +4,25 @@ placeholder="Username or E-mail Address" ng-model="user.username" autofocus> - - - - - Sign In with GitHub - - +
    + Too many attempts have been made to login. Please try again in {{ tryAgainSoon }} seconds. +
    + + + + + + + + Sign In with GitHub + + +
    Invalid username or password.
    diff --git a/static/js/app.js b/static/js/app.js index f5c612c5f..5fb21205b 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -2245,7 +2245,10 @@ quayApp.directive('signinForm', function () { 'signInStarted': '&signInStarted', 'signedIn': '&signedIn' }, - controller: function($scope, $location, $timeout, ApiService, KeyService, UserService, CookieService, Features, Config) { + controller: function($scope, $location, $timeout, $interval, ApiService, KeyService, UserService, CookieService, Features, Config) { + $scope.tryAgainSoon = 0; + $scope.tryAgainInterval = null; + $scope.showGithub = function() { if (!Features.GITHUB_LOGIN) { return; } @@ -2275,7 +2278,15 @@ quayApp.directive('signinForm', function () { } }; + $scope.$on('$destroy', function() { + if ($scope.tryAgainInterval) { + $interval.cancel($scope.tryAgainInterval); + } + }); + $scope.signin = function() { + if ($scope.tryAgainSoon > 0) { return; } + $scope.markStarted(); ApiService.signinUser($scope.user).then(function() { @@ -2298,8 +2309,26 @@ quayApp.directive('signinForm', function () { $location.path($scope.redirectUrl ? $scope.redirectUrl : '/'); }, 500); }, function(result) { - $scope.needsEmailVerification = result.data.needsEmailVerification; - $scope.invalidCredentials = result.data.invalidCredentials; + if (result.status == 429 /* try again later */) { + $scope.tryAgainSoon = result.headers('Retry-After'); + + // Cancel any existing interval. + if ($scope.tryAgainInterval) { + $interval.cancel($scope.tryAgainInterval); + } + + // Setup a new interval. + $scope.tryAgainInterval = $interval(function() { + $scope.tryAgainSoon--; + if ($scope.tryAgainSoon <= 0) { + $scope.tryAgainInterval = null; + $scope.tryAgainSoon = 0; + } + }, 1000, $scope.tryAgainSoon); + } else { + $scope.needsEmailVerification = result.data.needsEmailVerification; + $scope.invalidCredentials = result.data.invalidCredentials; + } }); }; } From 2dcdd7ba5b93d6deece46760c5b219f21be264fe Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Tue, 2 Sep 2014 15:27:05 -0400 Subject: [PATCH 18/49] Add exponential backoff of login attempts. --- data/database.py | 2 ++ data/model/legacy.py | 32 +++++++++++++++++++++++++++++++- endpoints/api/__init__.py | 8 ++++++++ test/data/test.db | Bin 614400 -> 231424 bytes util/backoff.py | 5 +++++ 5 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 util/backoff.py diff --git a/data/database.py b/data/database.py index 349ad1b58..69932273d 100644 --- a/data/database.py +++ b/data/database.py @@ -76,6 +76,8 @@ class User(BaseModel): organization = BooleanField(default=False, index=True) robot = BooleanField(default=False, index=True) invoice_email = BooleanField(default=False) + invalid_login_attempts = IntegerField(default=0) + last_invalid_login = DateTimeField(default=datetime.utcnow) class TeamRole(BaseModel): diff --git a/data/model/legacy.py b/data/model/legacy.py index 52723bd11..2e703b003 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -1,12 +1,17 @@ import bcrypt import logging -import datetime import dateutil.parser import json +from datetime import datetime, timedelta + from data.database import * from util.validation import * from util.names import format_robot_username +from util.backoff import exponential_backoff + + +EXPONENTIAL_BACKOFF_SCALE = timedelta(seconds=1) logger = logging.getLogger(__name__) @@ -68,6 +73,12 @@ class TooManyUsersException(DataModelException): pass +class TooManyLoginAttemptsException(Exception): + def __init__(self, message, retry_after): + super(TooManyLoginAttemptsException, self).__init__(message) + self.retry_after = retry_after + + def is_create_user_allowed(): return True @@ -551,11 +562,30 @@ def verify_user(username_or_email, password): except User.DoesNotExist: return None + now = datetime.utcnow() + + if fetched.invalid_login_attempts > 0: + can_retry_at = exponential_backoff(fetched.invalid_login_attempts, EXPONENTIAL_BACKOFF_SCALE, + fetched.last_invalid_login) + + if can_retry_at > now: + retry_after = can_retry_at - now + raise TooManyLoginAttemptsException('Too many login attempts.', retry_after.total_seconds()) + if (fetched.password_hash and bcrypt.hashpw(password, fetched.password_hash) == fetched.password_hash): + + if fetched.invalid_login_attempts > 0: + fetched.invalid_login_attempts = 0 + fetched.save() + return fetched + fetched.invalid_login_attempts += 1 + fetched.last_invalid_login = now + fetched.save() + # We weren't able to authorize the user return None diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index e8dab28dc..854c3cad1 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -87,6 +87,14 @@ def handle_api_error(error): return response +@api_bp.app_errorhandler(model.TooManyLoginAttemptsException) +@crossdomain(origin='*', headers=['Authorization', 'Content-Type']) +def handle_too_many_login_attempts(error): + response = make_response('Too many login attempts', 429) + response.headers['Retry-After'] = int(error.retry_after) + return response + + def resource(*urls, **kwargs): def wrapper(api_resource): if not api_resource: diff --git a/test/data/test.db b/test/data/test.db index 34882e11701b07d1a0e1ea1631cdc6636489f033..3e631b5d72e5c2c290e95affc3a9794853a6a573 100644 GIT binary patch delta 19857 zcmc(H2Ygh;_VArKcX#hiA)$m2l8_}N1PGh$yLTah^g?Z6GWq9`H=7E}}jd)M#Wy9)$F-ur)k-|wFv%$YlV&Y3xL=A1LT zXj|f<#l5tZi2$!zaavbDw9QVNzZD#hzRE5!Kn1>F}d;%Z9Q8)x|z-zD@{ta8!diF?mcawS;4ZiwZiegOT9~)FxAraCb@xJS z;r6*0+}wu2^-UObO~YWm2ZK5F7_?2nVCEzYrqy83P>I3SWf;^HVo;HbL188a*{K+e zv0>maVqn!_U>t!#auNm!@fgI$V9-ATgRpQ6f`Y;@6x5*@68M&bZ*fqcgYVn8a20&V z!FTw13Vh$`!$M#^2VcSa@H$Rvb8o`~;9Ea-LI^z0!Q(jT1F#KK(;o!iN~*qW}Q)lgB(`=91ddAPVgOUN$w|is^)OGHf}$eL^X5`)cSgB&~;SC^N~&r-fhL;&`b>8kdghm3xn6DV6gWp z3|^gp!R{&ycFB0&S%ks%d<K4`K#r-4s6 z_y#_~gm=OB$-_~-WPB3%8AI-E_!-mBLf2dFApBfFv>g0`3;8V4?jv{)m+=936?PzH z{{>GXl~=(dxPO<$l-(2>rPI-9oA>C{E9rW0rd zEuy(JosOawnnH)scp5|d({LI@dGZ_ifqX?yk&nr7a+thHUL(86OXNB76xm4DkVnZw z^m^ zkzxE3?uI+z7GxI-VGgvx3~)mo)Iuc`Lmp(nXt07FhQknug-GZP!JwkQ(;w;A^fdj1 zzE9t!2kAb#o4!n+r`>cDT}xNchiTV+)JJcpf1_TyfX<>Vw2?N@$+U)+(?Xg<$5ID1 z<3dcLgK0F4pkY*`Kz=3PlP}5Vq-W>q%A%jC6{T|lX5A3}#k|e@0BC_P`hz$hMA!U{<`Iw2%S*%u&nu5cb}BTwzh&!yhLj z2*E-d$N=9WniM0CA_#IWFw*sSA}m6(PJ$fle>kulJ=qosxfht~Ot=cXa4+uaTr4pH zSngP|mGBt<3GOK)sw+nq6TXjz1aTyY%pqUXdGrLBU=7ab8{dAMP_#U>3|90KF_4&Z z!xY&=FOjRsApdtXaj=q?Nq-W@+%Mxawf<>t+lE|dF#Ba2SwQ01+!u*iEw5@WFtEi$ zCBla?W>3L~IM#!}K75$O_L9p0e8j=`$YD-n##_KX+Df!R;HSf93N>Xq93w+TI4&3b z9FAkb1Hg`-A=;k&>(Ri1=)YVeBUa6AL#|9~>1O&16v0}Y<+=ga_m9;?MnsUC!aAMx zGu^W~-0kxJA>F6{O>`t`q&B{>dA4(AqpNOa%hbl^I%j96yQ#IagO#r${d+q*I_oaV z_mx9mHgE^gl0-iq%65=+GL$XZLGnq``6LzlW(ThLVRCw3ma~)Ok>URM(w!t;D`O!9 z%pC6KVlX3rK89R&Ep)(}$X~C6+i|*mftkbTfoRw+^W?AKH1de|;2kWn4^i?dyaL;? z)Cyo5yHq^e9Y<56=~xbQ@CfeT8{tOy6j|>3Fc37*LHA(MILgMRP*qocWsal9ZYefp z6Vs)Oi zs3s$?%27Shm{qH>jWd`Fq)PTm3Z0g2G+K*mswS2@Y<63QxhTIpBfBiWGP|gA!wh_;;&yVTtV`+LcEaTt=N;dDpTgb0pg`Mz{BHDchYq69U*s@aU3Z{oR zcnZ(PczOs^chRnVUX4PeH#CI)Rs83JzolUMKcGR^#Y70uAfNi1QqjLkhkW91iPFo{ zBK5x~hP;ziTnopwko9=vKL%I%dTS@Mmv)k5cKQ_}bbq@OcN`hXY`e)qqGM0)Cby6j z|IW+aO`_P$SBRJBdnCl{A@*>2%kgmX{E>@fc?Xn3h<(vQBZDu^@@b#v+37AiAeg?- z;mP|9Zl>W_>2=zbr$R!Kzf4Fz(9ZvWj0zRI&hMoF86nWnUnL^Vl_w_t2?@_r(D)zs zlONZJwjc3MbQ0c|(FW}OhiSsO!%99L$U_`?i2eWzFncj!m3vS;Fm&Iz2ag_N@{{k! zS4pedoK=yNQN_-_il^1Bdr9&|=S+_Sa6eUPD2l(=Q$_rx?Bn-n#IRhOy&$Ke#4Z&V z<ayBWxk^SaMX(?-9EqXFVJgn07hzlW5Zp>7aMMasbs92usAebSztNZs*KE{95&bi1K2hR`0jCb7)nq& zYLj3&$&^7N8|8ozfm!wFwFSg;E4#;n>=BjMCfv`#3vfRU{0?CEZvao=s)1MdLX?_M z(^PsL01nqa;5xfq5n662SA>>Mx3fw9pVfVh1M&&UVcSQ+)g+fCjD|vz$67~2HOcpH znIog2ffTUxRCp(_!v3(HwnB(TzT4wf4+ry^f0g~T4}vxDnJlq=hNC9l=t+L$nBI0~be3`#|BU;epASInK=elK{B`MUh*5D2A}8 zG#Ej~v2|$>#U`gg2`Q4}t)$pLfq$hzAC^4^YDh^pO4u&KjF0dsq?Ap0gl{8dGKglU zAK^y?WXVgqj^iUl_*~vXU&H4(oTI=#*Yg84bR!4z=tfL=l(LOu_|PE^$!u|%OuBlz zr$J|Nx=lJ~z1yL)TMTBi+2b(OJ0*5}3~%VFn&lp;#hp^=Zq>>gM{9J9GTBC1tXiD` z|Kd@G>};uXIXm6)qqOmVC~Y!aRC-~Ly=S!gvvBV@91fe8Cn7hVh+b8OP+#w|+3aSQ z&N9&CFzYNPr(GvW4U*1=<7%)w4K};W#8!{xv$V=K?$H!7H=Cci$od>bMn1<%N_iyx z_cF1*M@)PkkRE@oooMl&>FxCPYIWu%i%c84yNgkbKvzlC1vr}@pJQVaqsX*{*<}ft4-A1=r=kS=3)RG%3*f3@?m@W25 zd1;s&5`auofG=h0eW`5S4`~-<_!1}ap|ZPU0eEClSVR=%d8)X* zsOKUJ=n%RdMj?@&Wp6yj=aXtb-p8!uZy^)T(?DeFS0c53TFK8Kwf@wbSMk%xL^=Kt zndDDuSdAnuTgk_=PgnDU0-F;DE9*sNHPeTI#rEw!K3s$|0YMBi-uE-0uaEJ4l=L%! z7mi`^Lv@EiH3n#G&+~g<2R51$w`bJYgSj}m{AVfos8Ovl?|K8*XT~!&>j~I9S_ARd#~Q<2KsdxWPSU$*Qy1Y!02% zVsPp#o_be<&0;sYYzFquNR>6gCb>**6Rr=N%Ye(p)6k%koOZj;BRM?yXRz5U5;N&k zX$c;w0dp-*oz-N+-X*(7=WsL_bxyP7^f((Nsa~=(pH7uEL~3a87+lzp#fD3^!EBM& zxJBo38YPdV!7aJWR`#P#m7HL~s!}~JZWorqCT&)oRB!X>Tm}Qq&SSLL><*TnqRL3H zS*<3gJVH-{-HoD&(WR@GJT{$M^3)qG^$i}U&B@lMsA3WtEN+LV!DH81U6S0U$D?!F zJrZAP7~!G$H=4cMm5V%3>k4pTkO(d>3PSgl?) zBEjJ{*(`30S?4yGao0*t6pXBr9oai#$m%vbjEMM^dX+WP;BcAEMns~)V8nyKh?`up zI1%xZQF2%%v&mRrFYl{h*v6sK;+)E^hk6uO;3L4cjaB7D0nb4+NT>{b0k4HFv;**{ zCS9y3U_Ykae5s0%!7xKsYkYzC$iv8OcEEG+1gu7`<3qh_7iz0#V1qfpX4I+f4v3dY z97&?@K?`O?5tg)`x08BhxiZ}ws$Oi>9x|kR)p~vwAuj*I4Lzs|W*e8{w$0zbdx*P7 z>VXZsDLxFgD5tLww)t$h$w5ZAk1D>6DVqn?#;Rxy}55S9P(s&hB zJ0IidpM@=0_Eu$kj~@)HiI37?m`K z@XG%R4(mBY(BruC?}dl4`*vXWZcsH1h9#V$r#l4)aUv^W2_oww*o%sQpRn|HrR)q~ zOSY;y&q<4@o(thQ2ij}sQ2G=&eW(Lu?t`k)WJWh~qb~_*lBbZ!@OTVjdk(9ESo0xO zI6LyTs+2VQvxGycPyD@rf@JCtxQ~PT5HYxZ z?$fJ7+a1@Wvw@B6rRzq%$v8vosQ&gT+S(#_FmlafJ zNF|_``cwcYr{p_(LNbcGG-F)X8Y84BA9hactuv*g{dQ`Q9lDTZt zyXsguSzrr}`H3CQx{jPHI^``j&mz)5RJT z5-EqBjl(_sPez_>-5($miIJElhpjXjT(1FDgh~N!v)I{bbQx<+i z9m&SLi=+sDR~<$QS1FYrSQYSKjJU|)KFU#Umc z(UiN%UjRQs*mXzMNdfd+Jmat8G<`3J1@>K-CQO7|QDJ~*;Rc-dHNb8itC`(X?f-$Y zyw5_P_k)$i%%Nd?65Wg@u-lQ2g~AQvn>CTKv6O8(s*a@G-_RDmZq&u6Y0Rjfes;&a<{Q`hma$0 zvX~I~7l%gCbF#oi<~tnp{Q+6`g*r`MG`z^+v9SZsO#c}wy9k`Zqv0*sgGHACd$C26 z9WC=PIsaJ{mp{id@EGd)-@wCTC%lNI{)w`19Is;Ax->a|qNrAmZY+u>f$eu{M)#Ld zqv0lU`0ov3CX9Z7w%Pl9y|vkzIqjOrG@RDo9YPZ2{_e-Z9*1B`y1GLhsf(pujm<9i zTr`Ndn>*X*)y=@qx<=QzZ|)9%jZve7_~G@Pnn-;t^}@wfN^DDh#CD5|*4Q*fo>p z;P^1HF;&2&b{D%bRd9&zxk%JV>J4ToX~F<@ELBiZ&c}k@P>&4|>lqHMtbY6IRy>IP z_SR=)@*%?R^GhSoKdi}*rnf7XV0aV_C?2%14nu9%XLJu*sb!l?6ICw8T|N1VBgJ4f zpr9Bs2r|eY52|z7!B@@>x^lL7QFfO}8@TPCYhAb~{}0)k3)zy+Vlu+58&}yD{Y6LwYGrHoY$PvO&VEf$Vo? zW0C#SG1zl820LUInHObmo#$mYji+UIm?wv0uwgI;t0OU36(U=aDXhVeJAlIH8f@u7 z;QytC9OivRGced@)~93MGb3R#Grp=B-ZfN~4F1>LB$jsl@488l^KKF+XcrlB?oMzT z$?-ng!ww<}ccXCpJaoe*v}Ue=hXMB%x;y<14cH4{7JA_{LIX^O8YqWC$bqroK5KGPx`l3_tLbw3AZ2t(7rl+%NW0LW z-9ek^RO+OY=y+O63uqQiqjqYf$uyD1(J0!NhN5Q?AwQFE$r*Bzd`OOxx5?{d57~}( z^k>LsvYxCY%g8^;-Q-R*C0&nJr8%UH%s}&h9jPUiq!_KY8EEacqEdf2&RhQT_19`; z?RLN~&;8IS4t-%1Xq19*gW_;YI$~D=fW0 z%n!!2fBOwML2S_hAzf90DO*|i*Md^M`=0Opkxgva8GmdYTlBdh&i-ESAt$bIK;L_uu#SfMu)B*9!Q_mf2M;`kNbhTg`T(DUe~vJ^>tJziR; zq7zUtq$5ESkd$g9;%Df7@*2ANJWd}%ZmTkiNVHSq(~?gi-{9XmP3?()&Aj(m2a^qy{(4j@F2QmEP>nb1n7c! zXp?WksX5U>V?2~X0c1fM*ueHxmQL;O8Fz&D_j zunQe{u0^-0N3pSKxcn<{=}XwaAn1#D`WgM4UP70JQuJ~%qJL9g%A-E%7=0aGbr`(` zJ#=JcTH@ABBBOlNLg||5riQdLZ?Ez~)(2+~$MEinHymag`vT09qnctlIQJFs;lc;z22&?`^ zx*||d>uhLfnbq7`*D%f5Jk?#-&_JVM^rfk-&W?^bE$uG6Je$Q!YP8ODHur_(3sqX0 zXS>@w`2k=hPM6E?t%P;l9UY2wu1^%${TaSne~G9)U!>LD-qhI9f$mJbWF(zWpX08d z*3vQ~QBG7O{5n@l!wh%3r*Wn`Vjwt3VC0Hd)P+XEe4BoYj_eOc($MZkd9BXb+B!4f zFE!Y=^jEE~?e~7>Kwf)GeM_gR7YsgExYIc`SgsrJxav`Nh;PcT+VC(KaxPEqH^_I* zZ!rNPNF0@DC7nU9N3Xx-=$y76T?2oCaNHC5a5cJ3%tyDkXHd-~>vjd|d-b}n=wb+|k0cp4SwQP) zF?#qV($HjdpFD_%;KS&$SBL&f1`4TZAoM_}koY>OuGik@X8F&;teu zgZ`xXr=kP*j0GIbM6bgZ@(r2A{lLi{lkTns=$19us~RqF0{&&pl2<)?=umK2A@gvRg%>MnopqQCccXS1uNN%s8;^gef9iKnr( zeom%ow#jDC%g)R+Nwa32)7vuJnjN)ySvjWpolORy1Z5HJ8pa7Pif)o>A3WIj_;$*f1???(Fv2#q%p>7)p$m z+`{IT|1ehz>ewwdFM&`l%BxBXbly6v9@)q_EN43^Yi8QYtDW--yUY{kPHZ#I$e8Q) z%y8y5>z(5>3OqHn_TpKyrnaP%Stc5%Ia}LoQqy#2LrZ;1Q%lD*ck_&v^r=nG#+iRc zZlhhYI?QuuD5txC?{jBfva3_!T#8##*I)1vyBz9`HiyY>^7ba7T&!1R6^+U=Ki72t z0rT92h9bLElR10ph{haidHu}JsdF1ki(IbEf(m!RyfS-k!^|A(__@`N*+quJSp_AY zwsEuA;or&7k$>#eoZ>JVC6hrB`N(q|%GN>jDLRTghdXi&>AD8>i}nj>sGl{jd8V^{ zsyn&aji=L81s8wS1MQEaLgx{qK{8A8d!a_ge4%?*9!C3i{ZAd7F75N|WEk$uE8-xCYhYeG*S(tA<)5!)9NmS4M0 zqb<2@i+2QJq5Z^EcGttg^!~l)KU?Q-@Gu9FbPjoiJIu+Rd&T5h4CjYN1Y|TYR2_Fq%sy zbI)`Almkf~>Iz@_><9S7lDo=$$7i45lOFtfu5b68?Yy{c{aW8$b5AE|)8^I7^+UK4 z*%z}B2BIJQLaImi_1j4X_ciwjSAy-NbeFs;JOzV;IW+yRh6%uamhyuP9v4o>j}4pMTP8^m=jBjAP)UD7#aa#d9p7r?awjm+x+G&D z@nbr5i>B+`3@(dl@aD#_bywj6+_Flzi68d&Yxc0QtA*42fXVz0W>_Pf;*&n?RnN3* zh1EkwG{tKE8^me+H*uczJ}RVlhn*6l&W+(uuvoSN`!Vk1uVLwr39B`O+P59{#tvee zLUCLpR|=CgVRy6~@Ooldc`wAr-j%|Nm@z-RqP^0>25)QvdtJZ=iXOoRl20z{<@F3< zZ6Y>s`VnD;CLF$cz?&QITX#(=JGdO@KIGlu@3Oi_h0}(Fd1Ytiwgs*eO>m&cn`H$q z_(tw5Cdki{F8=eQdgCU|-_K3+&rm(g_v)NfcH;}!8@c1rvn*^YVtUA#+Hlso6fqF` z{4F7D$}(Yf|DhW{`P4tVKf##xSu8(Nd%PbDe-1Hbi|_TcH+BU35^%!5JckobijN)X z^`N^vj}w0Id11wn@RQS)|2NcQt!ISA?CIY%`KDi`4mnMx zkY3y~oNQ|?5UY(EJg)Gm6+3eZDZcKgP(`~R--rq$<&jP*873Q=^H2v=iMpUV)CtWX zZKxkwhW!A6{%Aewk)Av?~w!Ju~sVkZ66K+K|V48&~u8G)Fj zZ}P{C27OB)X41C>VitWzAm+5`X9c2?evSYU*rBBF>+f088y)(7J>uxgEssb^kM0rI z_lR5M8)Bf0$)Jy5q5DutZq^TAufL|zMcDN*y&>s*0lVHL>0{ZQuW2Iim6ci61^46Q z@Omi6BJoAcnAfnjtTx8)SU!FT6v{CzKV&GB%dsJR!f>dOV?#$L$}2gF$bMUBmhq4W z$U$5q<#-c5Oi$9E@fJJ`6->G4K{5gTN19Ltbu-+L>tQE+fD2+bKKE*n8yd+cnV?aQ z>G@$+Xpv(kez*j)7K>gdKjKiny70?l)u;48 zh{diq^q4+=S`eACav^{*e~QJSH_12hOB1aIy}3tgR+HX(@uy8$L{`08rfI(3s<-Ji z3T@<=T`wpEwp#U)KhdhU+Vl>k2~5Pe)vh0guh z*tM@{1}SXdPYcm=9VXtyCmI@eu-1(z0fc=$MG?w^xeWAYhYWfYUwe)q z?;2UW$!-wld$SPAnXgk#z(BaEtK8CBf?TwZ1~lio2#6#PpQPe!KSR!EL|<0 zwi;eBel8C^gfq$y0&7u?vxRzSKRjqUNCJ0&TZkE#{&goG5cd5i>=nst`|h_8dHr_m z8qTH=MBc#LzRzLh6p`U5Ngly60Fh_g+cNTs$U7W^<)go;FK4gBimNs1>vQoqFJZ^K zg;e&`AaSzBx={Su>nUc7o)*$rO`NzQCZ%{pw7<>Eim zWUcG4fjI-k6@1jA4{TxDXmL9~nB4FaI~gsW9uofLlo$T1Rek(7;W(4N5pvlJzYCdc zrYtRJ^F_u=^Td4Coi8qCnmjS%f`jKKd~q9uFO_T1WN|Mp)4eFD$_}30@9M-vD%MU= zBnTa0bwy%&_n!U26mq2-0!#0Y1MgQdu7Qb>h;`Mp@*wt8Bw|szd&?2_*#N|PihI(g zD`I_MUo4-h?|hyG^+T*jCN5s-jh*NhJHP9PNE`D|))ud))-QIl2*i4tNA=#7u+FB8 z6(7)qzWL$rUQeB0+UyQTWRJZ5?PfNmH;&nSBq@;%?<1ZbJaG3*n`LANbDwg5dJEap zeRF}>LfF<^#4AJOc7H!kj3SEc8^RTFsAhLVUo`E_q*}6{+<>V4l)G2CXpQVn8X-P; z#Wl>tR2%9p|{)hCH_JVeVMf2z@_4`JSYni#O?!&t#SO>Fu<9K)y|YKWUKy%C-6824uC!xN|1j;s+K{i|Y={HbMyf0CI`)PG z+5V7@Pk(2NMj>Y&R+V^=X-A7Ey80$0Im(o?JPJfzi zqr1?2aflvA6UJG5&BlR%BHRF!28ZGAAV_Es&4)4+<))zNuN7T^7oj+}7~Mpd;wst* zPs27e5AH{Ar{k!+KMOxx#LPDj(w-ssyjLV5)yVJRDfL63pnUU`SD*@?7s>f*s#c(e zY7{6?L4hI_6&OT=6c|i{6&ON86c|cF71)dRQeYSjQ(!m^*L2AZ_ols-gg&&70{haw z3hYPwDKLUYD6l{6ufRweslWkrfC8gvlmZ9RfeMVK(F%;AF$#>Ou?if758&kf;%J-# zwN%@sL@p(P3|rKJiiqh$&#r{xN) zpcM+Nq?HP+qE!gH2pLbuD+$%KT7fmRMu8LP1O?X8S_Mv|6BRg#PEz1i^eP2Trjr$T zHN9Gab+k@_Q|J^0I;m5E^|W4r4YWalF6z?o*gtVox02wY9+^G4@p;I(SFw0}yp9et zFQTP$6KWopp}uAb9xqY6p6wgABo+_xKWTitnRy z%uUMIWpM?9V(oA4(*mXZ6&5){O}`12iYq>(sL;Zcr1-H|~YsFEB`2BUSb zHxcotkbdO8?E^0`CE z?{*>Idm8!QD&&LrB0szt`QlvUkJFG(PJl98!ZF|*zHZ}{FCI&+$gOQt*51w1ZQ==D zYW-pf8*Y~+l|{Q^*jIM(bVBI03+*!ELr`kR{#?)x`Sn~nk{lt6Njmop_b4YnQj@y7 zhKj2oTHew@#K2ufu_`}Ah#&z4VKCd4B&M^td{JO?P`>F#+3!cfKFtx+uPFN!n8fWG zN6hTu-sTo|DjF|r6=re0rgy~d!``N5b~FZ4vMk~{js5D|9NrO4iei?Ttm5?kBX3-O z=t?IX-T^!PcfjRWV*^89-Kz4A=wPP@;9Yc)0S9}aP8`Aa$^J2!Z8IR=VkQRqzL0lw)!fZi8OKIQ{LuTEJVI~J~3Hb-=B_- zdQfglMd$XwH`3EXyT4BmZzh2UE$sob1Loih&$}jwr!`S`dVlsd%~z}p?B!aV!{D@f zuXn^eznS5QiQ@Dq=>yXtnMN9XZFfOGP6g$e$k&A3n)5}!(q(qD*1A{q9 zb_SUb3HU2G_%l=FN#aDx-@!?!ZXQ+@jK7Y8uki3jI#mq1%-hC*pCg-6q{-=f8P`vC zv{*Bo8IV)46GfWD%ND`9(M&4VjJ<3wHZX!cS*+>*XStc-mpX0AqlsqYN;Jd6FLl`T zk7r$pCi#z9pOk2-djG)*Q*Nw3t1s20TqbW>sb-Y^4^MGRyBk@1KmoUaJ{6FFs!~g&Q delta 21587 zcmd^nd3+Sbw(wN-OlIruVh<#Q1PFvA472tuEJF5uCkY9JFqtI@kUe3&8U{o~uLz7< zh>Ex(3WB2Mx}YGJtDv|cD!724tm2M{UZ3CTo(Y76sQ2@|_xQhyxPn|kd zb?Tg2vWj1_b5vAOL3VAcC(7Hkg0 za{u>D7Bs}at8}n`O0&a1zsZ0GLu$BxZ&RFK(-eQ4ziP`qz zasIze7>)Y+ADmF(=O)CXK4Nz61ik;}@uLv&ZyR6Xf6$>WHvjX{;r_2<2)$_q zit?{)8S3{(#~>Bpzwa-Pe8m6Qux)kt8s*-Qqajlie))sUQ}{HNp;@x+ zG6A?E{ash=ixnSe<5$>B!=A)b=)NcMCgsrnWk2{L`q7=I@I?CTQ@BDodM|s2&)b*I zI*ljM((U+RIk|EFIlAc~yhlFf%VE1{^TYU@vd`_CPW$S3`pJ8M_246Tv$Efzfs=f3 z9Nm8qYFe`q&zARJU-2y6u@>JW$M^CbwBj!Oh4PwLQ_lKQHFWdafVF%bzCqsCHM5eE zyYV^s!26?rqla(DU&;pzocJx>cL&fXH2<^R7ZE~tz9!OlC(su*(`@#66?E2KkvsqnlO%eLB~iX4eFS z;b%=H{H)G~pT9fc=a$j%b8~-DpfEWhBj1w~y>v;I|3rMGlAM&0AIQmGaI*a;^+Vdl z$`-S+GIA00z-j2`Lm*#!$u9C7d6H}*_mjKGYO;d($U@RYYKfDSkphxI5{QEsNh}#o zhLFBQO%Q*P|CT?^f5so;-{JT2yZGn$C;3hM{rp}0YJLUp;}`Nxd@b+f%lHC5gHPZc zypfOP+lTW*_`bZFN8Cm3TkbUX8Fz?#huh2T;-2H4CCo{LsOX;rd=a)hKrJse}j(v2FA(f=aDzlmSU z|Bau=H}bQ2H$Rh~#uxJ0d@4VMAJ1F(ar_v51V5A?#P{JfyqxYq>St-?jmx^ZY#*wgWSCw1qr)_Th6tEjJ0re zTs279bgr1o1363LCUG{9w0Le5e7E?weNfGrE*cdW(@ejKL5l6slRn04w#nNUvvD%= zGpOn_po0&C!0aQhfVezG9s^-nN7jI-EG3ITP-c@V5R*cZ2|_ZSm_S5EkT4Js4Z$ED z-|=TaI1cmgf@r+LKM#WO82$rEGWkT11`{6#vJl3%_XAPD z+|M8gXSib^2JdqFKnR}ao&phgfLjLwa5J|Qy1$v54V_=k6++i1a^s=n{f<4u9%B!)@3Q;YSJ>y-r`X5X2iSG&8un&(DZ7YmW@ociY&l!VX0nOwc-92p zE%Ikz&DkJ-3~U8MUn^j?w0p5vDMtu-ut$P!?3SPlyChhJt0d^eP6^J!vm{uFDSr{n1oEXU;%EW>3IoQ9`KuoRbyupO1)5-Fn?7fWy|o+`m2TqMCl zTqwZ;Tp+=GoG-yVoF~CtoGZZ`oFl<(oGrmDoF&0boGHN!oFTzA6c(2UIzG+~njjo2tb12#xdkM$B9hsQ}!hjkJh zi^obZ9>+^C4#!C_7RO3(3?3uF7#yQ)7yE299xY{z!lNV@jiV(v5|5PN2s}c9!|`wl zM&T$4M&d{b4#UGF7=a@sI1~?+U^ot!U>FXQ;1E1Sf`jp335Md(cIjad9wfnmc%TFa z-~kfskNZooAMPi?zPPUh``|tjB$!B$$GikN%t?^NtOT`KD?ts`NKlQ{5>#Q81eLg5 zDLsVX5D6-fQE}b*L~GVtr)}@QCp+i35_}%8|Ma_8zE-9nmygJhj(wQwYj#s+S42*qPKWj z+ZqG(1+ydR(~EIbR3wvx>Kfd&-i4Ko%?+&$tqU7Hm5rX}y4sc&__PR`16YFwGJ4d| zTwMv(RJMAYb(PHxb3NfSz8y!+=Ddr6lKf0=|cCNe9 z-Qb$zY4+C6^$Z-$2qJX>+T5NNS95Kn$kYMh%mhT2EXARyKfPrU9zw$x<8XNx<3RN7 zrFanPb0?(fON;Q}2|P2rlgq8nYK~dQaR%;0}vJ+M8M-83Xj)zu> zxj`f`SJ{UdbqTAXs-ZPR%;-cAGZf-5x~_=_2TYURtQ zPxm*$#nzlPZpHup%LNKGIWF?$OLDvy!czUm7e_cm1vLo0EkkeP+4wwjzdTv~y<%Mm zRa%t0)k^ilnpAB+EoBw3RQQQo!7E{@=jYcDp3uHKQS5f>(0T+v`m;UR4yFDZFt zo$?j6Ozl%&)NIujv0FefKLZ=8YSbqCSoR^Bj3$6-B|khXEF?5?AZn_ucY79yYM`>- zS?3WShOTMJN5lLDEsIg4e}9WvwKOS089Hzvy2AXDc13FDB_mB93iGdOZb02@8QfXR z^|h_es=1!F7EiPIAGQ6%zw3~H+R7+2Tr`cWTsZ}e@V~V(9gP%|y4w;_w7>DTax}{S z`fY;0;n8?Bdi&jvrXdvLcRm)e=CN653_Pqrv4Fnj@pULpEVk|OeE<4gMijsO?Oihw z8Y|`{?`{myB^9P5O!ZIL9qvE4JKjI^9X%SiJ^P*g2H9<>=|;T zmtKGoyKJR+9hdjZslzg{*dk47Mm^`8~WB_6ZMg3GCy3JY_K@-rRg{4`5qR<0#IFSV@L zkx*J{D=xAdGLurL358aTMiwd4tBh)uivDvY9@}22*2;$a6tQZVN~LOOYi+4@dw~9E zlQY_AiWX`XWjC~C7B$spS{B)=9OZLc9km`?UQTh+xZh?4TE6;9jOlmiJ>{T9fQ;{=YC^i?C z)FzIboip1d)HLMUr%tUaE3u~KnOcMtZ-Z-rd46h=C9A%{*roPLWq^F>1eK~yoETu8 z;Bg0ASd=?1)6_gwu#_~qOY%MT;hPw8&+4ka+5{t`_T0YyH zm|;(>YpZV%y0#F~g+a~EdUr#eI3R*86yzrqTC$td^=-`tw=*#}WuZB%WL#5m>74xX z>YTd5tTN-gW_Q`RI>F%0FwQHUpOIn7t8cJ$HqZw+HYVp6 zr4=S6CTHfCmE@%4CYKl6tmfRTl7y1;&n+%VOUlX1%PlUn z30W0#tt`@~h><$g+w5tn=|XaoscBqI>(nMgQnn*)ZjQ@1wb7N^HrLczkTlm*Yf7sv zXvi%yE^Jv~vo_iFb@PSVT=ycNxszm|*uzMDI!UU(v8@W$Nv5UruAA}T_UTwF({-}J z?U^U(pLlnxG?X$Dj25$_B&8@bH$68wEwMBs(QFcOO-55*QK>CEEzgjZTwa=ACKMzV z=9Lu{*|M??R*SvZnvtDtPfN45A+0RJrx4Uekdw~JgEVKQ)+Bk{`c!A;w1V`E3{#rX zknd>DODaq?yXIT%^)2?K9LIw4By(bUQ(3isL2Bt-PhuL~c?%vLCC&vbG0VV~kweU2 zS~~)kpvOrIc@JzrOUbQWm4;~KHLz~RZ}z_%PWFgar0>W%u$O!a7Pj}uTcD#(fni`z z_cFohe-Pc~2UGiAIx(D|EaNA`7C^@@C(FqxFdKaaRstny;a~4w*uXYo|KxRw_P?r( z*c7>sL;u$bNEy^X{(b9erudw;6{_yX)Z%R11Uq-$Fd-+G?eXx1GL0$#J&Ld#$Izt}sR<0vTj^AVa4?b9>nHwTW%aO5UtrRSHIq?K5T%q5mg zQ&LKLrXj(WoK|4XNGunU?ZzBiiNXK!TciBxZ{>|K8j1=t4AXK;GfbJuc0+ktdU~ck zrz|rmy}U?(^7Y>z6B6;y6RzV(+kq54Z zwK`v2GXrM1Vzqksz|aJabK5LdgWGJ@83c<sIDGLD>%v~n_`$^ z^`B`sP}NT|^3&Bm1?=aQDN1IbS|mzbmJdKq8FHeZaOWc1e^jrc>P?SO=l395jbN)F z_V>SGozhDux%~hBTCMoY7{93hr^Y^6CnGPBbv?;H!q9bERc#q)SL z`RiGobQU&U9HaE-P^49oGor!dOfQIe^h`JkSBl5AKlFZFOHV|iNVQm#bZ*p3T{-m3 zNEEIn-vrMjdqFD_=RY_PgdS!71g;(pRFku!Li(PZ?WM|Gde)4>l>9asS;%kej$~lZ zW4g_XxRGXy*XcI9btaR`uCsWm+&a76?9kc0c6XK2;&7NPHhR>G4DC4$^<$$MS{xaGucb;?WpW5Qhr?p3G8(*s)nK8UY$z!TRN@s? znap;J$WSafa*$uTvmmGxCDlzYP7vkig4jzJdO@5at?%hClaWSAcFW+PaCa}5W9Y5p zv1W32g2Zpb5ln*m>Ysj@-mRd*$<|&fA3(QeAw~J0=r$~B`^yaMoqqvSO`Qz3IO|DW zk9NmyV`yC-(zct-wyG+d!K-uGy+)lyunD>MPS{SUyDj6zP!va}j zHdGm$7Qq5@nn}=E3|^DY=@tw+0lH2w+Px0D%Zb65E)_sBq!El3x8U*EbS}5is1JLv}EraNN_n`q&o*a+@QW&AEpBi|6KOV*5`Xo9u z9yEJpGU|`gq!)70p#C41qAQb80ZJEZXhazu1t};8qf9!h9F3ss%aEGxOG8$=e;Uj~ z*@4_|rlEc`vkc8gIhWA-&^^;p1pO)v3WSxT6qGBzx*6pKFy1alN;+UVYD4*A#u)0F z4yO=MM+7}K9Sx<G0qpKye#diVDSPYF!)3MnyEg01xi?tqtiBntJC}G?=E%hxx8J z$UC5IKF}7k0@+ZBgi9x_N16`b3Um)brBZu?&pc8kto5&%;$2s*n%aOecD$!jngT^^HAMGq}NLbTIr zw%S0C=*$+E8ESGlbxxZJ2Dq!r>IL;;^El1av=B{-HVA^%BzPSvjRn zF00w;anX&7k&x(iz+3_Z=!_N-6Fx%c@Cq>T*ewPpC{ByXYZ66XOLofOFr?E3sEbon zx8CnOaO+QZwxg6GL@p!Y#139jFM-v>4JXWU@RwTE(?QL^+L#XcGH@9fPUeW#{I9_3 zau_C*e~}l#jb;-}EPimJX(z2@PLC$X-pSB8H=;X6i=-0rj5G^(v-G?rTHeS`@(g*t zM|l|Q^qJdG{%`_|9hktO=x5|(aB_MVoZ_~D(dSEY0tk75Jkzu2M~pbwi-(g(#iGCC zHP*A}0Y)18n<*-b68$%~L-B9H)btiG>k0V$v%rA8!1LW;Jfy=P#S^>vk#3I~yG3eU zLAT$Ha)MvKR@65?fZg!~*eX8+o7Fz3;6?cECt&!lCHM89W9)8*u6+>I4JWH*(#hqQ zHFx#}K?xhBq>q%t!JK3Xif-c}#Vy zdaU|2ZK3u>@F9AH6!RNNhHNt&h%KQTPoPvZN7B<9D*LotaA#u9+h`KYH8(tFh%68hcCHeX1-jdlR7q| z38*44C*8Uk9YUxQ%8#Ode*(3jS@8G>a!MI*Y(*A&!%mcps-*NcJ7M?b>PX*(Zbir~ zPH$&lK^4eDckLFZ_T6X(@`}%sHOt33m7xv?TG2wk{R!3vzkLW)z5XE@gsKDco9-m) z)1f(pZpWww>K;rb7A^h=KJG{)4xy8au!f3Abop*X?%aUnSoXhu+T17)LF6tOxvPhy ziR0|9R}t$;F{B&p`G(@s1bb3Zp^%@PnN?a;l%AB6QIuy&wwER3SW7ajW##td+|=TX zyyEO!TUKd-!8)zLl3P@iqUx~#wg|BH7i$^j8&rHriXS<69&uKWOTBb-LJ$7>XHaI{vXvzXkVqRlWt zS}iWCV07DIxl4C^fRcjS`QL9&qUiT_{Jgl({WIToRwVrQYWic_0sZ&)NUi3QWMm*8 z$0c>=fPuXM(WLWe?|?2FA?Rqq0H>m68ucUkpxYaZLoiy+RC@t^kbh+v3v5XRgXArv z6A##>vtUJ-OcADdEu=*`Q2Bs9?)lD_ZP$3@q2z~MdXr4{K zz$T@kvw(tr{0oZK_}E%yXc{1y14ygXm=~3*CPZRv<7ISsV^jB=tqjcq>7fY#cl!To8L2KQ65bFOG$kp@Sg=F3GO&C>-ZU=wvng9oUYxysN4Xdoug zfJTeMP|Ia(NNZ`X7LQh|>y)7>fT#~3zM;h&U9W|1XuXULnU<*Qr`9V&BO=gEizP$c z+=gmTeQWc=PS^%fCxbfH48_B7(A08+jNCxBUd>>-m!UWOhT8vl{ZGHqi;H>&!=nDX z;PRvUPzahDiUa$KANyl)zyCwr?+EVyC(y)0M;fT|X3=hN^VKW|`x$z3D4rMep(v5T zYG4tG21o7bypnsGbFx3MD_Og?qzmB&wwj^CZe#}a?>ZFi<7{S|*-F!HWNy#-W6fq9 zaqU#CMvQ4fvIJSRTGP|^Vd<=L))FS=s%?GZV`8bn$H?gyOPEntMS-&6)b3*@U-d0i z5J#W(_3|n3HnEQ}XoTuYO*qY5%Eb0Dbblq$vw0~K-}BYUrOed6Mo0G&K;9r)wTu~e zm3Q|qW5%cSDB@T=LNpUshw)gn4H;!m z$i(12nJgO)Zwd8%ijz8i{S-5(qxB8^cc#Pl58Tdl)NR0irX%+r{3z27xGSf@&#m#| zP+P3rD{jR;fPGz(HcV5h30EIcKcl`y^_uEA)iPC!%B3n&C8^A+k*fYGM)|$+xbgs4 zbWW2`R0~zLsxnoE$^oXuAu6@W57Y7#YO&2UXWO^Et}`U~~@>V4`L)SJ~C)T`A?)$`QV>SFa2 z^%!8{*Qz(c4`u_I2cF?RQUmc}r5e@Cs%@%=RBKg1UpfQ(F{Wca!(r_Th9La{H4}>T zMx$LnUrNslrmgxmDcu@OTl6hbx;dCO>6@fYVRCc1Cy?Pk5*qPGiryH#(u z>8JH&Vy?_Al}NgTRw9uRNES=&!t+$A-69Fo=Fr;>db?3yC}9SY1rjDa=S!G*fZ2}p zHbHN*>TNcCZU9wE<^)g$eYS*}B@tvZ=xs*5&7`-P^_hWYI&(4tWu;_#psYckCY4Q< z%31}z)vCAJ^j5n*MJg-hM2T6+QsDqbl2jO;6Q#ll^ob-qoQYs2(=mX0ennx0v+t^qw!UgAtepG0P;6H8Hi~lU_bX&$z^=vGSO) zOojLqD<2)pl!{NI!8oz?IF4Z;5JF5x$w!K!2~QDV+|NFahf(8khyjp3XVr6fKw_8A zG1>GcyWZr`59{(dfy{_5p948VyL?VKedQR|5wm`9m&{Ol|1mslm_a|VOXdKH`@p6C z68HPjN7IL zN*dPUvr@tl^dSv_AdP#Bj5&_x{=EMK#5YXR?0CoMp z4FMh5X&XHUffk>EL)<}_M)rbnaR-DAJqeS+!{j~)9rlx3$&D~2EGBKF0m6q}q=J+} z0C5&cA(J75*i3XJ27-vg$v{G2AH(p!@)sbG_$>bge*{8_5Agf>S0R}A1^yX+D})n2 z$lnVAPpjMcTlnRCJH!;X@O6AOL={ixi}^f=D^B7k@ivGoj)!QdNC++N&vU#If{TCQ z&U0Twc=2)W6Yc{DFn*oe194K%aof2q+(rm8zMH#)yA8sOmvPr}^C8f94(EXgsxq#K z%i+=>*mwdbaC!(g9tm^#)j~k$iR;9Um<;?0ZpWlXM7je4glm8-j@!fEt6i?0#8$8m za{@b%J;zX>{hY4OMk0#HIlCyUH>yqr{i|x)l~lx zJmr56q4oMhSVcGPgygHc@GAQ17Iv%7rJxE_gb57+#c$KQHfwyvC0p zFGj@Mc?h1`55bF#KjAc5@(RwPr(eOT)F$Sgd;)%mSnNKj0x|jWRhJl_FZD$ zN04`FC**DXP(*%3{CNUrP{UTJ>cnI4bl+o;XWN3)5JR00;wgxyV;;wpZrm)Ee_SlT z8Rt@7#C>%W)6#DzGE)>{|J+6mKyis^ z{E6Yqfz{S2kgk7mJ&pzp>^9W#Ofk;K1te5l2>v%lPj}zA7}$Q|S%QKN zosLfzkRgGQ+GDFFo?w8rViS3g+zW9{tH~{}uW5&@(s%?$_~T|`vkhOrLaK|#&%vc=#ifJXk7t7|sCW*Tfh05bZ?ho~?$jJSm^RPG1?VQpN&`6g z^sQQ)if+8JT*_>$qswY=BDx`9CI7Sre@kZ-;fZMZCCnF#AojMp0W540h(13Ik3jS2 zr^6u5wysebYJ{~*71}P8_<7Za2z?Zd_FWEirTwaCq>JoXr-)mOVwT%Fi*CcAsY8; z*2W%2$%O%XWdiDA$aJD*qERzXJY6V^;o*SqJZ$Mdh3))Sm>pk(=$h-`a)gIKrfNX* z7lPKegB%Sb{b4Zu0=F5w!fyw)Jp~l@2vF5>I8QzTyO@VT$FG2})*jv?2KG19A+N#+ zE6D~iM0G=VX$+Vt6m&y19y!EZXtAf4m`YO;OG{IX>DGAB6IGZEL_u5@b*~rqXLRbT*LGeS?7h@hE;%aGpIab)SPT)^ z)2oA69NjY?PY8@IxIBdvK}c;gyGgrK8>MMcFT463D{zFi{Qvj6RAyTS9U4DT(n_*M z8R^4EawTk?c0cgVpq{Lnr@S@f9NbNlFRx_o#9J^2chfY=5>_V+(Cdq(8C5MH;V4RL%h%B z0Nghd*FPnc?syIEn;CNPd@Zfm3-`^8{ovmoU+M_Dd6jtI%s#kp#+bU}-;}&A-Z%56 z{0u$(Pq=TU@A$b(=)OI0-^|!K)4%jZ45K^$E#5ctFSrLGdRmy?=Z&DVeiiSVc@^%P z8UEgzFVRge!hJLPW9P1=%{$<}8S_(PzxUOJ(NBI5>3a$2i$3zrLSNhvy8paL-^)Os z{lKivbjQ;`pYGkHduhcpKwr!O{G%^*5ZzqEOrkeDiA>O}z zLz~Y5leCR<2m9(w^pkBOlfD8bjU9e5#206z`=1n<^ffR^6_UP-?l=WZl5Z^;Kr2oI zlT_&stoEgjqnjTBtmS8*6XUkUo}lC`FlnOi{=xL{d(ep!H7Td*zJov?+xO%IUql?; zxmKj_eV~usu+8T4#?o1LiS&H{^u^|_E1{d-2KvV5{}4@^-@)gq3@>Qch-{7rI}{wj z*OD-}8wKoJ|9$2yP!{XCFhyvm^x+{egn_P z@#tgl3=?FZ$d&>DQ5~gm%qG-f9KgKAJd?n@%yg7|gCAo$oFC(Rv2-C%oLrE}1^Ht6 zD+;wDU$ItkG-OOjW5_?0GG)4QmGVPXl&S{oPCtQR>L&Hun!%b`nnyL?Xzki1+Sk}V zK|CUF zK$yrZhR)5y=^Yz(ijPt7NH=_lR|P-7Ad@eU?^LM%`^c9Y{`3c^rL957br7dI(*x0e#g)68BBXY~IDHPZL( z8K-H*ub@UkJJ1GS>Ug^OMNy6X7rwy~cik7i1h_k2Mnd`d+&=cO_B3$UrlzVh%Ab{L z;BKkB1Fky=!?D2K`LguY3E`KR>#OPv{mAG*~SQ68L! zOu!rYld625w=6ghnSnPXyWuUmX{^|FHsLAStOMTYYv$hUt1Ai4L*sxq{D`dmzPRGx zJfsKSjQ{4w+v$!d=(=H+^?#=o!-2kQ4u(GAOD&|EzZ7{h0_a2AuNguta8&ZRqM0PCYtZ+V7RQ~>J@+Pty8)Y-w=Z6>gO?8FbZQBn!4H(j%N6g`{{ zte3ZLZl(J&fc2I)EGv8w-r($(DGr;b_GJ6Kp5W}31@y&N-fyIvrT~5M|Jpf>HYWgm z5g$dI_0?4cmr#j7pAd8XL|>dUxP(dq`X*fbXE@zqVeTm&u*7smw9JiCtd)uCUYRZ` ze5I_b%7+PI)n&D>{4k{PN(HYtDk}ewF#+|jSliX54uZuHHFy{$CD$<-^xb*PWZ1y1 zy^aC9hlc?Z!+7|4IfhA~M@K{6ePYrky%^1`qWT#4xsZbsI!@V{2GntKJQyzNLpeBo z<4zBQ=>xG)T{+E%>N-wd2e+vVQmfWnWkG*=%K%cfi8e0;KBZKX_alkKUR#DOd d{b)V*RVzI_7y7Z^@h#`*zB*=;+@e`&{y$O#TS))_ diff --git a/util/backoff.py b/util/backoff.py new file mode 100644 index 000000000..15429936e --- /dev/null +++ b/util/backoff.py @@ -0,0 +1,5 @@ +def exponential_backoff(attempts, scaling_factor, base): + backoff = 5 * (pow(2, attempts) - 1) + backoff_time = backoff * scaling_factor + retry_at = backoff_time/10 + base + return retry_at From 2cfab6e252cf298caf067145762dba2acf17cb5a Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 2 Sep 2014 15:28:56 -0400 Subject: [PATCH 19/49] Reshow the sign in button when the username is changed --- static/js/app.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/static/js/app.js b/static/js/app.js index 5fb21205b..f44e4294e 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -2278,6 +2278,14 @@ quayApp.directive('signinForm', function () { } }; + $scope.$watch('user.username', function() { + $scope.tryAgainSoon = 0; + + if ($scope.tryAgainInterval) { + $interval.cancel($scope.tryAgainInterval); + } + }); + $scope.$on('$destroy', function() { if ($scope.tryAgainInterval) { $interval.cancel($scope.tryAgainInterval); @@ -2325,6 +2333,9 @@ quayApp.directive('signinForm', function () { $scope.tryAgainSoon = 0; } }, 1000, $scope.tryAgainSoon); + + $scope.needsEmailVerification = false; + $scope.invalidCredentials = false; } else { $scope.needsEmailVerification = result.data.needsEmailVerification; $scope.invalidCredentials = result.data.invalidCredentials; From 53939f596d3c2d6fd68f74b5d3522628c5aa7995 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 2 Sep 2014 16:45:25 -0400 Subject: [PATCH 20/49] Properly escape the $ in $token for the auth dialog command --- static/js/app.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/static/js/app.js b/static/js/app.js index f5c612c5f..3838d561c 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -2425,7 +2425,10 @@ quayApp.directive('dockerAuthDialog', function (Config) { }, controller: function($scope, $element) { var updateCommand = function() { - $scope.command = 'docker login -e="." -u="' + $scope.username + + var escape = function(v) { + return v.replace('$', '\\$'); + }; + $scope.command = 'docker login -e="." -u="' + escape($scope.username) + '" -p="' + $scope.token + '" ' + Config['SERVER_HOSTNAME']; }; From 232e3cc1dadddb7451df63386681bc237426ca5a Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 3 Sep 2014 12:10:36 -0400 Subject: [PATCH 21/49] Move cancelInterval into its own method to remove code duplication --- static/js/app.js | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/static/js/app.js b/static/js/app.js index f44e4294e..1f7eb060f 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -2278,24 +2278,29 @@ quayApp.directive('signinForm', function () { } }; - $scope.$watch('user.username', function() { + $scope.cancelInterval = function() { $scope.tryAgainSoon = 0; if ($scope.tryAgainInterval) { $interval.cancel($scope.tryAgainInterval); } + + $scope.tryAgainInterval = null; + }; + + $scope.$watch('user.username', function() { + $scope.cancelInterval(); }); $scope.$on('$destroy', function() { - if ($scope.tryAgainInterval) { - $interval.cancel($scope.tryAgainInterval); - } + $scope.cancelInterval(); }); $scope.signin = function() { if ($scope.tryAgainSoon > 0) { return; } $scope.markStarted(); + $scope.cancelInterval(); ApiService.signinUser($scope.user).then(function() { $scope.needsEmailVerification = false; @@ -2318,24 +2323,18 @@ quayApp.directive('signinForm', function () { }, 500); }, function(result) { if (result.status == 429 /* try again later */) { + $scope.needsEmailVerification = false; + $scope.invalidCredentials = false; + + $scope.cancelInterval(); + $scope.tryAgainSoon = result.headers('Retry-After'); - - // Cancel any existing interval. - if ($scope.tryAgainInterval) { - $interval.cancel($scope.tryAgainInterval); - } - - // Setup a new interval. $scope.tryAgainInterval = $interval(function() { $scope.tryAgainSoon--; if ($scope.tryAgainSoon <= 0) { - $scope.tryAgainInterval = null; - $scope.tryAgainSoon = 0; + $scope.cancelInterval(); } }, 1000, $scope.tryAgainSoon); - - $scope.needsEmailVerification = false; - $scope.invalidCredentials = false; } else { $scope.needsEmailVerification = result.data.needsEmailVerification; $scope.invalidCredentials = result.data.invalidCredentials; From 0bd9ba523e1d5604492fe3cc7b49a4c19d227003 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 3 Sep 2014 13:07:53 -0400 Subject: [PATCH 22/49] Add a migration for the brute force prevention fields to the user table. --- ...add_brute_force_prevention_metadata_to_.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py diff --git a/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py b/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py new file mode 100644 index 000000000..55351a50a --- /dev/null +++ b/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py @@ -0,0 +1,28 @@ +"""Add brute force prevention metadata to the user table. + +Revision ID: 4fdb65816b8d +Revises: 43e943c0639f +Create Date: 2014-09-03 12:35:33.722435 + +""" + +# revision identifiers, used by Alembic. +revision = '4fdb65816b8d' +down_revision = '43e943c0639f' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('user', sa.Column('invalid_login_attempts', sa.Integer(), nullable=False, server_default=0)) + op.add_column('user', sa.Column('last_invalid_login', sa.DateTime(), nullable=False, server_default=sa.func.now())) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('user', 'last_invalid_login') + op.drop_column('user', 'invalid_login_attempts') + ### end Alembic commands ### From 18ec0c3e0ace4c699369bcc73cf59c18c8abcf51 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 3 Sep 2014 13:09:17 -0400 Subject: [PATCH 23/49] Update the audit ancestry tool to not affect pushes in progress. --- tools/auditancestry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/auditancestry.py b/tools/auditancestry.py index dce445e4e..95e082642 100644 --- a/tools/auditancestry.py +++ b/tools/auditancestry.py @@ -20,7 +20,7 @@ query = (Image .join(ImageStorage) .switch(Image) .join(Repository) - .where(Repository.name == 'userportal', Repository.namespace == 'crsinc')) + .where(ImageStorage.uploading == False)) bad_count = 0 good_count = 0 From 21f7acf7ca14a1870336664ce7beaa5723b4cecb Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 3 Sep 2014 13:34:36 -0400 Subject: [PATCH 24/49] Fix the default value for the migration to use a string --- .../4fdb65816b8d_add_brute_force_prevention_metadata_to_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py b/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py index 55351a50a..a1c8c95dd 100644 --- a/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py +++ b/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py @@ -16,7 +16,7 @@ from sqlalchemy.dialects import mysql def upgrade(): ### commands auto generated by Alembic - please adjust! ### - op.add_column('user', sa.Column('invalid_login_attempts', sa.Integer(), nullable=False, server_default=0)) + op.add_column('user', sa.Column('invalid_login_attempts', sa.Integer(), nullable=False, server_default="0")) op.add_column('user', sa.Column('last_invalid_login', sa.DateTime(), nullable=False, server_default=sa.func.now())) ### end Alembic commands ### From 8910c6ff019f859c3f2f72c4f7f8f996b7f76712 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 3 Sep 2014 13:44:05 -0400 Subject: [PATCH 25/49] Add a migration to remove the webhooks table. --- ...2b0ea7a4d_remove_the_old_webhooks_table.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py diff --git a/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py b/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py new file mode 100644 index 000000000..79ea17be0 --- /dev/null +++ b/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py @@ -0,0 +1,35 @@ +"""Remove the old webhooks table. + +Revision ID: f42b0ea7a4d +Revises: 4fdb65816b8d +Create Date: 2014-09-03 13:43:23.391464 + +""" + +# revision identifiers, used by Alembic. +revision = 'f42b0ea7a4d' +down_revision = '4fdb65816b8d' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('webhook') + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('webhook', + sa.Column('id', mysql.INTEGER(display_width=11), nullable=False), + sa.Column('public_id', mysql.VARCHAR(length=255), nullable=False), + sa.Column('repository_id', mysql.INTEGER(display_width=11), autoincrement=False, nullable=False), + sa.Column('parameters', mysql.LONGTEXT(), nullable=False), + sa.ForeignKeyConstraint(['repository_id'], [u'repository.id'], name=u'fk_webhook_repository_repository_id'), + sa.PrimaryKeyConstraint('id'), + mysql_default_charset=u'latin1', + mysql_engine=u'InnoDB' + ) + ### end Alembic commands ### From 6c60e078fc207b0fb751f45586845825115bc421 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 3 Sep 2014 15:35:29 -0400 Subject: [PATCH 26/49] Fix NPE --- static/js/app.js | 1 + 1 file changed, 1 insertion(+) diff --git a/static/js/app.js b/static/js/app.js index c55cf5eb5..4e51e4708 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -2465,6 +2465,7 @@ quayApp.directive('dockerAuthDialog', function (Config) { controller: function($scope, $element) { var updateCommand = function() { var escape = function(v) { + if (!v) { return v; } return v.replace('$', '\\$'); }; $scope.command = 'docker login -e="." -u="' + escape($scope.username) + From 1e7e012b923b5b4cbf7fc850ecd54c4c487240e6 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 3 Sep 2014 15:41:25 -0400 Subject: [PATCH 27/49] Add a requirement for the current password to change the user's password or email address --- endpoints/api/user.py | 22 +++++++++++++++++++++- static/js/app.js | 2 +- static/js/controllers.js | 2 ++ static/partials/user-admin.html | 11 ++++++++--- test/test_api_usage.py | 26 +++++++++++++++++++++++--- 5 files changed, 55 insertions(+), 8 deletions(-) diff --git a/endpoints/api/user.py b/endpoints/api/user.py index e2e6a0ff4..054db7041 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -117,6 +117,10 @@ class User(ApiResource): 'type': 'object', 'description': 'Fields which can be updated in a user.', 'properties': { + 'current_password': { + 'type': 'string', + 'description': 'The user\'s current password', + }, 'password': { 'type': 'string', 'description': 'The user\'s password', @@ -152,8 +156,22 @@ class User(ApiResource): user = get_authenticated_user() user_data = request.get_json() - try: + def verify_current_password(user, user_data): + current_password = user_data.get('current_password', '') + + verified = False + try: + verified = model.verify_user(user.username, current_password) + except: + pass + + if not verified: + raise request_error(message='Current password does not match') + + try: if 'password' in user_data: + verify_current_password(user, user_data) + logger.debug('Changing password for user: %s', user.username) log_action('account_change_password', user.username) model.change_password(user, user_data['password']) @@ -163,6 +181,8 @@ class User(ApiResource): model.change_invoice_email(user, user_data['invoice_email']) if 'email' in user_data and user_data['email'] != user.email: + verify_current_password(user, user_data) + new_email = user_data['email'] if model.find_user_by_email(new_email): # Email already used. diff --git a/static/js/app.js b/static/js/app.js index 4e51e4708..72eabc41a 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -384,7 +384,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading var uiService = {}; uiService.hidePopover = function(elem) { - var popover = $('#signupButton').data('bs.popover'); + var popover = $(elem).data('bs.popover'); if (popover) { popover.hide(); } diff --git a/static/js/controllers.js b/static/js/controllers.js index 4d1c8484f..485b7f529 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -1763,6 +1763,7 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use // Reset the form. delete $scope.cuser['repeatEmail']; + delete $scope.cuser['current_password']; $scope.changeEmailForm.$setPristine(); }, function(result) { @@ -1784,6 +1785,7 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use // Reset the form delete $scope.cuser['password'] delete $scope.cuser['repeatPassword'] + delete $scope.cuser['current_password']; $scope.changePasswordForm.$setPristine(); diff --git a/static/partials/user-admin.html b/static/partials/user-admin.html index 1b2ad7fd1..260aa47e2 100644 --- a/static/partials/user-admin.html +++ b/static/partials/user-admin.html @@ -128,6 +128,8 @@
    +
    @@ -138,18 +140,21 @@
    -
    - -
    Change Password
    +
    + +
    + Password changed successfully
    + Date: Wed, 3 Sep 2014 17:24:52 -0400 Subject: [PATCH 28/49] Up the gunicorn worker count (under protest) --- conf/gunicorn_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/gunicorn_config.py b/conf/gunicorn_config.py index 4d9d50499..ca8ad5363 100644 --- a/conf/gunicorn_config.py +++ b/conf/gunicorn_config.py @@ -1,5 +1,5 @@ bind = 'unix:/tmp/gunicorn.sock' -workers = 8 +workers = 16 worker_class = 'gevent' timeout = 2000 logconfig = 'conf/logging.conf' From e783df31e0471346d25affaf74172038a17fcba6 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 4 Sep 2014 14:24:20 -0400 Subject: [PATCH 29/49] Add the concept of require_fresh_login to both the backend and frontend. Sensitive methods will now be marked with the annotation, which requires that the user has performed a login within 10 minutes or they are asked to do so in the UI before running the operation again. --- endpoints/api/__init__.py | 28 ++++++++++- endpoints/api/discovery.py | 5 ++ endpoints/api/user.py | 54 ++++++++++++--------- endpoints/common.py | 4 +- static/js/app.js | 84 +++++++++++++++++++++++++++++---- static/js/controllers.js | 7 ++- static/partials/user-admin.html | 4 -- test/test_api_security.py | 27 +++++++++-- test/test_api_usage.py | 22 ++------- 9 files changed, 174 insertions(+), 61 deletions(-) diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index 854c3cad1..9f9a8c941 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -1,7 +1,8 @@ import logging import json +import datetime -from flask import Blueprint, request, make_response, jsonify +from flask import Blueprint, request, make_response, jsonify, session from flask.ext.restful import Resource, abort, Api, reqparse from flask.ext.restful.utils.cors import crossdomain from werkzeug.exceptions import HTTPException @@ -66,6 +67,11 @@ class Unauthorized(ApiException): ApiException.__init__(self, 'insufficient_scope', 403, 'Unauthorized', payload) +class FreshLoginRequired(ApiException): + def __init__(self, payload=None): + ApiException.__init__(self, 'fresh_login_required', 401, "Requires fresh login", payload) + + class ExceedsLicenseException(ApiException): def __init__(self, payload=None): ApiException.__init__(self, None, 402, 'Payment Required', payload) @@ -264,6 +270,26 @@ def require_user_permission(permission_class, scope=None): require_user_read = require_user_permission(UserReadPermission, scopes.READ_USER) require_user_admin = require_user_permission(UserAdminPermission, None) +require_fresh_user_admin = require_user_permission(UserAdminPermission, None) + +def require_fresh_login(func): + @add_method_metadata('requires_fresh_login', True) + @wraps(func) + def wrapped(*args, **kwargs): + user = get_authenticated_user() + if not user: + raise Unauthorized() + + logger.debug('Checking fresh login for user %s', user.username) + + last_login = session.get('login_time', datetime.datetime.now() - datetime.timedelta(minutes=60)) + valid_span = datetime.datetime.now() - datetime.timedelta(minutes=10) + + if last_login >= valid_span: + return func(*args, **kwargs) + + raise FreshLoginRequired() + return wrapped def require_scope(scope_object): diff --git a/endpoints/api/discovery.py b/endpoints/api/discovery.py index ee8702636..1995c6b42 100644 --- a/endpoints/api/discovery.py +++ b/endpoints/api/discovery.py @@ -119,6 +119,11 @@ def swagger_route_data(include_internal=False, compact=False): if internal is not None: new_operation['internal'] = True + if include_internal: + requires_fresh_login = method_metadata(method, 'requires_fresh_login') + if requires_fresh_login is not None: + new_operation['requires_fresh_login'] = True + if not internal or (internal and include_internal): operations.append(new_operation) diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 054db7041..eb99ba0fa 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -9,7 +9,7 @@ from app import app, billing as stripe, authentication from endpoints.api import (ApiResource, nickname, resource, validate_json_request, request_error, log_action, internal_only, NotFound, require_user_admin, parse_args, query_param, InvalidToken, require_scope, format_date, hide_if, show_if, - license_error) + license_error, require_fresh_login) from endpoints.api.subscribe import subscribe from endpoints.common import common_login from data import model @@ -117,10 +117,6 @@ class User(ApiResource): 'type': 'object', 'description': 'Fields which can be updated in a user.', 'properties': { - 'current_password': { - 'type': 'string', - 'description': 'The user\'s current password', - }, 'password': { 'type': 'string', 'description': 'The user\'s password', @@ -148,6 +144,7 @@ class User(ApiResource): return user_view(user) @require_user_admin + @require_fresh_login @nickname('changeUserDetails') @internal_only @validate_json_request('UpdateUser') @@ -156,22 +153,8 @@ class User(ApiResource): user = get_authenticated_user() user_data = request.get_json() - def verify_current_password(user, user_data): - current_password = user_data.get('current_password', '') - - verified = False - try: - verified = model.verify_user(user.username, current_password) - except: - pass - - if not verified: - raise request_error(message='Current password does not match') - try: if 'password' in user_data: - verify_current_password(user, user_data) - logger.debug('Changing password for user: %s', user.username) log_action('account_change_password', user.username) model.change_password(user, user_data['password']) @@ -181,8 +164,6 @@ class User(ApiResource): model.change_invoice_email(user, user_data['invoice_email']) if 'email' in user_data and user_data['email'] != user.email: - verify_current_password(user, user_data) - new_email = user_data['email'] if model.find_user_by_email(new_email): # Email already used. @@ -377,6 +358,37 @@ class Signin(ApiResource): return conduct_signin(username, password) +@resource('/v1/signin/verify') +@internal_only +class VerifyUser(ApiResource): + """ Operations for verifying the existing user. """ + schemas = { + 'VerifyUser': { + 'id': 'VerifyUser', + 'type': 'object', + 'description': 'Information required to verify the signed in user.', + 'required': [ + 'password', + ], + 'properties': { + 'password': { + 'type': 'string', + 'description': 'The user\'s password', + }, + }, + }, + } + + @require_user_admin + @nickname('verifyUser') + @validate_json_request('VerifyUser') + def post(self): + """ Verifies the signed in the user with the specified credentials. """ + signin_data = request.get_json() + password = signin_data['password'] + return conduct_signin(get_authenticated_user().username, password) + + @resource('/v1/signout') @internal_only class Signout(ApiResource): diff --git a/endpoints/common.py b/endpoints/common.py index fe09104ca..52715a1d1 100644 --- a/endpoints/common.py +++ b/endpoints/common.py @@ -2,8 +2,9 @@ import logging import urlparse import json import string +import datetime -from flask import make_response, render_template, request, abort +from flask import make_response, render_template, request, abort, session from flask.ext.login import login_user, UserMixin from flask.ext.principal import identity_changed from random import SystemRandom @@ -112,6 +113,7 @@ def common_login(db_user): logger.debug('Successfully signed in as: %s' % db_user.username) new_identity = QuayDeferredPermissionUser(db_user.username, 'username', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=new_identity) + session['login_time'] = datetime.datetime.now() return True else: logger.debug('User could not be logged in, inactive?.') diff --git a/static/js/app.js b/static/js/app.js index 72eabc41a..d23235a66 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -713,7 +713,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading return config; }]); - $provide.factory('ApiService', ['Restangular', function(Restangular) { + $provide.factory('ApiService', ['Restangular', '$q', function(Restangular, $q) { var apiService = {}; var getResource = function(path, opt_background) { @@ -810,6 +810,65 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading } }; + var freshLoginFailCheck = function(opName, opArgs) { + return function(resp) { + var deferred = $q.defer(); + + // If the error is a fresh login required, show the dialog. + if (resp.status == 401 && resp.data['error_type'] == 'fresh_login_required') { + bootbox.dialog({ + "message": 'It has been more than a few minutes since you last logged in, ' + + 'so please verify your password to perform this sensitive operation:' + + '' + + '' + + '', + "title": 'Please Verify', + "buttons": { + "verify": { + "label": "Verify", + "className": "btn-success", + "callback": function() { + var info = { + 'password': $('#freshPassword').val() + }; + + $('#freshPassword').val(''); + + // Conduct the sign in of the user. + apiService.verifyUser(info).then(function() { + // On success, retry the operation. if it succeeds, then resolve the + // deferred promise with the result. Otherwise, reject the same. + apiService[opName].apply(apiService, opArgs).then(function(resp) { + deferred.resolve(resp); + }, function(resp) { + deferred.reject(resp); + }); + }, function(resp) { + // Reject with the sign in error. + deferred.reject({'data': {'message': 'Invalid verification credentials'}}); + }); + } + }, + "close": { + "label": "Cancel", + "className": "btn-default", + "callback": function() { + deferred.reject(resp); + } + } + } + }); + + // Return a new promise. We'll accept or reject it based on the result + // of the login. + return deferred.promise; + } + + // Otherwise, we just 'raise' the error via the reject method on the promise. + return $q.reject(resp); + }; + }; + var buildMethodsForOperation = function(operation, resource, resourceMap) { var method = operation['method'].toLowerCase(); var operationName = operation['nickname']; @@ -823,7 +882,15 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading 'ignoreLoadingBar': true }); } - return one['custom' + method.toUpperCase()](opt_options); + + var opObj = one['custom' + method.toUpperCase()](opt_options); + + // If the operation requires_fresh_login, then add a specialized error handler that + // will defer the operation's result if sudo is requested. + if (operation['requires_fresh_login']) { + opObj = opObj.catch(freshLoginFailCheck(operationName, arguments)); + } + return opObj; }; // If the method for the operation is a GET, add an operationAsResource method. @@ -3923,9 +3990,11 @@ quayApp.directive('billingOptions', function () { var save = function() { $scope.working = true; + + var errorHandler = ApiService.errorDisplay('Could not change user details'); ApiService.changeDetails($scope.organization, $scope.obj).then(function(resp) { $scope.working = false; - }); + }, errorHandler); }; var checkSave = function() { @@ -5699,11 +5768,10 @@ quayApp.run(['$location', '$rootScope', 'Restangular', 'UserService', 'PlanServi // Handle session expiration. Restangular.setErrorInterceptor(function(response) { - if (response.status == 401) { - if (response.data['session_required'] == null || response.data['session_required'] === true) { - $('#sessionexpiredModal').modal({}); - return false; - } + if (response.status == 401 && response.data['error_type'] == 'invalid_token' && + response.data['session_required'] !== false) { + $('#sessionexpiredModal').modal({}); + return false; } if (response.status == 503) { diff --git a/static/js/controllers.js b/static/js/controllers.js index 485b7f529..d95a760a9 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -1763,12 +1763,11 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use // Reset the form. delete $scope.cuser['repeatEmail']; - delete $scope.cuser['current_password']; $scope.changeEmailForm.$setPristine(); }, function(result) { $scope.updatingUser = false; - UIService.showFormError('#changeEmailForm', result); + UIService.showFormError('#changeEmailForm', result); }); }; @@ -1778,14 +1777,14 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use $scope.updatingUser = true; $scope.changePasswordSuccess = false; - ApiService.changeUserDetails($scope.cuser).then(function() { + ApiService.changeUserDetails($scope.cuser).then(function(resp) { + $scope.updatingUser = false; $scope.changePasswordSuccess = true; // Reset the form delete $scope.cuser['password'] delete $scope.cuser['repeatPassword'] - delete $scope.cuser['current_password']; $scope.changePasswordForm.$setPristine(); diff --git a/static/partials/user-admin.html b/static/partials/user-admin.html index 260aa47e2..4349d9df9 100644 --- a/static/partials/user-admin.html +++ b/static/partials/user-admin.html @@ -128,8 +128,6 @@
    -
    @@ -153,8 +151,6 @@
    - Date: Thu, 4 Sep 2014 17:54:51 -0400 Subject: [PATCH 30/49] Code review fixes --- endpoints/api/user.py | 1 - endpoints/callbacks.py | 43 +++++++++++++++++--------------------- templates/ologinerror.html | 2 +- 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/endpoints/api/user.py b/endpoints/api/user.py index d0d089dcd..fb09d012a 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -39,7 +39,6 @@ def user_view(user): organizations = model.get_user_organizations(user.username) def login_view(login): - print login.metadata_json try: metadata = json.loads(login.metadata_json) except: diff --git a/endpoints/callbacks.py b/endpoints/callbacks.py index 49fa1e8a6..1cbd46192 100644 --- a/endpoints/callbacks.py +++ b/endpoints/callbacks.py @@ -4,7 +4,7 @@ from flask import request, redirect, url_for, Blueprint from flask.ext.login import current_user from endpoints.common import render_page_template, common_login, route_show_if -from app import app, analytics +from app import app, analytics, get_app_url from data import model from util.names import parse_repository_name from util.validation import generate_valid_usernames @@ -22,6 +22,11 @@ client = app.config['HTTPCLIENT'] callback = Blueprint('callback', __name__) +def render_ologin_error(service_name, + error_message='Could not load user data. The token may have expired.'): + return render_page_template('ologinerror.html', service_name=service_name, + error_message=error_message, + service_url=get_app_url()) def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_encode=False, redirect_suffix=''): @@ -96,15 +101,12 @@ def conduct_oauth_login(service_name, user_id, username, email, metadata={}): analytics.alias(to_login.username, state) except model.DataModelException, ex: - return render_page_template('ologinerror.html', service_name=service_name, - error_message=ex.message) + return render_ologin_error(service_name, ex.message) if common_login(to_login): return redirect(url_for('web.index')) - return render_page_template('ologinerror.html', service_name=service_name, - error_message='Unknown error') - + return render_ologin_error(service_name) def get_google_username(user_data): username = user_data['email'] @@ -120,17 +122,16 @@ def get_google_username(user_data): def google_oauth_callback(): error = request.args.get('error', None) if error: - return render_page_template('ologinerror.html', service_name='Google', error_message=error) + return render_ologin_error('Google', error) token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE', form_encode=True) user_data = get_google_user(token) if not user_data or not user_data.get('id', None) or not user_data.get('email', None): - return render_page_template('ologinerror.html', service_name = 'Google', - error_message='Could not load user data') - + return render_ologin_error('Google') + username = get_google_username(user_data) metadata = { - 'service_username': username + 'service_username': user_data['email'] } return conduct_oauth_login('Google', user_data['id'], username, user_data['email'], @@ -142,13 +143,12 @@ def google_oauth_callback(): def github_oauth_callback(): error = request.args.get('error', None) if error: - return render_page_template('ologinerror.html', service_name = 'GitHub', error_message=error) + return render_ologin_error('GitHub', error) token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB') user_data = get_github_user(token) if not user_data: - return render_page_template('ologinerror.html', service_name = 'GitHub', - error_message='Could not load user data') + return render_ologin_error('GitHub') username = user_data['login'] github_id = user_data['id'] @@ -186,15 +186,14 @@ def google_oauth_attach(): user_data = get_google_user(token) if not user_data or not user_data.get('id', None): - return render_page_template('ologinerror.html', service_name = 'Google', - error_message='Could not load user data') + return render_ologin_error('Google') google_id = user_data['id'] user_obj = current_user.db_user() username = get_google_username(user_data) metadata = { - 'service_username': username + 'service_username': user_data['email'] } try: @@ -202,9 +201,7 @@ def google_oauth_attach(): except IntegrityError: err = 'Google account %s is already attached to a %s account' % ( username, app.config['REGISTRY_TITLE_SHORT']) - - return render_page_template('ologinerror.html', service_name = 'Google', - error_message=err) + return render_ologin_error('Google', err) return redirect(url_for('web.user')) @@ -216,8 +213,7 @@ def github_oauth_attach(): token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB') user_data = get_github_user(token) if not user_data: - return render_page_template('ologinerror.html', service_name = 'GitHub', - error_message='Could not load user data') + return render_ologin_error('GitHub') github_id = user_data['id'] user_obj = current_user.db_user() @@ -233,8 +229,7 @@ def github_oauth_attach(): err = 'Github account %s is already attached to a %s account' % ( username, app.config['REGISTRY_TITLE_SHORT']) - return render_page_template('ologinerror.html', service_name = 'Github', - error_message=err) + return render_ologin_error('GitHub', err) return redirect(url_for('web.user')) diff --git a/templates/ologinerror.html b/templates/ologinerror.html index cd921eec2..304b7f554 100644 --- a/templates/ologinerror.html +++ b/templates/ologinerror.html @@ -15,7 +15,7 @@ {% endif %}
    - Please register using the registration form to continue. + Please register using the registration form to continue. You will be able to connect your account to your Quay.io account in the user settings.
    From b9a4d2835f89da831abf3ce35ce79808a2cd79b4 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 4 Sep 2014 18:18:19 -0400 Subject: [PATCH 31/49] Add migration for the new DB field --- ...a_add_metadata_field_to_external_logins.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py diff --git a/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py b/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py new file mode 100644 index 000000000..c642dcbee --- /dev/null +++ b/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py @@ -0,0 +1,26 @@ +"""add metadata field to external logins + +Revision ID: 1594a74a74ca +Revises: f42b0ea7a4d +Create Date: 2014-09-04 18:17:35.205698 + +""" + +# revision identifiers, used by Alembic. +revision = '1594a74a74ca' +down_revision = 'f42b0ea7a4d' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('federatedlogin', sa.Column('metadata_json', sa.Text(), nullable=False)) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('federatedlogin', 'metadata_json') + ### end Alembic commands ### From 6fa5a365b3f664346d150fd18a7fd0255194034b Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 4 Sep 2014 18:45:23 -0400 Subject: [PATCH 32/49] Add loginservice for Google --- ...4ca_add_metadata_field_to_external_logins.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py b/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py index c642dcbee..a59116c7f 100644 --- a/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py +++ b/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py @@ -12,6 +12,9 @@ down_revision = 'f42b0ea7a4d' from alembic import op import sqlalchemy as sa +from sqlalchemy.dialects import mysql +from data.model.sqlalchemybridge import gen_sqlalchemy_metadata +from data.database import all_models def upgrade(): @@ -19,8 +22,22 @@ def upgrade(): op.add_column('federatedlogin', sa.Column('metadata_json', sa.Text(), nullable=False)) ### end Alembic commands ### + schema = gen_sqlalchemy_metadata(all_models) + + op.bulk_insert(schema.tables['loginservice'], + [ + {'id':4, 'name':'google'}, + ]) def downgrade(): ### commands auto generated by Alembic - please adjust! ### op.drop_column('federatedlogin', 'metadata_json') ### end Alembic commands ### + + schema = gen_sqlalchemy_metadata(all_models) + loginservice = schema.table['loginservice'] + + op.execute( + (loginservice.delete() + .where(loginservice.c.name == op.inline_literal('google'))) + ) From 1a230f635af589cf75a8a65aae0fcf8ba2a6348e Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Thu, 4 Sep 2014 19:15:06 -0400 Subject: [PATCH 33/49] Use datetime.min instead of a fixed span for the last login default time. --- endpoints/api/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index 9f9a8c941..a9d2ecdb0 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -282,7 +282,7 @@ def require_fresh_login(func): logger.debug('Checking fresh login for user %s', user.username) - last_login = session.get('login_time', datetime.datetime.now() - datetime.timedelta(minutes=60)) + last_login = session.get('login_time', datetime.datetime.min) valid_span = datetime.datetime.now() - datetime.timedelta(minutes=10) if last_login >= valid_span: From 987177fd7e45800a549e2fed6e410741abf226de Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 4 Sep 2014 19:47:12 -0400 Subject: [PATCH 34/49] Have require_fresh_login not apply if there is no password set for the user --- endpoints/api/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index a9d2ecdb0..2f5e2045e 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -285,7 +285,7 @@ def require_fresh_login(func): last_login = session.get('login_time', datetime.datetime.min) valid_span = datetime.datetime.now() - datetime.timedelta(minutes=10) - if last_login >= valid_span: + if not user.password_hash or last_login >= valid_span: return func(*args, **kwargs) raise FreshLoginRequired() From f746eb33819fcf1ba66bbd75c02991792cdbf114 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 4 Sep 2014 20:04:49 -0400 Subject: [PATCH 35/49] Make the fresh login dialog autofocus the input and make it handle the enter key properly. --- static/js/app.js | 58 +++++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/static/js/app.js b/static/js/app.js index d23235a66..e0b431d7b 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -816,7 +816,31 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading // If the error is a fresh login required, show the dialog. if (resp.status == 401 && resp.data['error_type'] == 'fresh_login_required') { - bootbox.dialog({ + var verifyNow = function() { + if (!$('#freshPassword').val()) { return; } + + var info = { + 'password': $('#freshPassword').val() + }; + + $('#freshPassword').val(''); + + // Conduct the sign in of the user. + apiService.verifyUser(info).then(function() { + // On success, retry the operation. if it succeeds, then resolve the + // deferred promise with the result. Otherwise, reject the same. + apiService[opName].apply(apiService, opArgs).then(function(resp) { + deferred.resolve(resp); + }, function(resp) { + deferred.reject(resp); + }); + }, function(resp) { + // Reject with the sign in error. + deferred.reject({'data': {'message': 'Invalid verification credentials'}}); + }); + }; + + var box = bootbox.dialog({ "message": 'It has been more than a few minutes since you last logged in, ' + 'so please verify your password to perform this sensitive operation:' + '' + @@ -827,38 +851,26 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading "verify": { "label": "Verify", "className": "btn-success", - "callback": function() { - var info = { - 'password': $('#freshPassword').val() - }; - - $('#freshPassword').val(''); - - // Conduct the sign in of the user. - apiService.verifyUser(info).then(function() { - // On success, retry the operation. if it succeeds, then resolve the - // deferred promise with the result. Otherwise, reject the same. - apiService[opName].apply(apiService, opArgs).then(function(resp) { - deferred.resolve(resp); - }, function(resp) { - deferred.reject(resp); - }); - }, function(resp) { - // Reject with the sign in error. - deferred.reject({'data': {'message': 'Invalid verification credentials'}}); - }); - } + "callback": verifyNow }, "close": { "label": "Cancel", "className": "btn-default", "callback": function() { - deferred.reject(resp); + deferred.reject({'data': {'message': 'Verification canceled'}}); } } } }); + box.bind('shown.bs.modal', function(){ + box.find("input").focus(); + box.find("form").submit(function() { + box.modal('hide'); + verifyNow(); + }); + }); + // Return a new promise. We'll accept or reject it based on the result // of the login. return deferred.promise; From 4e04ad5ca7f4192f23040899641178e46b02d02e Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 4 Sep 2014 20:05:21 -0400 Subject: [PATCH 36/49] Move the password check before we hide the modal --- static/js/app.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/static/js/app.js b/static/js/app.js index e0b431d7b..97568e7fb 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -817,8 +817,6 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading // If the error is a fresh login required, show the dialog. if (resp.status == 401 && resp.data['error_type'] == 'fresh_login_required') { var verifyNow = function() { - if (!$('#freshPassword').val()) { return; } - var info = { 'password': $('#freshPassword').val() }; @@ -866,6 +864,8 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading box.bind('shown.bs.modal', function(){ box.find("input").focus(); box.find("form").submit(function() { + if (!$('#freshPassword').val()) { return; } + box.modal('hide'); verifyNow(); }); From 19a589ba54d64944a3e6777cd1cb692d9797291c Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Thu, 4 Sep 2014 20:11:42 -0400 Subject: [PATCH 37/49] Update the test db to have the google login service. --- test/data/test.db | Bin 231424 -> 231424 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/test/data/test.db b/test/data/test.db index 3e631b5d72e5c2c290e95affc3a9794853a6a573..f76110cc23548c74a99fe0ce64bfd130c91fc2aa 100644 GIT binary patch delta 6273 zcmbuCeRvery~lUvOxWxuAqG$k5(pt62~INaFDX)Y*|%hqeMz#(?%D`5yE6pBCSgeo z32Y(1+); ze{8bzJ1?K#_x!$R&RMs4{JPEKH(!SZ?wPs(3p~BZ0NCtZzXuh8LIS*D1D}D9z`Nj$ zT_ra>4{o#*z;^Yrw9;D$fNlEneq}J*qxSSUa9}H4yri$QUv*&Czdb#RmuXI$%|?L3 zHt;bx2@ZoJXowfU@Ov%!l|&Jkj-BldoS+mt-5Ym+Qpf4uJ6jzd`=j_b@eP37b=6i5 zUmN!GUKi&HGE!?}vlyVI03QhQbUPnvr2T%mt)AoA#ujft@Up>1IYtL%)*WKyxIY{d z++HD3=p~FItX&KQTB5ulgxjQ;ONc~VR-2HozO9){NUd>qAQXs(d9HCmlebk2#_HMj zh+Bxn<91)NUX-Kq@aXqDZ;HqHWVBrt>Jwhk=cc10gr916i?N1qG?;WZR2@ zYewN-d~J}h+c3;1Dk%h4D&49zL694)09>YmYsX5MQxJ`Y=SIr|SE?i$49g9MgYy;j z{eL7~wWi3bTHB{K+oyOG|21eBo>BNm=lbMK`{b%B?AtxVgHIfUy=6GLt7&^NzH{U_ zmIl#`R9EANB_)$;=mZQaZ9fyyM;Pxx7V8Wk=))Twj6}Wf-*2d$x z6#%(`u2g*7X8}1b(EY1rpYqkEyz`kqfhnMnuoD=PvH+a^n}?4>za0QKpTNcu&cEAi zf5#rizW}SyqI@*`tsRf;diX#`(HvS5m`FX-=1$O!u7nhF$Hn$gy}Lcm2ZeeMAC)3u z!R3mz4L^5i)^OlZcuw5qrV{Qr6;A}4U5RLm$Rs$qt<4)sMx&e@X^6JP8{*;m_^#&; zohXjV950EoK#{7hFeE3l5~(QYi`EoIOYv$tMR8fJx2H}?XO?vLX)F5rdb+f3hN3x= zl1Yj$r|0rLzQVXJ1?e0Qe|igP0tCWno`mvqo)-y zMbjeBv+#WZqt^PeOKvDf)3s>5<<{)w6jeu4bv#o}Qs`50LwQAC&%(6Qr&ZLISBz~< zU(kBZ*#tN=+UZaY%gU0(8U*IB#W07F=gLo|8Ie&{ie#tg5>Ilvs3AU0C)0wYN-QTc zDOQBrMa)-n!8n{u(WZpGRzO|w*cu)$11`sD zgP7wS?N`H^jo3=CHkGDzJ}shUW_Xe0gfvB>MVCpMPEl+s&1$li26!!=Z^L2^w9h&f zRiRW`q)47gqYf%9krJcu=qJr^T8dG%G>d~;>{JKLhB+lo(kO)zNM6y=!t%OE%A%s8 zFICd)DyIvQNbUJo7z@v-R5ePHRE{TgjZ2YSnnTU_6idpwEXfQi1ujLwO!H(8C{}OHZ7q_gu{cFyHZ1|t>}tG@*JuNPU2}& zVPuV@(u|;@`B^#5z`qS*b1Nl*(m7etNr6EH!%1|SRA~kgFp9*bDUsD^0S1OJS0&HO z5{D!qWlhjYjzNTqEYT$LfXGQ2$BDEKpB%!bR_dy#>yoOIdMbrDDdbi~qct)`X{nT; zYl@;u@c0lGsN@xaQaJ|YVO1(8rxa46Re=;G5#@rUW+)yeR$9O2(6evD-5qO z3dO2>_PvEgr=4>);t)7qpkVn)?41i9qZ1d+!Gg~(dW`lnUf6T;bL=tq+Or!0UnBAq zT7$xAz=e%X_Ds0o>%-2k4?8XnuRXoVof$p<;`oJO`?;`0vh7lf&c!+Dno zccQKOOn2h^I3r)&9ejQGybV+$$M!bo;m^-4<+xCkYvtL7q{kauko3*P#@R5Qbl|5> zlvjm9L4T7g+#L7%d7rl_W_IFe*L@3of_B}PLDeKcVi%8qPXwf6Out_I;cFR9ul4AW zfm*FA5NS=eE=~0H(+!Q?UG1@D14?~{%VflXq1uL6)Emfn25Yj(e)!r(d~ykTcb=A= zu0gLkWI6c7M!cMNw=8f42UFaN=18I?BSf=wQ)5_*2NX3G>e8B{Nl$m8xmJpE-Aniu zaUdhqGYi77Y14%xvJ@|JGVI@k&q7C|D?8AitSnJ`~5GJqh-F2QT9X`r`e8!3EyM?R~wvkjNxE6HE1QXQQ9d zTYYJ@D;Z*kc%Pmb2o5Fnl^(5M>8xeof$!im)}H03YDAf0czOUYvXM5UDBqUvK>TU# z);>jDrp-;ElC})v4hI)EDWZ@H zW}<_=HQ{i)zP_!;zeJ7&Ljy`ni{8j;Qdg>PB|X4I0_|1}*Pc^{s^J;5|A^*y@%OQX zS9%+|2pJJuCp(GZn4y9q8w(G751%xA^ROd%nKz}2&*^86>6e;_m09$DhUQ)Pf^+$y zZC7MDN`%h4kvm$K$7^(^ry=a+vNirD@6vEy#s^Nt^H~!y0pHVV%i!5g1gyY z8|$oX5SrWJ!n^Uw+S%1-YA8YCIe8$z$X06PRpqKTt10QTr8`)Yq~n?MJwq+sJ@lB;l_;4OD04I3jS&@* z9Us8#J@|3^#Ai;77#SQMc>%u>{^(wO#C~4=Y%7#p?({_kKaz@?NtCQeh@!S@IO3enh)XK4r1Q%|I}f2 z8{%_rxPFt784r&fw)lRG_$03H+dg_{PT{Tmwg+Mtoou z=P}AA!2KUsd^-_eu}HKVdI?~9G!joCc4iCig7=7i*29nYIHS4pY7=O zvkgAG8-IwHF!d+~JARHIyNVllWYF+lSo|{Lb51K?WWHkYIrFys0cKxE zeB4`}qef;nJaV7K_iMyg^ZT%2RL_F1Zn5~@Kz!E~>z|s3tsNdmQ1_<{JY>hJi<(vheqx#02h3sG{QTXxqLS{Zd>Rsj8@jE!` z0Nd?XRX#jql(};ja)J(f^|D9j7`iKGp#rdtn0wP-lV%}6coR`_u%iSVvzOd`SDoSW zFU*!Az8MQw{KUvK+;AinD-ZkcUV$8zqdMSP6h_8%r?@sXwfUIa&J#5d!`wGS9&$(%bF#K$C8 zD~6uPxr0S~bDSFAaU(Vvk@S7T(?(y33hZKJ}15B4#T%J=Z;$tAMyU0 zDX@4U;>-V14V#NBKK3^&N?>*g;^SiLvqq*nXOE?buc~x?w^5zR*`o{b3HFVDF_&5T zv1^(hhoc$9H=p`;zEPIV*`pirIX9F4W9Z9s_UJ);Rd+r2so87khdpw?2|Jb}zUjW1 ze>Qx#=IoJ0eEg2BTVe4E#3%XhJZbiUh2!iXm?||gLpfIrA*%8B92zmI2Q633hxe@l zodm(mi<+xJggAGJIde9L$~xe`#N*!?U=E}VK)_?`z?@4oK>emb$pG$4#8BrdxZ41w zmyX>uVeA#d;+O&()`K|{*fAFjc#39ZJ*d7^@PqZBwPeh#!W!^usNM!@FA;n6Hc&VA Y@t)(ifd+61=D;0bc-CcHAS>4VA9FG*y8r+H delta 6218 zcmbuCeRvery~lUvY_i#e5Q1~3Pfuw{fsNXRaDiGK|d|lvZOyQQuU0Va)q7#S|Jb% z^Kr>H*!dvmuBh^OgT8t(QWSGjxezKga-sKI)8$3ZL7l^prwY9GL zs#r`6hr&!nkaTfTd z1LrE`;NL;LO_$1-R?ml@E0uv?sf0f4TqPWzSjTI=y^y)6K{G6E7E2rU3|@{u24McZ zcxaHEG$nh>H4AJr7B9wbrUp+xaRBDrj!lM7=V3G~KZX?yuXyU0*x=tscI_D&18{5; zHe=ZOY=+g=0sa&JEm(|};Y}R=WC+jPbj3?x^=6=@#J+kDnITq zf$aZ6{BhdP{)R5z^THp%Y;xn4rAR`s~y{E!FJ-Ozzs;$ z9oX>B{f)VcJiOp{M?!)a47lT*BjSk&j9A5qbht8FT~p6ji;Tego$4d!u?0 z>xoiygWFkKAL86BE4dqdQpo2Zd75^+!gQ4sp{rvx4b{Uhzwt?4nCC@~A^8-cvJ^$I zoRA`tyv!0QmE~DlQW;v&I@R{J<+9S!+}f>nb$7R|S6gY4VhK_rNV3R;R_WtAr6q$&{tOEL_jNn}zM;RifMZmR83 zmlmPnqH24QIeHN#EvNbAELTL3=qg@XRNURRT9Lce;^jrfmpx6BmppybMH8U7b0IVV zNs@ToirFo7m|f3sWF?ad&kKw~u(N21Ay`@#2vJOl1dq5vsD1k@6Xp&(Cn<+FciTh4h zW7UgF1X^JgMPLXrnG^|DLd9jCqX?QudaDwblr#x;^kJ?NMN=tWMfz)u$Pp~hON7jl zGQny|CB?G>rSK&Dbsxr+@S;MiG#ZdsNHo5dN)e(g2!tj|8oEfHXGKW&W2+X3DHLR7 zf}?pfxF~3ZB&8@qW<*(&Q=*s@1vuD`IZJ3UmC{HB6=Hd$L5g8b(O5!}DN$ons;Dp= z{IVZgQo^EKF^SYt&{L=;&k!n)K`RoSM3ZDxMS=}CU}YteO7pDBGK5Mps0~CJnV%B{Z7rBK(1tc-*gDN*7?hNhB9<3?;()pb?gusarE{Q;>$ViGYX zTE|lkDOl%YMXsJBJ#nfw%*F%ZifGhJ2{rzh!^0$8bgaf*@1vt!B-*@%PV z1x|!TpJT@^xy?$55axU_{x;2V!oJVHz;-z|eZ3J#+3K2O{gq5XE|`TH9dFZTPf{^U8lU!sFr@90yt^|GgRnOx`aYw-pl*weGNt-OX!P)%}s z2QRk3{kP-uO%tByrv#Ccn2mU@g|O(kS(Yq2;#bt(ZaKL@U7~cK{l1&{&^_+l_YKFs zal+*w6R_nJFVO-GH*Ck}as6r)84$#{vv+N?n+qp5bg%7et_~=Q(;rd&{WXFowZYBR z^~I##09n=J4{05hJ!TItyPytJE>V<7lg1tR2iWQ>{R|q1jEK?6jv_@y9LUMFFmV45 z@o9tC4cZgm;VbE)L;8g?`a}bv!zjxz(6|$?xG+Do?V?i#MZ@g7kULye@yLd*s?M7DinAK7z6+nBe!co!ImL?7!qJ=Tt!dgQ`^tHicQr|V!JlX5Dvb1A;tZu+x)7;udjcZ-5oRS1dq>LSSD`w4asapU$@5aY$xnoC{=`A=s{!4r% zeDofC$W|2EzE>~9;Ne&B74V)xe4RCa;J;QI_u>aIYvEI`)xy#H@F#66{{uUs7iPkd zgXk&aM|h(xYr(okUCV%LUPG1Q5Z`T`x!Uo8aX;RGSr>-xsez3@#y_%^KKyB)?#YHj zM^N6j2k;+Q?IX`T4f7tv$E-yw`;*2)c&pvI%FF&5cJ4-eS)awL^p?r+_**95V~CHf z=(<@i%Y}#EHu)Y$d{Yixmu)SYD+@Mk9D z%ZRb8rO;-K;0H3S)T`DT;pi)fac<|xF}-ju9QmiocmOfx-P!glT`PoZzA_nKMU1OX zjTRXPO%0cv{P)SQ@ioLZ1^o3P-BS#QcA&g%qe#OA?=F59<{d(OCH*y@7_Xar*qysR zft_z6z6F12n4-5_4UgYz^8E_&<-T~^B)x1QJba(Y_uq)`>SzCW!g$MEe^zs=Eo-AJBV+}$)-n*qj*EUb>7U6Ddgn55Lc=#V4dd2wIY{A?o9(VzEjv+kz=&9{`3k{DyYvTPG z;pLq9`Dwk3f`^|o@&1DFGT+@c-T1_m&mJh40!KeXc++or>{Gpvhx?y5@jgd*%kR5; zr>=2O-fQB0VJ`6tq|Z2oH)L4BmiPY&8^1z$(%Rqd*FDSO(9cY~(@4IBWb*--cLwq0 zeYz(3JV5G+S4}+%J2SwTZONgY2EAof+98=>$Tod=RzNRXnRbW`th3@=(P<+KAoa*E z_pO7YcJQPv^Omm;=!MR-L$X1m&35?Ia$R$z9g+ifTZ^6=>oX<+gje?D_FUMQ4?eQ3 z*!|P5bdM+Pkg4DY))~)LjKRFCz?hXBrT%10Gx??+IJg*g79hS^+rMz@EtP49%tCxp zXi1q~=1)6hHsa$Cw=Fg1n0&MUmJGwuLX&T+2d!=>?U1>MkGskKsjdan4w;Ae%7bOI zjUv#1*~ky)16_-rllI~HV7IMc*F$@CPXrEqY|2=SIOk(G{~hKnK%7(lZ_P_a2|!Ms z7OG6aP6BaeG=;PEmRQ;)<%mzVu*8L(Jr4Z+gg*zVC z3lnLV(1>%@&WgRd){u4ygE&`e8Sfh`;>~__1e#e4bkLV{CoGau+xF~a-JzXqPHZ|E^(Uc{FAQl>t%AmR;KC%7J zck7;YX_s7Y@@+mi2j;Cte7WDhNif!!eA1j=w9d_lFMs-L$Mlxgv`y9_KH>W9l3v!5 zw#j)tR<}yYzqPEdyzH3?Qoc zE0elj)^ECF6}mamf*Py&F1_@C59Q6kJHhSub+8DMI Date: Thu, 4 Sep 2014 20:58:29 -0400 Subject: [PATCH 38/49] Inject the tables metadata into the upgrade and downgrade functions. Fix a bunch of the downgrades to actually work. --- data/billing.py | 16 +-- data/migrations/env.py | 6 +- data/migrations/script.py.mako | 4 +- ...a_add_metadata_field_to_external_logins.py | 18 +-- ...49_remove_fields_from_image_table_that_.py | 4 +- ...c79d9_prepare_the_database_for_the_new_.py | 56 ++++----- ...9f_add_log_kind_for_regenerating_robot_.py | 18 +-- ...670cbeced_migrate_existing_webhooks_to_.py | 4 +- ...2_add_the_maintenance_notification_type.py | 15 +-- ...add_brute_force_prevention_metadata_to_.py | 4 +- .../5a07499ce53f_set_up_initial_database.py | 108 ++---------------- .../82297d834ad_add_us_west_location.py | 17 +-- ..._add_placements_and_locations_to_the_db.py | 14 +-- ...2b0ea7a4d_remove_the_old_webhooks_table.py | 4 +- util/collections.py | 12 ++ 15 files changed, 80 insertions(+), 220 deletions(-) create mode 100644 util/collections.py diff --git a/data/billing.py b/data/billing.py index 4847dd3f8..8c604aac2 100644 --- a/data/billing.py +++ b/data/billing.py @@ -3,6 +3,8 @@ import stripe from datetime import datetime, timedelta from calendar import timegm +from util.collections import AttrDict + PLANS = [ # Deprecated Plans { @@ -118,20 +120,6 @@ def get_plan(plan_id): return None -class AttrDict(dict): - def __init__(self, *args, **kwargs): - super(AttrDict, self).__init__(*args, **kwargs) - self.__dict__ = self - - @classmethod - def deep_copy(cls, attr_dict): - copy = AttrDict(attr_dict) - for key, value in copy.items(): - if isinstance(value, AttrDict): - copy[key] = cls.deep_copy(value) - return copy - - class FakeStripe(object): class Customer(AttrDict): FAKE_PLAN = AttrDict({ diff --git a/data/migrations/env.py b/data/migrations/env.py index c267c2f50..863e3d98f 100644 --- a/data/migrations/env.py +++ b/data/migrations/env.py @@ -8,6 +8,7 @@ from peewee import SqliteDatabase from data.database import all_models, db from app import app from data.model.sqlalchemybridge import gen_sqlalchemy_metadata +from util.collections import AttrDict # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -23,6 +24,7 @@ fileConfig(config.config_file_name) # from myapp import mymodel # target_metadata = mymodel.Base.metadata target_metadata = gen_sqlalchemy_metadata(all_models) +tables = AttrDict(target_metadata.tables) # other values from the config, defined by the needs of env.py, # can be acquired: @@ -45,7 +47,7 @@ def run_migrations_offline(): context.configure(url=url, target_metadata=target_metadata, transactional_ddl=True) with context.begin_transaction(): - context.run_migrations() + context.run_migrations(tables=tables) def run_migrations_online(): """Run migrations in 'online' mode. @@ -72,7 +74,7 @@ def run_migrations_online(): try: with context.begin_transaction(): - context.run_migrations() + context.run_migrations(tables=tables) finally: connection.close() diff --git a/data/migrations/script.py.mako b/data/migrations/script.py.mako index 95702017e..1b92f9f48 100644 --- a/data/migrations/script.py.mako +++ b/data/migrations/script.py.mako @@ -14,9 +14,9 @@ from alembic import op import sqlalchemy as sa ${imports if imports else ""} -def upgrade(): +def upgrade(tables): ${upgrades if upgrades else "pass"} -def downgrade(): +def downgrade(tables): ${downgrades if downgrades else "pass"} diff --git a/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py b/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py index a59116c7f..2f6c60706 100644 --- a/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py +++ b/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py @@ -13,31 +13,23 @@ down_revision = 'f42b0ea7a4d' from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models - -def upgrade(): +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.add_column('federatedlogin', sa.Column('metadata_json', sa.Text(), nullable=False)) ### end Alembic commands ### - schema = gen_sqlalchemy_metadata(all_models) - - op.bulk_insert(schema.tables['loginservice'], + op.bulk_insert(tables.loginservice, [ {'id':4, 'name':'google'}, ]) -def downgrade(): +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.drop_column('federatedlogin', 'metadata_json') ### end Alembic commands ### - schema = gen_sqlalchemy_metadata(all_models) - loginservice = schema.table['loginservice'] - op.execute( - (loginservice.delete() - .where(loginservice.c.name == op.inline_literal('google'))) + (tables.loginservice.delete() + .where(tables.loginservice.c.name == op.inline_literal('google'))) ) diff --git a/data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py b/data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py index ea36e3f57..d50c3a592 100644 --- a/data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py +++ b/data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py @@ -14,7 +14,7 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(): +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.drop_index('buildtriggerservice_name', table_name='buildtriggerservice') op.create_index('buildtriggerservice_name', 'buildtriggerservice', ['name'], unique=True) @@ -34,7 +34,7 @@ def upgrade(): ### end Alembic commands ### -def downgrade(): +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.drop_index('visibility_name', table_name='visibility') op.create_index('visibility_name', 'visibility', ['name'], unique=False) diff --git a/data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py b/data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py index 18c8bf654..e3be811b6 100644 --- a/data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py +++ b/data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py @@ -13,12 +13,8 @@ down_revision = '4b7ef0c7bdb2' from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models - -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.create_table('externalnotificationmethod', sa.Column('id', sa.Integer(), nullable=False), @@ -26,7 +22,7 @@ def upgrade(): sa.PrimaryKeyConstraint('id') ) op.create_index('externalnotificationmethod_name', 'externalnotificationmethod', ['name'], unique=True) - op.bulk_insert(schema.tables['externalnotificationmethod'], + op.bulk_insert(tables.externalnotificationmethod, [ {'id':1, 'name':'quay_notification'}, {'id':2, 'name':'email'}, @@ -38,7 +34,7 @@ def upgrade(): sa.PrimaryKeyConstraint('id') ) op.create_index('externalnotificationevent_name', 'externalnotificationevent', ['name'], unique=True) - op.bulk_insert(schema.tables['externalnotificationevent'], + op.bulk_insert(tables.externalnotificationevent, [ {'id':1, 'name':'repo_push'}, {'id':2, 'name':'build_queued'}, @@ -77,7 +73,7 @@ def upgrade(): op.add_column(u'notification', sa.Column('dismissed', sa.Boolean(), nullable=False)) # Manually add the new notificationkind types - op.bulk_insert(schema.tables['notificationkind'], + op.bulk_insert(tables.notificationkind, [ {'id':5, 'name':'repo_push'}, {'id':6, 'name':'build_queued'}, @@ -87,7 +83,7 @@ def upgrade(): ]) # Manually add the new logentrykind types - op.bulk_insert(schema.tables['logentrykind'], + op.bulk_insert(tables.logentrykind, [ {'id':39, 'name':'add_repo_notification'}, {'id':40, 'name':'delete_repo_notification'}, @@ -97,61 +93,49 @@ def upgrade(): ### end Alembic commands ### -def downgrade(): - schema = gen_sqlalchemy_metadata(all_models) - +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.drop_column(u'notification', 'dismissed') - op.drop_index('repositorynotification_uuid', table_name='repositorynotification') - op.drop_index('repositorynotification_repository_id', table_name='repositorynotification') - op.drop_index('repositorynotification_method_id', table_name='repositorynotification') - op.drop_index('repositorynotification_event_id', table_name='repositorynotification') op.drop_table('repositorynotification') - op.drop_index('repositoryauthorizedemail_repository_id', table_name='repositoryauthorizedemail') - op.drop_index('repositoryauthorizedemail_email_repository_id', table_name='repositoryauthorizedemail') - op.drop_index('repositoryauthorizedemail_code', table_name='repositoryauthorizedemail') op.drop_table('repositoryauthorizedemail') - op.drop_index('externalnotificationevent_name', table_name='externalnotificationevent') op.drop_table('externalnotificationevent') - op.drop_index('externalnotificationmethod_name', table_name='externalnotificationmethod') op.drop_table('externalnotificationmethod') # Manually remove the notificationkind and logentrykind types - notificationkind = schema.tables['notificationkind'] op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('repo_push'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('repo_push'))) ) op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('build_queued'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('build_queued'))) ) op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('build_start'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('build_start'))) ) op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('build_success'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('build_success'))) ) op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('build_failure'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('build_failure'))) ) op.execute( - (logentrykind.delete() - .where(logentrykind.c.name == op.inline_literal('add_repo_notification'))) + (tables.logentrykind.delete() + .where(tables.logentrykind.c.name == op.inline_literal('add_repo_notification'))) ) op.execute( - (logentrykind.delete() - .where(logentrykind.c.name == op.inline_literal('delete_repo_notification'))) + (tables.logentrykind.delete() + .where(tables.logentrykind.c.name == op.inline_literal('delete_repo_notification'))) ) ### end Alembic commands ### diff --git a/data/migrations/versions/43e943c0639f_add_log_kind_for_regenerating_robot_.py b/data/migrations/versions/43e943c0639f_add_log_kind_for_regenerating_robot_.py index 6ee041e4c..f676bf972 100644 --- a/data/migrations/versions/43e943c0639f_add_log_kind_for_regenerating_robot_.py +++ b/data/migrations/versions/43e943c0639f_add_log_kind_for_regenerating_robot_.py @@ -13,25 +13,17 @@ down_revision = '82297d834ad' from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models - -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) - - op.bulk_insert(schema.tables['logentrykind'], +def upgrade(tables): + op.bulk_insert(tables.logentrykind, [ {'id': 41, 'name':'regenerate_robot_token'}, ]) -def downgrade(): - schema = gen_sqlalchemy_metadata(all_models) - - logentrykind = schema.tables['logentrykind'] +def downgrade(tables): op.execute( - (logentrykind.delete() - .where(logentrykind.c.name == op.inline_literal('regenerate_robot_token'))) + (tables.logentrykind.delete() + .where(tables.logentrykind.c.name == op.inline_literal('regenerate_robot_token'))) ) diff --git a/data/migrations/versions/47670cbeced_migrate_existing_webhooks_to_.py b/data/migrations/versions/47670cbeced_migrate_existing_webhooks_to_.py index 726145167..eaa687c73 100644 --- a/data/migrations/versions/47670cbeced_migrate_existing_webhooks_to_.py +++ b/data/migrations/versions/47670cbeced_migrate_existing_webhooks_to_.py @@ -18,13 +18,13 @@ def get_id(query): conn = op.get_bind() return list(conn.execute(query, ()).fetchall())[0][0] -def upgrade(): +def upgrade(tables): conn = op.get_bind() event_id = get_id('Select id From externalnotificationevent Where name=\'repo_push\' Limit 1') method_id = get_id('Select id From externalnotificationmethod Where name=\'webhook\' Limit 1') conn.execute('Insert Into repositorynotification (uuid, repository_id, event_id, method_id, config_json) Select public_id, repository_id, %s, %s, parameters FROM webhook' % (event_id, method_id)) -def downgrade(): +def downgrade(tables): conn = op.get_bind() event_id = get_id('Select id From externalnotificationevent Where name=\'repo_push\' Limit 1') method_id = get_id('Select id From externalnotificationmethod Where name=\'webhook\' Limit 1') diff --git a/data/migrations/versions/4b7ef0c7bdb2_add_the_maintenance_notification_type.py b/data/migrations/versions/4b7ef0c7bdb2_add_the_maintenance_notification_type.py index 9e5fff425..9f48ca6c6 100644 --- a/data/migrations/versions/4b7ef0c7bdb2_add_the_maintenance_notification_type.py +++ b/data/migrations/versions/4b7ef0c7bdb2_add_the_maintenance_notification_type.py @@ -11,23 +11,18 @@ revision = '4b7ef0c7bdb2' down_revision = 'bcdde200a1b' from alembic import op -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models import sqlalchemy as sa - -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) - op.bulk_insert(schema.tables['notificationkind'], +def upgrade(tables): + op.bulk_insert(tables.notificationkind, [ {'id':4, 'name':'maintenance'}, ]) -def downgrade(): - notificationkind = schema.tables['notificationkind'] +def downgrade(tables): op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('maintenance'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('maintenance'))) ) diff --git a/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py b/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py index a1c8c95dd..1ce802eca 100644 --- a/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py +++ b/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py @@ -14,14 +14,14 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(): +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.add_column('user', sa.Column('invalid_login_attempts', sa.Integer(), nullable=False, server_default="0")) op.add_column('user', sa.Column('last_invalid_login', sa.DateTime(), nullable=False, server_default=sa.func.now())) ### end Alembic commands ### -def downgrade(): +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.drop_column('user', 'last_invalid_login') op.drop_column('user', 'invalid_login_attempts') diff --git a/data/migrations/versions/5a07499ce53f_set_up_initial_database.py b/data/migrations/versions/5a07499ce53f_set_up_initial_database.py index ffc9d28e6..f67224645 100644 --- a/data/migrations/versions/5a07499ce53f_set_up_initial_database.py +++ b/data/migrations/versions/5a07499ce53f_set_up_initial_database.py @@ -11,14 +11,9 @@ revision = '5a07499ce53f' down_revision = None from alembic import op -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models import sqlalchemy as sa - -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) - +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.create_table('loginservice', sa.Column('id', sa.Integer(), nullable=False), @@ -27,7 +22,7 @@ def upgrade(): ) op.create_index('loginservice_name', 'loginservice', ['name'], unique=True) - op.bulk_insert(schema.tables['loginservice'], + op.bulk_insert(tables.loginservice, [ {'id':1, 'name':'github'}, {'id':2, 'name':'quayrobot'}, @@ -66,7 +61,7 @@ def upgrade(): ) op.create_index('role_name', 'role', ['name'], unique=False) - op.bulk_insert(schema.tables['role'], + op.bulk_insert(tables.role, [ {'id':1, 'name':'admin'}, {'id':2, 'name':'write'}, @@ -80,7 +75,7 @@ def upgrade(): ) op.create_index('logentrykind_name', 'logentrykind', ['name'], unique=False) - op.bulk_insert(schema.tables['logentrykind'], + op.bulk_insert(tables.logentrykind, [ {'id':1, 'name':'account_change_plan'}, {'id':2, 'name':'account_change_cc'}, @@ -136,7 +131,7 @@ def upgrade(): ) op.create_index('notificationkind_name', 'notificationkind', ['name'], unique=False) - op.bulk_insert(schema.tables['notificationkind'], + op.bulk_insert(tables.notificationkind, [ {'id':1, 'name':'password_required'}, {'id':2, 'name':'over_private_usage'}, @@ -150,7 +145,7 @@ def upgrade(): ) op.create_index('teamrole_name', 'teamrole', ['name'], unique=False) - op.bulk_insert(schema.tables['teamrole'], + op.bulk_insert(tables.teamrole, [ {'id':1, 'name':'admin'}, {'id':2, 'name':'creator'}, @@ -164,7 +159,7 @@ def upgrade(): ) op.create_index('visibility_name', 'visibility', ['name'], unique=False) - op.bulk_insert(schema.tables['visibility'], + op.bulk_insert(tables.visibility, [ {'id':1, 'name':'public'}, {'id':2, 'name':'private'}, @@ -194,7 +189,7 @@ def upgrade(): ) op.create_index('buildtriggerservice_name', 'buildtriggerservice', ['name'], unique=False) - op.bulk_insert(schema.tables['buildtriggerservice'], + op.bulk_insert(tables.buildtriggerservice, [ {'id':1, 'name':'github'}, ]) @@ -490,119 +485,34 @@ def upgrade(): ### end Alembic commands ### -def downgrade(): +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### - op.drop_index('repositorybuild_uuid', table_name='repositorybuild') - op.drop_index('repositorybuild_trigger_id', table_name='repositorybuild') - op.drop_index('repositorybuild_resource_key', table_name='repositorybuild') - op.drop_index('repositorybuild_repository_id', table_name='repositorybuild') - op.drop_index('repositorybuild_pull_robot_id', table_name='repositorybuild') - op.drop_index('repositorybuild_access_token_id', table_name='repositorybuild') op.drop_table('repositorybuild') - op.drop_index('repositorybuildtrigger_write_token_id', table_name='repositorybuildtrigger') - op.drop_index('repositorybuildtrigger_service_id', table_name='repositorybuildtrigger') - op.drop_index('repositorybuildtrigger_repository_id', table_name='repositorybuildtrigger') - op.drop_index('repositorybuildtrigger_pull_robot_id', table_name='repositorybuildtrigger') - op.drop_index('repositorybuildtrigger_connected_user_id', table_name='repositorybuildtrigger') op.drop_table('repositorybuildtrigger') - op.drop_index('logentry_repository_id', table_name='logentry') - op.drop_index('logentry_performer_id', table_name='logentry') - op.drop_index('logentry_kind_id', table_name='logentry') - op.drop_index('logentry_datetime', table_name='logentry') - op.drop_index('logentry_account_id', table_name='logentry') - op.drop_index('logentry_access_token_id', table_name='logentry') op.drop_table('logentry') - op.drop_index('repositorytag_repository_id_name', table_name='repositorytag') - op.drop_index('repositorytag_repository_id', table_name='repositorytag') - op.drop_index('repositorytag_image_id', table_name='repositorytag') op.drop_table('repositorytag') - op.drop_index('permissionprototype_role_id', table_name='permissionprototype') - op.drop_index('permissionprototype_org_id_activating_user_id', table_name='permissionprototype') - op.drop_index('permissionprototype_org_id', table_name='permissionprototype') - op.drop_index('permissionprototype_delegate_user_id', table_name='permissionprototype') - op.drop_index('permissionprototype_delegate_team_id', table_name='permissionprototype') - op.drop_index('permissionprototype_activating_user_id', table_name='permissionprototype') op.drop_table('permissionprototype') - op.drop_index('image_storage_id', table_name='image') - op.drop_index('image_repository_id_docker_image_id', table_name='image') - op.drop_index('image_repository_id', table_name='image') - op.drop_index('image_ancestors', table_name='image') op.drop_table('image') - op.drop_index('oauthauthorizationcode_code', table_name='oauthauthorizationcode') - op.drop_index('oauthauthorizationcode_application_id', table_name='oauthauthorizationcode') op.drop_table('oauthauthorizationcode') - op.drop_index('webhook_repository_id', table_name='webhook') - op.drop_index('webhook_public_id', table_name='webhook') op.drop_table('webhook') - op.drop_index('teammember_user_id_team_id', table_name='teammember') - op.drop_index('teammember_user_id', table_name='teammember') - op.drop_index('teammember_team_id', table_name='teammember') op.drop_table('teammember') - op.drop_index('oauthaccesstoken_uuid', table_name='oauthaccesstoken') - op.drop_index('oauthaccesstoken_refresh_token', table_name='oauthaccesstoken') - op.drop_index('oauthaccesstoken_authorized_user_id', table_name='oauthaccesstoken') - op.drop_index('oauthaccesstoken_application_id', table_name='oauthaccesstoken') - op.drop_index('oauthaccesstoken_access_token', table_name='oauthaccesstoken') op.drop_table('oauthaccesstoken') - op.drop_index('repositorypermission_user_id_repository_id', table_name='repositorypermission') - op.drop_index('repositorypermission_user_id', table_name='repositorypermission') - op.drop_index('repositorypermission_team_id_repository_id', table_name='repositorypermission') - op.drop_index('repositorypermission_team_id', table_name='repositorypermission') - op.drop_index('repositorypermission_role_id', table_name='repositorypermission') - op.drop_index('repositorypermission_repository_id', table_name='repositorypermission') op.drop_table('repositorypermission') - op.drop_index('accesstoken_role_id', table_name='accesstoken') - op.drop_index('accesstoken_repository_id', table_name='accesstoken') - op.drop_index('accesstoken_code', table_name='accesstoken') op.drop_table('accesstoken') - op.drop_index('repository_visibility_id', table_name='repository') - op.drop_index('repository_namespace_name', table_name='repository') op.drop_table('repository') - op.drop_index('team_role_id', table_name='team') - op.drop_index('team_organization_id', table_name='team') - op.drop_index('team_name_organization_id', table_name='team') - op.drop_index('team_name', table_name='team') op.drop_table('team') - op.drop_index('emailconfirmation_user_id', table_name='emailconfirmation') - op.drop_index('emailconfirmation_code', table_name='emailconfirmation') op.drop_table('emailconfirmation') - op.drop_index('notification_uuid', table_name='notification') - op.drop_index('notification_target_id', table_name='notification') - op.drop_index('notification_kind_id', table_name='notification') - op.drop_index('notification_created', table_name='notification') op.drop_table('notification') - op.drop_index('oauthapplication_organization_id', table_name='oauthapplication') - op.drop_index('oauthapplication_client_id', table_name='oauthapplication') op.drop_table('oauthapplication') - op.drop_index('federatedlogin_user_id', table_name='federatedlogin') - op.drop_index('federatedlogin_service_id_user_id', table_name='federatedlogin') - op.drop_index('federatedlogin_service_id_service_ident', table_name='federatedlogin') - op.drop_index('federatedlogin_service_id', table_name='federatedlogin') op.drop_table('federatedlogin') - op.drop_index('buildtriggerservice_name', table_name='buildtriggerservice') op.drop_table('buildtriggerservice') - op.drop_index('user_username', table_name='user') - op.drop_index('user_stripe_id', table_name='user') - op.drop_index('user_robot', table_name='user') - op.drop_index('user_organization', table_name='user') - op.drop_index('user_email', table_name='user') op.drop_table('user') - op.drop_index('visibility_name', table_name='visibility') op.drop_table('visibility') - op.drop_index('teamrole_name', table_name='teamrole') op.drop_table('teamrole') - op.drop_index('notificationkind_name', table_name='notificationkind') op.drop_table('notificationkind') - op.drop_index('logentrykind_name', table_name='logentrykind') op.drop_table('logentrykind') - op.drop_index('role_name', table_name='role') op.drop_table('role') - op.drop_index('queueitem_queue_name', table_name='queueitem') - op.drop_index('queueitem_processing_expires', table_name='queueitem') - op.drop_index('queueitem_available_after', table_name='queueitem') - op.drop_index('queueitem_available', table_name='queueitem') op.drop_table('queueitem') op.drop_table('imagestorage') - op.drop_index('loginservice_name', table_name='loginservice') op.drop_table('loginservice') ### end Alembic commands ### diff --git a/data/migrations/versions/82297d834ad_add_us_west_location.py b/data/migrations/versions/82297d834ad_add_us_west_location.py index 59eb1f800..b939a939e 100644 --- a/data/migrations/versions/82297d834ad_add_us_west_location.py +++ b/data/migrations/versions/82297d834ad_add_us_west_location.py @@ -13,24 +13,17 @@ down_revision = '47670cbeced' from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models - -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) - - op.bulk_insert(schema.tables['imagestoragelocation'], +def upgrade(tables): + op.bulk_insert(tables.imagestoragelocation, [ {'id':8, 'name':'s3_us_west_1'}, ]) -def downgrade(): - schema = gen_sqlalchemy_metadata(all_models) - +def downgrade(tables): op.execute( - (imagestoragelocation.delete() - .where(imagestoragelocation.c.name == op.inline_literal('s3_us_west_1'))) + (tables.imagestoragelocation.delete() + .where(tables.imagestoragelocation.c.name == op.inline_literal('s3_us_west_1'))) ) diff --git a/data/migrations/versions/bcdde200a1b_add_placements_and_locations_to_the_db.py b/data/migrations/versions/bcdde200a1b_add_placements_and_locations_to_the_db.py index eda4b2840..9fc433126 100644 --- a/data/migrations/versions/bcdde200a1b_add_placements_and_locations_to_the_db.py +++ b/data/migrations/versions/bcdde200a1b_add_placements_and_locations_to_the_db.py @@ -11,14 +11,10 @@ revision = 'bcdde200a1b' down_revision = '201d55b38649' from alembic import op -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models import sqlalchemy as sa -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) - +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.create_table('imagestoragelocation', sa.Column('id', sa.Integer(), nullable=False), @@ -27,7 +23,7 @@ def upgrade(): ) op.create_index('imagestoragelocation_name', 'imagestoragelocation', ['name'], unique=True) - op.bulk_insert(schema.tables['imagestoragelocation'], + op.bulk_insert(tables.imagestoragelocation, [ {'id':1, 'name':'s3_us_east_1'}, {'id':2, 'name':'s3_eu_west_1'}, @@ -52,12 +48,8 @@ def upgrade(): ### end Alembic commands ### -def downgrade(): +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### - op.drop_index('imagestorageplacement_storage_id_location_id', table_name='imagestorageplacement') - op.drop_index('imagestorageplacement_storage_id', table_name='imagestorageplacement') - op.drop_index('imagestorageplacement_location_id', table_name='imagestorageplacement') op.drop_table('imagestorageplacement') - op.drop_index('imagestoragelocation_name', table_name='imagestoragelocation') op.drop_table('imagestoragelocation') ### end Alembic commands ### diff --git a/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py b/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py index 79ea17be0..9ceab4218 100644 --- a/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py +++ b/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py @@ -14,13 +14,13 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(): +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.drop_table('webhook') ### end Alembic commands ### -def downgrade(): +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.create_table('webhook', sa.Column('id', mysql.INTEGER(display_width=11), nullable=False), diff --git a/util/collections.py b/util/collections.py new file mode 100644 index 000000000..b34dc00ed --- /dev/null +++ b/util/collections.py @@ -0,0 +1,12 @@ +class AttrDict(dict): + def __init__(self, *args, **kwargs): + super(AttrDict, self).__init__(*args, **kwargs) + self.__dict__ = self + + @classmethod + def deep_copy(cls, attr_dict): + copy = AttrDict(attr_dict) + for key, value in copy.items(): + if isinstance(value, AttrDict): + copy[key] = cls.deep_copy(value) + return copy \ No newline at end of file From 9eccdb7696d7b44c4bc9330e246e7e3323d697e5 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 8 Sep 2014 12:00:20 -0400 Subject: [PATCH 39/49] Fix NPE --- static/js/controllers.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/static/js/controllers.js b/static/js/controllers.js index e4e364c87..9131a0140 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -1647,14 +1647,17 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use if ($scope.cuser.logins) { for (var i = 0; i < $scope.cuser.logins.length; i++) { - if ($scope.cuser.logins[i].service == 'github') { + var login = $scope.cuser.logins[i]; + login.metadata = login.metadata || {}; + + if (login.service == 'github') { $scope.hasGithubLogin = true; - $scope.githubLogin = $scope.cuser.logins[i].metadata['service_username']; + $scope.githubLogin = login.metadata['service_username']; } - if ($scope.cuser.logins[i].service == 'google') { + if (login.service == 'google') { $scope.hasGoogleLogin = true; - $scope.googleLogin = $scope.cuser.logins[i].metadata['service_username']; + $scope.googleLogin = login.metadata['service_username']; } } } From dd4037e3243f0c59ff79f134be2110e5a2658957 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 8 Sep 2014 12:17:00 -0400 Subject: [PATCH 40/49] Allow github trigger setup folder paths to be specified even if a Dockerfile is not found --- static/directives/dropdown-select.html | 2 +- static/directives/trigger-setup-github.html | 3 ++- static/js/app.js | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/static/directives/dropdown-select.html b/static/directives/dropdown-select.html index c1157e3d0..69404e161 100644 --- a/static/directives/dropdown-select.html +++ b/static/directives/dropdown-select.html @@ -2,7 +2,7 @@
    + ng-readonly="!allowCustomInput">
    - +
    From 29d40db5ea532e069ec8ceb8c9cb1d38d30724f3 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Tue, 9 Sep 2014 15:54:03 -0400 Subject: [PATCH 42/49] Add a new RadosGW storage engine. Allow engines to distinguish not only between those that can support direct uploads and downloads, but those that support doing it through the browser. Rename resumeable->resumable. --- app.py | 2 +- data/userfiles.py | 210 +++++++++------------------------- endpoints/api/build.py | 4 +- endpoints/registry.py | 6 +- storage/__init__.py | 3 +- storage/basestorage.py | 10 +- storage/cloud.py | 65 +++++++++-- storage/distributedstorage.py | 4 +- storage/fakestorage.py | 3 + storage/local.py | 12 +- test/testconfig.py | 2 +- workers/dockerfilebuild.py | 2 +- 12 files changed, 147 insertions(+), 176 deletions(-) diff --git a/app.py b/app.py index 81c59a30c..bcc4e86d7 100644 --- a/app.py +++ b/app.py @@ -88,7 +88,7 @@ Principal(app, use_sessions=False) login_manager = LoginManager(app) mail = Mail(app) storage = Storage(app) -userfiles = Userfiles(app) +userfiles = Userfiles(app, storage) analytics = Analytics(app) billing = Billing(app) sentry = Sentry(app) diff --git a/data/userfiles.py b/data/userfiles.py index 79fbcb507..e6d21c1c1 100644 --- a/data/userfiles.py +++ b/data/userfiles.py @@ -1,110 +1,31 @@ -import boto import os import logging -import hashlib import magic -from boto.s3.key import Key from uuid import uuid4 from flask import url_for, request, send_file, make_response, abort from flask.views import View - logger = logging.getLogger(__name__) -class FakeUserfiles(object): - def prepare_for_drop(self, mime_type): - return ('http://fake/url', uuid4()) - - def store_file(self, file_like_obj, content_type): - raise NotImplementedError() - - def get_file_url(self, file_id, expires_in=300): - return ('http://fake/url') - - def get_file_checksum(self, file_id): - return 'abcdefg' - - -class S3FileWriteException(Exception): - pass - - -class S3Userfiles(object): - def __init__(self, path, s3_access_key, s3_secret_key, bucket_name): - self._initialized = False - self._bucket_name = bucket_name - self._access_key = s3_access_key - self._secret_key = s3_secret_key - self._prefix = path - self._s3_conn = None - self._bucket = None - - def _initialize_s3(self): - if not self._initialized: - self._s3_conn = boto.connect_s3(self._access_key, self._secret_key) - self._bucket = self._s3_conn.get_bucket(self._bucket_name) - self._initialized = True - - def prepare_for_drop(self, mime_type): - """ Returns a signed URL to upload a file to our bucket. """ - self._initialize_s3() - logger.debug('Requested upload url with content type: %s' % mime_type) - file_id = str(uuid4()) - full_key = os.path.join(self._prefix, file_id) - k = Key(self._bucket, full_key) - url = k.generate_url(300, 'PUT', headers={'Content-Type': mime_type}, - encrypt_key=True) - return (url, file_id) - - def store_file(self, file_like_obj, content_type): - self._initialize_s3() - file_id = str(uuid4()) - full_key = os.path.join(self._prefix, file_id) - k = Key(self._bucket, full_key) - logger.debug('Setting s3 content type to: %s' % content_type) - k.set_metadata('Content-Type', content_type) - bytes_written = k.set_contents_from_file(file_like_obj, encrypt_key=True, - rewind=True) - - if bytes_written == 0: - raise S3FileWriteException('Unable to write file to S3') - - return file_id - - def get_file_url(self, file_id, expires_in=300, mime_type=None): - self._initialize_s3() - full_key = os.path.join(self._prefix, file_id) - k = Key(self._bucket, full_key) - headers = None - if mime_type: - headers={'Content-Type': mime_type} - - return k.generate_url(expires_in, headers=headers) - - def get_file_checksum(self, file_id): - self._initialize_s3() - full_key = os.path.join(self._prefix, file_id) - k = self._bucket.lookup(full_key) - return k.etag[1:-1][:7] - - class UserfilesHandlers(View): methods = ['GET', 'PUT'] - def __init__(self, local_userfiles): - self._userfiles = local_userfiles + def __init__(self, distributed_storage, location, files): + self._storage = distributed_storage + self._files = files + self._locations = {location} self._magic = magic.Magic(mime=True) def get(self, file_id): - path = self._userfiles.file_path(file_id) - if not os.path.exists(path): + path = self._files.get_file_id_path(file_id) + try: + file_stream = self._storage.stream_read_file(self._locations, path) + return send_file(file_stream) + except IOError: abort(404) - logger.debug('Sending path: %s' % path) - return send_file(path, mimetype=self._magic.from_file(path)) - def put(self, file_id): input_stream = request.stream if request.headers.get('transfer-encoding') == 'chunked': @@ -112,7 +33,8 @@ class UserfilesHandlers(View): # encoding (Gunicorn) input_stream = request.environ['wsgi.input'] - self._userfiles.store_stream(input_stream, file_id) + path = self._files.get_file_id_path(file_id) + self._storage.stream_write(self._locations, path, input_stream) return make_response('Okay') @@ -123,99 +45,79 @@ class UserfilesHandlers(View): return self.put(file_id) -class LocalUserfiles(object): - def __init__(self, app, path): - self._root_path = path - self._buffer_size = 64 * 1024 # 64 KB +class DelegateUserfiles(object): + def __init__(self, app, distributed_storage, location, path, handler_name): self._app = app + self._storage = distributed_storage + self._locations = {location} + self._prefix = path + self._handler_name = handler_name def _build_url_adapter(self): return self._app.url_map.bind(self._app.config['SERVER_HOSTNAME'], script_name=self._app.config['APPLICATION_ROOT'] or '/', url_scheme=self._app.config['PREFERRED_URL_SCHEME']) - def prepare_for_drop(self, mime_type): + def get_file_id_path(self, file_id): + return os.path.join(self._prefix, file_id) + + def prepare_for_drop(self, mime_type, requires_cors=True): + """ Returns a signed URL to upload a file to our bucket. """ + logger.debug('Requested upload url with content type: %s' % mime_type) file_id = str(uuid4()) - with self._app.app_context() as ctx: - ctx.url_adapter = self._build_url_adapter() - return (url_for('userfiles_handlers', file_id=file_id, _external=True), file_id) + path = self.get_file_id_path(file_id) + url = self._storage.get_direct_upload_url(self._locations, path, mime_type, requires_cors) - def file_path(self, file_id): - if '..' in file_id or file_id.startswith('/'): - raise RuntimeError('Invalid Filename') - return os.path.join(self._root_path, file_id) + if url is None: + with self._app.app_context() as ctx: + ctx.url_adapter = self._build_url_adapter() + return (url_for(self._handler_name, file_id=file_id, _external=True), file_id) - def store_stream(self, stream, file_id): - path = self.file_path(file_id) - dirname = os.path.dirname(path) - if not os.path.exists(dirname): - os.makedirs(dirname) - - with open(path, 'w') as to_write: - while True: - try: - buf = stream.read(self._buffer_size) - if not buf: - break - to_write.write(buf) - except IOError: - break + return (url, file_id) def store_file(self, file_like_obj, content_type): file_id = str(uuid4()) - - # Rewind the file to match what s3 does - file_like_obj.seek(0, os.SEEK_SET) - - self.store_stream(file_like_obj, file_id) + path = self.get_file_id_path(file_id) + self._storage.stream_write(self._locations, path, file_like_obj) return file_id - def get_file_url(self, file_id, expires_in=300): - with self._app.app_context() as ctx: - ctx.url_adapter = self._build_url_adapter() - return url_for('userfiles_handlers', file_id=file_id, _external=True) + def get_file_url(self, file_id, expires_in=300, requires_cors=False): + path = self.get_file_id_path(file_id) + url = self._storage.get_direct_download_url(self._locations, path, expires_in, requires_cors) + + if url is None: + with self._app.app_context() as ctx: + ctx.url_adapter = self._build_url_adapter() + return url_for(self._handler_name, file_id=file_id, _external=True) + + return url def get_file_checksum(self, file_id): - path = self.file_path(file_id) - sha_hash = hashlib.sha256() - with open(path, 'r') as to_hash: - while True: - buf = to_hash.read(self._buffer_size) - if not buf: - break - sha_hash.update(buf) - return sha_hash.hexdigest()[:7] + path = self.get_file_id_path(file_id) + return self._storage.get_checksum(self._locations, path) class Userfiles(object): - def __init__(self, app=None): + def __init__(self, app=None, distributed_storage=None): self.app = app if app is not None: - self.state = self.init_app(app) + self.state = self.init_app(app, distributed_storage) else: self.state = None - def init_app(self, app): - storage_type = app.config.get('USERFILES_TYPE', 'LocalUserfiles') - path = app.config.get('USERFILES_PATH', '') + def init_app(self, app, distributed_storage): + location = app.config.get('USERFILES_LOCATION') + path = app.config.get('USERFILES_PATH', None) - if storage_type == 'LocalUserfiles': - userfiles = LocalUserfiles(app, path) - app.add_url_rule('/userfiles/', - view_func=UserfilesHandlers.as_view('userfiles_handlers', - local_userfiles=userfiles)) + handler_name = 'userfiles_handlers' - elif storage_type == 'S3Userfiles': - access_key = app.config.get('USERFILES_AWS_ACCESS_KEY', '') - secret_key = app.config.get('USERFILES_AWS_SECRET_KEY', '') - bucket = app.config.get('USERFILES_S3_BUCKET', '') - userfiles = S3Userfiles(path, access_key, secret_key, bucket) + userfiles = DelegateUserfiles(app, distributed_storage, location, path, handler_name) - elif storage_type == 'FakeUserfiles': - userfiles = FakeUserfiles() - - else: - raise RuntimeError('Unknown userfiles type: %s' % storage_type) + app.add_url_rule('/userfiles/', + view_func=UserfilesHandlers.as_view(handler_name, + distributed_storage=distributed_storage, + location=location, + files=userfiles)) # register extension with app app.extensions = getattr(app, 'extensions', {}) diff --git a/endpoints/api/build.py b/endpoints/api/build.py index 21d554069..74677fadb 100644 --- a/endpoints/api/build.py +++ b/endpoints/api/build.py @@ -80,7 +80,7 @@ def build_status_view(build_obj, can_write=False): } if can_write: - resp['archive_url'] = user_files.get_file_url(build_obj.resource_key) + resp['archive_url'] = user_files.get_file_url(build_obj.resource_key, requires_cors=True) return resp @@ -257,7 +257,7 @@ class FileDropResource(ApiResource): def post(self): """ Request a URL to which a file may be uploaded. """ mime_type = request.get_json()['mimeType'] - (url, file_id) = user_files.prepare_for_drop(mime_type) + (url, file_id) = user_files.prepare_for_drop(mime_type, requires_cors=True) return { 'url': url, 'file_id': str(file_id), diff --git a/endpoints/registry.py b/endpoints/registry.py index 72633939e..94719905a 100644 --- a/endpoints/registry.py +++ b/endpoints/registry.py @@ -110,10 +110,10 @@ def head_image_layer(namespace, repository, image_id, headers): extra_headers = {} - # Add the Accept-Ranges header if the storage engine supports resumeable + # Add the Accept-Ranges header if the storage engine supports resumable # downloads. - if store.get_supports_resumeable_downloads(repo_image.storage.locations): - profile.debug('Storage supports resumeable downloads') + if store.get_supports_resumable_downloads(repo_image.storage.locations): + profile.debug('Storage supports resumable downloads') extra_headers['Accept-Ranges'] = 'bytes' resp = make_response('') diff --git a/storage/__init__.py b/storage/__init__.py index 6700dab0b..4d1134d4b 100644 --- a/storage/__init__.py +++ b/storage/__init__.py @@ -1,5 +1,5 @@ from storage.local import LocalStorage -from storage.cloud import S3Storage, GoogleCloudStorage +from storage.cloud import S3Storage, GoogleCloudStorage, RadosGWStorage from storage.fakestorage import FakeStorage from storage.distributedstorage import DistributedStorage @@ -8,6 +8,7 @@ STORAGE_DRIVER_CLASSES = { 'LocalStorage': LocalStorage, 'S3Storage': S3Storage, 'GoogleCloudStorage': GoogleCloudStorage, + 'RadosGWStorage': RadosGWStorage, } diff --git a/storage/basestorage.py b/storage/basestorage.py index 2d3727a5b..aa6434b8e 100644 --- a/storage/basestorage.py +++ b/storage/basestorage.py @@ -54,10 +54,13 @@ class BaseStorage(StoragePaths): # Set the IO buffer to 64kB buffer_size = 64 * 1024 - def get_direct_download_url(self, path, expires_in=60): + def get_direct_download_url(self, path, expires_in=60, requires_cors=False): return None - def get_supports_resumeable_downloads(self): + def get_direct_upload_url(self, path, mime_type, requires_cors=True): + return None + + def get_supports_resumable_downloads(self): return False def get_content(self, path): @@ -83,3 +86,6 @@ class BaseStorage(StoragePaths): def remove(self, path): raise NotImplementedError + + def get_checksum(self, path): + raise NotImplementedError \ No newline at end of file diff --git a/storage/cloud.py b/storage/cloud.py index d64415410..a576a6401 100644 --- a/storage/cloud.py +++ b/storage/cloud.py @@ -35,8 +35,8 @@ class StreamReadKeyAsFile(object): class _CloudStorage(BaseStorage): - def __init__(self, connection_class, key_class, upload_params, storage_path, access_key, - secret_key, bucket_name): + def __init__(self, connection_class, key_class, connect_kwargs, upload_params, storage_path, + access_key, secret_key, bucket_name): self._initialized = False self._bucket_name = bucket_name self._access_key = access_key @@ -45,12 +45,14 @@ class _CloudStorage(BaseStorage): self._connection_class = connection_class self._key_class = key_class self._upload_params = upload_params + self._connect_kwargs = connect_kwargs self._cloud_conn = None self._cloud_bucket = None def _initialize_cloud_conn(self): if not self._initialized: - self._cloud_conn = self._connection_class(self._access_key, self._secret_key) + self._cloud_conn = self._connection_class(self._access_key, self._secret_key, + **self._connect_kwargs) self._cloud_bucket = self._cloud_conn.get_bucket(self._bucket_name) self._initialized = True @@ -87,15 +89,22 @@ class _CloudStorage(BaseStorage): key.set_contents_from_string(content, **self._upload_params) return path - def get_supports_resumeable_downloads(self): + def get_supports_resumable_downloads(self): return True - def get_direct_download_url(self, path, expires_in=60): + def get_direct_download_url(self, path, expires_in=60, requires_cors=False): self._initialize_cloud_conn() path = self._init_path(path) k = self._key_class(self._cloud_bucket, path) return k.generate_url(expires_in) + def get_direct_upload_url(self, path, mime_type, requires_cors=True): + self._initialize_cloud_conn() + path = self._init_path(path) + key = self._key_class(self._cloud_bucket, path) + url = key.generate_url(300, 'PUT', headers={'Content-Type': mime_type}, encrypt_key=True) + return url + def stream_read(self, path): self._initialize_cloud_conn() path = self._init_path(path) @@ -179,21 +188,32 @@ class _CloudStorage(BaseStorage): for key in self._cloud_bucket.list(prefix=path): key.delete() + def get_checksum(self, path): + self._initialize_cloud_conn() + path = self._init_path(path) + key = self._key_class(self._cloud_bucket, path) + k = self._cloud_bucket.lookup(key) + return k.etag[1:-1][:7] + class S3Storage(_CloudStorage): def __init__(self, storage_path, s3_access_key, s3_secret_key, s3_bucket): upload_params = { 'encrypt_key': True, } + connect_kwargs = {} super(S3Storage, self).__init__(boto.s3.connection.S3Connection, boto.s3.key.Key, - upload_params, storage_path, s3_access_key, s3_secret_key, - s3_bucket) + connect_kwargs, upload_params, storage_path, s3_access_key, + s3_secret_key, s3_bucket) class GoogleCloudStorage(_CloudStorage): def __init__(self, storage_path, access_key, secret_key, bucket_name): - super(GoogleCloudStorage, self).__init__(boto.gs.connection.GSConnection, boto.gs.key.Key, {}, - storage_path, access_key, secret_key, bucket_name) + upload_params = {} + connect_kwargs = {} + super(GoogleCloudStorage, self).__init__(boto.gs.connection.GSConnection, boto.gs.key.Key, + connect_kwargs, upload_params, storage_path, + access_key, secret_key, bucket_name) def stream_write(self, path, fp): # Minimum size of upload part size on S3 is 5MB @@ -201,3 +221,30 @@ class GoogleCloudStorage(_CloudStorage): path = self._init_path(path) key = self._key_class(self._cloud_bucket, path) key.set_contents_from_stream(fp) + + +class RadosGWStorage(_CloudStorage): + def __init__(self, hostname, is_secure, storage_path, access_key, secret_key, bucket_name): + upload_params = {} + connect_kwargs = { + 'host': hostname, + 'is_secure': is_secure, + 'calling_format': boto.s3.connection.OrdinaryCallingFormat(), + } + super(RadosGWStorage, self).__init__(boto.s3.connection.S3Connection, boto.s3.key.Key, + connect_kwargs, upload_params, storage_path, access_key, + secret_key, bucket_name) + + # TODO remove when radosgw supports cors: http://tracker.ceph.com/issues/8718#change-38624 + def get_direct_download_url(self, path, expires_in=60, requires_cors=False): + if requires_cors: + return None + + return super(RadosGWStorage, self).get_direct_download_url(path, expires_in, requires_cors) + + # TODO remove when radosgw supports cors: http://tracker.ceph.com/issues/8718#change-38624 + def get_direct_upload_url(self, path, mime_type, requires_cors=True): + if requires_cors: + return None + + return super(RadosGWStorage, self).get_direct_upload_url(path, mime_type, requires_cors) diff --git a/storage/distributedstorage.py b/storage/distributedstorage.py index 9941f0fa5..1544d9725 100644 --- a/storage/distributedstorage.py +++ b/storage/distributedstorage.py @@ -31,6 +31,7 @@ class DistributedStorage(StoragePaths): self.preferred_locations = list(preferred_locations) get_direct_download_url = _location_aware(BaseStorage.get_direct_download_url) + get_direct_upload_url = _location_aware(BaseStorage.get_direct_upload_url) get_content = _location_aware(BaseStorage.get_content) put_content = _location_aware(BaseStorage.put_content) stream_read = _location_aware(BaseStorage.stream_read) @@ -39,4 +40,5 @@ class DistributedStorage(StoragePaths): list_directory = _location_aware(BaseStorage.list_directory) exists = _location_aware(BaseStorage.exists) remove = _location_aware(BaseStorage.remove) - get_supports_resumeable_downloads = _location_aware(BaseStorage.get_supports_resumeable_downloads) + get_checksum = _location_aware(BaseStorage.get_checksum) + get_supports_resumable_downloads = _location_aware(BaseStorage.get_supports_resumable_downloads) diff --git a/storage/fakestorage.py b/storage/fakestorage.py index 597d22af4..5761acf2f 100644 --- a/storage/fakestorage.py +++ b/storage/fakestorage.py @@ -22,3 +22,6 @@ class FakeStorage(BaseStorage): def exists(self, path): return False + + def get_checksum(self, path): + return 'abcdefg' \ No newline at end of file diff --git a/storage/local.py b/storage/local.py index 361d76403..55e79077b 100644 --- a/storage/local.py +++ b/storage/local.py @@ -1,4 +1,3 @@ - import os import shutil @@ -80,3 +79,14 @@ class LocalStorage(BaseStorage): os.remove(path) except OSError: pass + + def get_checksum(self, path): + path = self._init_path(path) + sha_hash = hashlib.sha256() + with open(path, 'r') as to_hash: + while True: + buf = to_hash.read(self.buffer_size) + if not buf: + break + sha_hash.update(buf) + return sha_hash.hexdigest()[:7] diff --git a/test/testconfig.py b/test/testconfig.py index c74e5712a..35c96a803 100644 --- a/test/testconfig.py +++ b/test/testconfig.py @@ -30,7 +30,7 @@ class TestConfig(DefaultConfig): BUILDLOGS_MODULE_AND_CLASS = ('test.testlogs', 'testlogs.TestBuildLogs') BUILDLOGS_OPTIONS = ['devtable', 'building', 'deadbeef-dead-beef-dead-beefdeadbeef', False] - USERFILES_TYPE = 'FakeUserfiles' + USERFILES_LOCATION = 'local_us' FEATURE_SUPER_USERS = True FEATURE_BILLING = True diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index b373a00a9..143a27103 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -495,7 +495,7 @@ class DockerfileBuildWorker(Worker): job_config = json.loads(repository_build.job_config) - resource_url = user_files.get_file_url(repository_build.resource_key) + resource_url = user_files.get_file_url(repository_build.resource_key, requires_cors=False) tag_names = job_config['docker_tags'] build_subdir = job_config['build_subdir'] repo = job_config['repository'] From 756e8ec84868a1f678b3efab5a18def9bc97ef9c Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Tue, 9 Sep 2014 16:52:53 -0400 Subject: [PATCH 43/49] Send the content type through to the cloud engines. --- data/userfiles.py | 6 ++++-- storage/basestorage.py | 2 +- storage/cloud.py | 16 +++++++++++++--- storage/fakestorage.py | 2 +- storage/local.py | 2 +- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/data/userfiles.py b/data/userfiles.py index e6d21c1c1..c3113802f 100644 --- a/data/userfiles.py +++ b/data/userfiles.py @@ -33,8 +33,10 @@ class UserfilesHandlers(View): # encoding (Gunicorn) input_stream = request.environ['wsgi.input'] + c_type = request.headers.get('Content-Type', None) + path = self._files.get_file_id_path(file_id) - self._storage.stream_write(self._locations, path, input_stream) + self._storage.stream_write(self._locations, path, input_stream, c_type) return make_response('Okay') @@ -78,7 +80,7 @@ class DelegateUserfiles(object): def store_file(self, file_like_obj, content_type): file_id = str(uuid4()) path = self.get_file_id_path(file_id) - self._storage.stream_write(self._locations, path, file_like_obj) + self._storage.stream_write(self._locations, path, file_like_obj, content_type) return file_id def get_file_url(self, file_id, expires_in=300, requires_cors=False): diff --git a/storage/basestorage.py b/storage/basestorage.py index aa6434b8e..78d49aa1f 100644 --- a/storage/basestorage.py +++ b/storage/basestorage.py @@ -75,7 +75,7 @@ class BaseStorage(StoragePaths): def stream_read_file(self, path): raise NotImplementedError - def stream_write(self, path, fp): + def stream_write(self, path, fp, content_type=None): raise NotImplementedError def list_directory(self, path=None): diff --git a/storage/cloud.py b/storage/cloud.py index a576a6401..28325e187 100644 --- a/storage/cloud.py +++ b/storage/cloud.py @@ -125,14 +125,20 @@ class _CloudStorage(BaseStorage): raise IOError('No such key: \'{0}\''.format(path)) return StreamReadKeyAsFile(key) - def stream_write(self, path, fp): + def stream_write(self, path, fp, content_type=None): # Minimum size of upload part size on S3 is 5MB self._initialize_cloud_conn() buffer_size = 5 * 1024 * 1024 if self.buffer_size > buffer_size: buffer_size = self.buffer_size path = self._init_path(path) - mp = self._cloud_bucket.initiate_multipart_upload(path, **self._upload_params) + + metadata = {} + if content_type is not None: + metadata['Content-Type'] = content_type + + mp = self._cloud_bucket.initiate_multipart_upload(path, metadata=metadata, + **self._upload_params) num_part = 1 while True: try: @@ -215,11 +221,15 @@ class GoogleCloudStorage(_CloudStorage): connect_kwargs, upload_params, storage_path, access_key, secret_key, bucket_name) - def stream_write(self, path, fp): + def stream_write(self, path, fp, content_type=None): # Minimum size of upload part size on S3 is 5MB self._initialize_cloud_conn() path = self._init_path(path) key = self._key_class(self._cloud_bucket, path) + + if content_type is not None: + key.set_metadata('Content-Type', content_type) + key.set_contents_from_stream(fp) diff --git a/storage/fakestorage.py b/storage/fakestorage.py index 5761acf2f..232f5af24 100644 --- a/storage/fakestorage.py +++ b/storage/fakestorage.py @@ -14,7 +14,7 @@ class FakeStorage(BaseStorage): def stream_read(self, path): yield '' - def stream_write(self, path, fp): + def stream_write(self, path, fp, content_type=None): pass def remove(self, path): diff --git a/storage/local.py b/storage/local.py index 55e79077b..a800645a8 100644 --- a/storage/local.py +++ b/storage/local.py @@ -41,7 +41,7 @@ class LocalStorage(BaseStorage): path = self._init_path(path) return open(path, mode='rb') - def stream_write(self, path, fp): + def stream_write(self, path, fp, content_type=None): # Size is mandatory path = self._init_path(path, create=True) with open(path, mode='wb') as f: From c9e16487817126b04d9b64085ed46a7e9ed35632 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Tue, 9 Sep 2014 18:30:14 -0400 Subject: [PATCH 44/49] Small fixes to bugs in the streaming handler for use with magic and radosgw. --- config.py | 8 ++++---- data/userfiles.py | 6 +++++- storage/cloud.py | 20 +++++++++++++------- storage/local.py | 4 +++- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/config.py b/config.py index f797cb36a..ffcf7f79e 100644 --- a/config.py +++ b/config.py @@ -89,10 +89,6 @@ class DefaultConfig(object): # Stripe config BILLING_TYPE = 'FakeStripe' - # Userfiles - USERFILES_TYPE = 'LocalUserfiles' - USERFILES_PATH = 'test/data/registry/userfiles' - # Analytics ANALYTICS_TYPE = 'FakeAnalytics' @@ -172,3 +168,7 @@ class DefaultConfig(object): } DISTRIBUTED_STORAGE_PREFERENCE = ['local_us'] + + # Userfiles + USERFILES_LOCATION = 'local_us' + USERFILES_PATH = 'userfiles/' diff --git a/data/userfiles.py b/data/userfiles.py index c3113802f..7ee7726e4 100644 --- a/data/userfiles.py +++ b/data/userfiles.py @@ -5,6 +5,8 @@ import magic from uuid import uuid4 from flask import url_for, request, send_file, make_response, abort from flask.views import View +from io import BufferedReader + logger = logging.getLogger(__name__) @@ -22,7 +24,9 @@ class UserfilesHandlers(View): path = self._files.get_file_id_path(file_id) try: file_stream = self._storage.stream_read_file(self._locations, path) - return send_file(file_stream) + buffered = BufferedReader(file_stream) + file_header_bytes = buffered.peek(1024) + return send_file(buffered, mimetype=self._magic.from_buffer(file_header_bytes)) except IOError: abort(404) diff --git a/storage/cloud.py b/storage/cloud.py index 28325e187..0d2028e1b 100644 --- a/storage/cloud.py +++ b/storage/cloud.py @@ -7,23 +7,19 @@ import boto.gs.connection import boto.s3.key import boto.gs.key +from io import UnsupportedOperation, BufferedIOBase + from storage.basestorage import BaseStorage logger = logging.getLogger(__name__) -class StreamReadKeyAsFile(object): +class StreamReadKeyAsFile(BufferedIOBase): def __init__(self, key): self._key = key self._finished = False - def __enter__(self): - return self - - def __exit__(self, type, value, tb): - self._key.close(fast=True) - def read(self, amt=None): if self._finished: return None @@ -33,6 +29,16 @@ class StreamReadKeyAsFile(object): self._finished = True return resp + def readable(self): + return True + + @property + def closed(self): + return self._key.closed + + def close(self): + self._key.close(fast=True) + class _CloudStorage(BaseStorage): def __init__(self, connection_class, key_class, connect_kwargs, upload_params, storage_path, diff --git a/storage/local.py b/storage/local.py index a800645a8..987431e33 100644 --- a/storage/local.py +++ b/storage/local.py @@ -1,5 +1,7 @@ import os import shutil +import hashlib +import io from storage.basestorage import BaseStorage @@ -39,7 +41,7 @@ class LocalStorage(BaseStorage): def stream_read_file(self, path): path = self._init_path(path) - return open(path, mode='rb') + return io.open(path, mode='rb') def stream_write(self, path, fp, content_type=None): # Size is mandatory From 548f855f71f89c5a375584386f006181ae014ac8 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Tue, 9 Sep 2014 22:28:25 -0400 Subject: [PATCH 45/49] Use the pure python io module to avoid some interaction between gunicorn, wsgi, and bufferedreader that prevents gunicorn from properly sending the files. --- data/userfiles.py | 2 +- storage/cloud.py | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/data/userfiles.py b/data/userfiles.py index 7ee7726e4..950c4dd60 100644 --- a/data/userfiles.py +++ b/data/userfiles.py @@ -5,7 +5,7 @@ import magic from uuid import uuid4 from flask import url_for, request, send_file, make_response, abort from flask.views import View -from io import BufferedReader +from _pyio import BufferedReader logger = logging.getLogger(__name__) diff --git a/storage/cloud.py b/storage/cloud.py index 0d2028e1b..f7d922d6c 100644 --- a/storage/cloud.py +++ b/storage/cloud.py @@ -7,7 +7,7 @@ import boto.gs.connection import boto.s3.key import boto.gs.key -from io import UnsupportedOperation, BufferedIOBase +from io import BufferedIOBase from storage.basestorage import BaseStorage @@ -18,15 +18,12 @@ logger = logging.getLogger(__name__) class StreamReadKeyAsFile(BufferedIOBase): def __init__(self, key): self._key = key - self._finished = False def read(self, amt=None): - if self._finished: + if self.closed: return None resp = self._key.read(amt) - if not resp: - self._finished = True return resp def readable(self): From 11b690cba93089cef2bd16eccfccf1e97434d8f0 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 10 Sep 2014 14:17:39 -0400 Subject: [PATCH 46/49] Fix slack help url --- static/js/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/js/app.js b/static/js/app.js index 0551df2dc..26b8a4be1 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1280,7 +1280,7 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading 'name': 'token', 'type': 'string', 'title': 'Token', - 'help_url': 'https://{subdomain}.slack.com/services/new/outgoing-webhook' + 'help_url': 'https://{subdomain}.slack.com/services/new/incoming-webhook' } ] } From 75f19dc6c6bd8e0cefa4e4a6094289c070be1486 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 10 Sep 2014 14:43:10 -0400 Subject: [PATCH 47/49] Refresh the version of phusion baseimage and the ubuntu package server contents. --- Dockerfile.buildworker | 4 ++-- Dockerfile.web | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile.buildworker b/Dockerfile.buildworker index 04efe38f0..159c7867c 100644 --- a/Dockerfile.buildworker +++ b/Dockerfile.buildworker @@ -1,10 +1,10 @@ -FROM phusion/baseimage:0.9.11 +FROM phusion/baseimage:0.9.13 ENV DEBIAN_FRONTEND noninteractive ENV HOME /root # Install the dependencies. -RUN apt-get update # 21AUG2014 +RUN apt-get update # 10SEP2014 # New ubuntu packages should be added as their own apt-get install lines below the existing install commands RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62-dev libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap2-dev libsasl2-dev libpq-dev diff --git a/Dockerfile.web b/Dockerfile.web index e1d253632..b24694b42 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -1,10 +1,10 @@ -FROM phusion/baseimage:0.9.11 +FROM phusion/baseimage:0.9.13 ENV DEBIAN_FRONTEND noninteractive ENV HOME /root # Install the dependencies. -RUN apt-get update # 21AUG2014 +RUN apt-get update # 10SEP2014 # New ubuntu packages should be added as their own apt-get install lines below the existing install commands RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62-dev libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap2-dev libsasl2-dev libpq-dev From 539fc0420578ec188c8b0da8afad8b71658f2069 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 10 Sep 2014 17:18:49 -0400 Subject: [PATCH 48/49] Seek the file pointer to zero since we now use multipart for upload of userfiles, which does not seek automatically. --- endpoints/trigger.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/endpoints/trigger.py b/endpoints/trigger.py index ab7aa9065..ae0b4b2b7 100644 --- a/endpoints/trigger.py +++ b/endpoints/trigger.py @@ -291,6 +291,9 @@ class GithubBuildTrigger(BuildTrigger): with tarfile.open(fileobj=tarball) as archive: tarball_subdir = archive.getnames()[0] + # Seek to position 0 to make boto multipart happy + tarball.seek(0) + dockerfile_id = user_files.store_file(tarball, TARBALL_MIME) logger.debug('Successfully prepared job') From da3d58890e2df5dc5d7e8aa02ffe117f6ca989c8 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Fri, 12 Sep 2014 10:46:35 -0400 Subject: [PATCH 49/49] Slight tweak in the text of the 403 pull base image error. --- static/directives/build-log-error.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/directives/build-log-error.html b/static/directives/build-log-error.html index cf03fa7b2..13b399bb9 100644 --- a/static/directives/build-log-error.html +++ b/static/directives/build-log-error.html @@ -17,7 +17,7 @@
    Note: No robot account is specified for this build. Without such credentials, this pull will always fail. Please setup a new - build trigger with a robot account that has access to {{ getLocalPullInfo().repo }} or make the repository public. + build trigger with a robot account that has access to {{ getLocalPullInfo().repo }} or make that repository public.