From 5845e37e321915f63e84e2a46d93080413cb83ca Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 21 May 2015 15:22:59 -0400 Subject: [PATCH 01/24] Add Swift storage library --- requirements-nover.txt | 3 +- requirements.txt | 81 +++++--- static/css/core-ui.css | 23 +++ .../directives/config/config-map-field.html | 20 ++ .../directives/config/config-setup-tool.html | 8 +- .../config/config-string-field.html | 2 +- static/js/core-config-setup.js | 52 ++++- storage/__init__.py | 2 + storage/swift.py | 183 ++++++++++++++++++ 9 files changed, 341 insertions(+), 33 deletions(-) create mode 100644 static/directives/config/config-map-field.html create mode 100644 storage/swift.py diff --git a/requirements-nover.txt b/requirements-nover.txt index d76e671b5..20df379e2 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -21,7 +21,6 @@ paramiko xhtml2pdf redis hiredis -docker-py flask-restful==0.2.12 jsonschema git+https://github.com/NateFerrero/oauth2lib.git @@ -52,3 +51,5 @@ mock psutil stringscore mockldap +python-swiftclient +python-keystoneclient \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 08eddb442..2a950f9dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,68 +1,91 @@ -APScheduler==3.0.1 +APScheduler==3.0.3 +Babel==1.3 Flask==0.10.1 Flask-Login==0.2.11 Flask-Mail==0.9.1 Flask-Principal==0.4.0 Flask-RESTful==0.2.12 Jinja2==2.7.3 -LogentriesLogger==0.2.1 -Mako==1.0.0 +Logentries==0.7 +Mako==1.0.1 MarkupSafe==0.23 -Pillow==2.7.0 -PyMySQL==0.6.3 +Pillow==2.8.1 +PyMySQL==0.6.6 PyPDF2==1.24 PyYAML==3.11 -SQLAlchemy==0.9.8 -WebOb==1.4 -Werkzeug==0.9.6 -aiowsgi==0.3 -alembic==0.7.4 +SQLAlchemy==1.0.3 +WebOb==1.4.1 +Werkzeug==0.10.4 +aiowsgi==0.5 +alembic==0.7.5.post2 +argparse==1.3.0 autobahn==0.9.3-3 backports.ssl-match-hostname==3.4.0.2 beautifulsoup4==4.3.2 blinker==1.3 -boto==2.35.1 +boto==2.38.0 cachetools==1.0.0 -docker-py==0.7.1 -ecdsa==0.11 +certifi==2015.04.28 +cffi==0.9.2 +cryptography==0.8.2 +ecdsa==0.13 +enum34==1.0.4 +funcparserlib==0.3.6 futures==2.2.0 gevent==1.0.1 gipc==0.5.0 greenlet==0.4.5 gunicorn==18.0 -hiredis==0.1.5 -html5lib==0.999 +hiredis==0.2.0 +html5lib==0.99999 +iso8601==0.1.10 itsdangerous==0.24 jsonschema==2.4.0 -marisa-trie==0.7 -mixpanel-py==3.2.1 +marisa-trie==0.7.2 +mixpanel-py==4.0.2 mock==1.0.1 mockldap==0.2.4 +msgpack-python==0.4.6 +netaddr==0.7.14 +netifaces==0.10.4 +oauthlib==0.7.2 +oslo.config==1.11.0 +oslo.i18n==1.6.0 +oslo.serialization==1.5.0 +oslo.utils==1.5.0 paramiko==1.15.2 -peewee==2.4.7 +pbr==0.11.0 +peewee==2.6.0 +prettytable==0.7.2 psutil==2.2.1 -psycopg2==2.5.4 +psycopg2==2.6 py-bcrypt==0.4 +pyOpenSSL==0.15.1 +pyasn1==0.1.7 +pycparser==2.12 pycrypto==2.6.1 -python-dateutil==2.4.0 +pygpgme==0.3 +python-dateutil==2.4.2 +python-keystoneclient==1.4.0 python-ldap==2.4.19 python-magic==0.4.6 -pygpgme==0.3 -pytz==2014.10 -pyOpenSSL==0.14 -raven==5.1.1 +python-swiftclient==2.4.0 +pytz==2015.2 +raven==5.3.0 redis==2.10.3 reportlab==2.7 -requests==2.5.1 +requests==2.6.2 requests-oauthlib==0.4.2 +simplejson==3.7.1 six==1.9.0 +stevedore==1.4.0 stringscore==0.1.0 -stripe==1.20.1 +stripe==1.22.2 trollius==1.0.4 -tzlocal==1.1.2 -urllib3==1.10.2 +tzlocal==1.1.3 +urllib3==1.10.3 waitress==0.8.9 -websocket-client==0.23.0 +websocket-client==0.30.0 wsgiref==0.1.2 xhtml2pdf==0.0.6 git+https://github.com/DevTable/aniso8601-fake.git diff --git a/static/css/core-ui.css b/static/css/core-ui.css index 58ab72b81..adf6bc084 100644 --- a/static/css/core-ui.css +++ b/static/css/core-ui.css @@ -388,6 +388,29 @@ a:focus { width: 400px; } +.config-map-field-element table { + margin-bottom: 10px; +} + +.config-map-field-element .form-control-container { + border-top: 1px solid #eee; + padding-top: 10px; +} + +.config-map-field-element .form-control-container select, .config-map-field-element .form-control-container input { + margin-bottom: 10px; +} + +.config-map-field-element .empty { + color: #ccc; + margin-bottom: 10px; + display: block; +} + +.config-map-field-element .item-title { + font-weight: bold; +} + .config-contact-field { margin-bottom: 4px; } diff --git a/static/directives/config/config-map-field.html b/static/directives/config/config-map-field.html new file mode 100644 index 000000000..7089e2010 --- /dev/null +++ b/static/directives/config/config-map-field.html @@ -0,0 +1,20 @@ +
+ + + + + + +
{{ key }}{{ value }} + Remove +
+ No entries defined +
+ Add Key-Value: + + + +
+
diff --git a/static/directives/config/config-setup-tool.html b/static/directives/config/config-setup-tool.html index 7c2fa2e19..a3a344286 100644 --- a/static/directives/config/config-setup-tool.html +++ b/static/directives/config/config-setup-tool.html @@ -184,6 +184,7 @@ + @@ -192,10 +193,15 @@ {{ field.title }}: + + ng-if="field.kind == 'text'" + is-optional="field.optional">
diff --git a/static/directives/config/config-string-field.html b/static/directives/config/config-string-field.html index 7714fd541..703891f89 100644 --- a/static/directives/config/config-string-field.html +++ b/static/directives/config/config-string-field.html @@ -2,7 +2,7 @@
+ ng-pattern="getRegexp(pattern)" ng-required="!isOptional">
{{ errorMessage }}
diff --git a/static/js/core-config-setup.js b/static/js/core-config-setup.js index 033bbfbb6..93c07c7a6 100644 --- a/static/js/core-config-setup.js +++ b/static/js/core-config-setup.js @@ -78,6 +78,19 @@ angular.module("core-config-setup", ['angularFileUpload']) {'name': 'secret_key', 'title': 'Secret Key', 'placeholder': 'secretkeyhere', 'kind': 'text'}, {'name': 'bucket_name', 'title': 'Bucket Name', 'placeholder': 'my-cool-bucket', 'kind': 'text'}, {'name': 'storage_path', 'title': 'Storage Directory', 'placeholder': '/path/inside/bucket', 'kind': 'text'} + ], + + 'SwiftStorage': [ + {'name': 'auth_url', 'title': 'Swift Auth URL', 'placeholder': '', 'kind': 'text'}, + {'name': 'swift_container', 'title': 'Swift Container Name', 'placeholder': 'mycontainer', 'kind': 'text'}, + {'name': 'storage_path', 'title': 'Storage Path', 'placeholder': '/path/inside/container', 'kind': 'text'}, + + {'name': 'swift_user', 'title': 'Username', 'placeholder': 'accesskeyhere', 'kind': 'text'}, + {'name': 'swift_password', 'title': 'Password/Key', 'placeholder': 'secretkeyhere', 'kind': 'text'}, + + {'name': 'ca_cert_path', 'title': 'CA Cert Filename', 'placeholder': 'conf/stack/swift.cert', 'kind': 'text', 'optional': true}, + {'name': 'os_options', 'title': 'OS Options', 'kind': 'map', + 'keys': ['tenant_id', 'auth_token', 'service_type', 'endpoint_type', 'tenant_name', 'object_storage_url', 'region_name']} ] }; @@ -760,6 +773,42 @@ angular.module("core-config-setup", ['angularFileUpload']) return directiveDefinitionObject; }) + .directive('configMapField', function () { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/config/config-map-field.html', + replace: false, + transclude: false, + restrict: 'C', + scope: { + 'binding': '=binding', + 'keys': '=keys' + }, + controller: function($scope, $element) { + $scope.newKey = null; + $scope.newValue = null; + + $scope.hasValues = function(binding) { + return binding && Object.keys(binding).length; + }; + + $scope.removeKey = function(key) { + delete $scope.binding[key]; + }; + + $scope.addEntry = function() { + if (!$scope.newKey || !$scope.newValue) { return; } + + $scope.binding = $scope.binding || {}; + $scope.binding[$scope.newKey] = $scope.newValue; + $scope.newKey = null; + $scope.newValue = null; + } + } + }; + return directiveDefinitionObject; + }) + .directive('configStringField', function () { var directiveDefinitionObject = { priority: 0, @@ -772,7 +821,8 @@ angular.module("core-config-setup", ['angularFileUpload']) 'placeholder': '@placeholder', 'pattern': '@pattern', 'defaultValue': '@defaultValue', - 'validator': '&validator' + 'validator': '&validator', + 'isOptional': '=isOptional' }, controller: function($scope, $element) { $scope.getRegexp = function(pattern) { diff --git a/storage/__init__.py b/storage/__init__.py index 7893343c2..69f26def4 100644 --- a/storage/__init__.py +++ b/storage/__init__.py @@ -2,6 +2,7 @@ from storage.local import LocalStorage from storage.cloud import S3Storage, GoogleCloudStorage, RadosGWStorage from storage.fakestorage import FakeStorage from storage.distributedstorage import DistributedStorage +from storage.swift import SwiftStorage STORAGE_DRIVER_CLASSES = { @@ -9,6 +10,7 @@ STORAGE_DRIVER_CLASSES = { 'S3Storage': S3Storage, 'GoogleCloudStorage': GoogleCloudStorage, 'RadosGWStorage': RadosGWStorage, + 'SwiftStorage': SwiftStorage, } def get_storage_driver(storage_params): diff --git a/storage/swift.py b/storage/swift.py new file mode 100644 index 000000000..243202223 --- /dev/null +++ b/storage/swift.py @@ -0,0 +1,183 @@ +""" Swift storage driver. Based on: github.com/bacongobbler/docker-registry-driver-swift/ """ +from swiftclient.client import Connection, ClientException +from storage.basestorage import BaseStorage + +from random import SystemRandom +import string +import logging + +logger = logging.getLogger(__name__) + +class SwiftStorage(BaseStorage): + def __init__(self, swift_container, storage_path, auth_url, swift_user, + swift_password, auth_version=None, os_options=None, ca_cert_path=None): + self._swift_container = swift_container + self._storage_path = storage_path + + self._auth_url = auth_url + self._ca_cert_path = ca_cert_path + + self._swift_user = swift_user + self._swift_password = swift_password + + self._auth_version = auth_version or 2 + self._os_options = os_options or {} + + self._initialized = False + self._swift_connection = None + + def _initialize(self): + if self._initialized: + return + + self._initialized = True + self._swift_connection = self._get_connection() + + def _get_connection(self): + return Connection( + authurl=self._auth_url, + cacert=self._ca_cert_path, + + user=self._swift_user, + key=self._swift_password, + + auth_version=self._auth_version, + os_options=self._os_options) + + def _get_relative_path(self, path): + if path.startswith(self._storage_path): + path = path[len(self._storage_path)] + + if path.endswith('/'): + path = path[:-1] + + return path + + def _normalize_path(self, path=None): + path = self._storage_path + (path or '') + + # Openstack does not like paths starting with '/' and we always normalize + # to remove trailing '/' + if path.startswith('/'): + path = path[1:] + + if path.endswith('/'): + path = path[:-1] + + return path + + def _get_container(self, path): + self._initialize() + path = self._normalize_path(path) + + if path and not path.endswith('/'): + path += '/' + + try: + _, container = self._swift_connection.get_container( + container=self._swift_container, + prefix=path, delimiter='/') + return container + except: + logger.exception('Could not get container: %s', path) + raise IOError('Unknown path: %s' % path) + + def _get_object(self, path, chunk_size=None): + self._initialize() + path = self._normalize_path(path) + try: + _, obj = self._swift_connection.get_object(self._swift_container, path, + resp_chunk_size=chunk_size) + return obj + except Exception: + logger.exception('Could not get object: %s', path) + raise IOError('Path %s not found' % path) + + def _put_object(self, path, content, chunk=None, content_type=None, content_encoding=None): + self._initialize() + path = self._normalize_path(path) + headers = {} + + if content_encoding is not None: + headers['Content-Encoding'] = content_encoding + + try: + self._swift_connection.put_object(self._swift_container, path, content, + chunk_size=chunk, content_type=content_type, + headers=headers) + except ClientException: + raise + except Exception: + logger.exception('Could not put object: %s', path) + raise IOError("Could not put content: %s" % path) + + def _head_object(self, path): + self._initialize() + path = self._normalize_path(path) + try: + return self._swift_connection.head_object(self._swift_container, path) + except Exception: + logger.exception('Could not head object: %s', path) + return None + + def get_direct_download_url(self, path, expires_in=60, requires_cors=False): + if requires_cors: + return None + + # TODO: http://docs.openstack.org/juno/config-reference/content/object-storage-tempurl.html + return None + + def get_content(self, path): + return self._get_object(path) + + def put_content(self, path, content): + self._put_object(path, content) + + def stream_read(self, path): + for data in self._get_object(path, self.buffer_size): + yield data + + def stream_read_file(self, path): + raise NotImplementedError + + def stream_write(self, path, fp, content_type=None, content_encoding=None): + self._put_object(path, fp, self.buffer_size, content_type=content_type, + content_encoding=content_encoding) + + def list_directory(self, path=None): + container = self._get_container(path) + if not container: + raise OSError('Unknown path: %s' % path) + + for entry in container: + param = None + if 'name' in entry: + param = 'name' + elif 'subdir' in entry: + param = 'subdir' + else: + continue + + yield self._get_relative_path(entry[param]) + + def exists(self, path): + return bool(self._head_object(path)) + + def remove(self, path): + self._initialize() + path = self._normalize_path(path) + try: + self._swift_connection.delete_object(self._swift_container, path) + except Exception: + raise IOError('Cannot delete path: %s' % path) + + def _random_checksum(self, count): + chars = string.ascii_uppercase + string.digits + return ''.join(SystemRandom().choice(chars) for _ in range(count)) + + def get_checksum(self, path): + headers = self._head_object(path) + if not headers: + raise IOError('Cannot lookup path: %s' % path) + + return headers.get('etag', '')[1:-1][:7] or self._random_checksum(7) From dbd119c365a5c61b910f48e4894df3390140af64 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 20 May 2015 17:40:43 -0400 Subject: [PATCH 02/24] Fix the DB health check Make sure to search for the proper DB identifier --- health/healthcheck.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/health/healthcheck.py b/health/healthcheck.py index 11a365e34..98de22435 100644 --- a/health/healthcheck.py +++ b/health/healthcheck.py @@ -77,10 +77,11 @@ class LocalHealthCheck(HealthCheck): class ProductionHealthCheck(HealthCheck): - def __init__(self, app, access_key, secret_key): + def __init__(self, app, access_key, secret_key, db_instance='quay'): super(ProductionHealthCheck, self).__init__(app) self.access_key = access_key self.secret_key = secret_key + self.db_instance = db_instance @classmethod def check_name(cls): @@ -115,7 +116,10 @@ class ProductionHealthCheck(HealthCheck): aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key) response = region.describe_db_instances()['DescribeDBInstancesResponse'] result = response['DescribeDBInstancesResult'] - instances = result['DBInstances'] + instances = [i for i in result['DBInstances'] if i['DBInstanceIdentifier'] == self.db_instance] + if not instances: + return 'error' + status = instances[0]['DBInstanceStatus'] return status except: From 0f18fc1c269a065b9c3a70d678a0961ef1c00e3f Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 21 May 2015 17:16:38 -0400 Subject: [PATCH 03/24] Disable the angular poll channel when the browser tab is hidden Quay pages that normally poll (repo view, build logs, etc) will skip the API call(s) when the tab is hidden. --- static/js/services/angular-poll-channel.js | 9 ++- .../services/document-visibility-service.js | 60 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 static/js/services/document-visibility-service.js diff --git a/static/js/services/angular-poll-channel.js b/static/js/services/angular-poll-channel.js index adba49757..f4028a65f 100644 --- a/static/js/services/angular-poll-channel.js +++ b/static/js/services/angular-poll-channel.js @@ -1,7 +1,8 @@ /** * Specialized class for conducting an HTTP poll, while properly preventing multiple calls. */ -angular.module('quay').factory('AngularPollChannel', ['ApiService', '$timeout', function(ApiService, $timeout) { +angular.module('quay').factory('AngularPollChannel', ['ApiService', '$timeout', 'DocumentVisibilityService', + function(ApiService, $timeout, DocumentVisibilityService) { var _PollChannel = function(scope, requester, opt_sleeptime) { this.scope_ = scope; this.requester_ = requester; @@ -50,6 +51,12 @@ angular.module('quay').factory('AngularPollChannel', ['ApiService', '$timeout', _PollChannel.prototype.call_ = function() { if (this.working) { return; } + // If the document is currently hidden, skip the call. + if (DocumentVisibilityService.isHidden()) { + this.setupTimer_(); + return; + } + var that = this; this.working = true; this.scope_.$apply(function() { diff --git a/static/js/services/document-visibility-service.js b/static/js/services/document-visibility-service.js new file mode 100644 index 000000000..f56ecc633 --- /dev/null +++ b/static/js/services/document-visibility-service.js @@ -0,0 +1,60 @@ +/** + * Helper service which fires off events when the document's visibility changes, as well as allowing + * other Angular code to query the state of the document's visibility directly. + */ +angular.module('quay').constant('CORE_EVENT', { + DOC_VISIBILITY_CHANGE: 'core.event.doc_visibility_change' +}); + +angular.module('quay').factory('DocumentVisibilityService', ['$rootScope', '$document', 'CORE_EVENT', + function($rootScope, $document, CORE_EVENT) { + var document = $document[0], + features, + detectedFeature; + + function broadcastChangeEvent() { + $rootScope.$broadcast(CORE_EVENT.DOC_VISIBILITY_CHANGE, + document[detectedFeature.propertyName]); + } + + features = { + standard: { + eventName: 'visibilitychange', + propertyName: 'hidden' + }, + moz: { + eventName: 'mozvisibilitychange', + propertyName: 'mozHidden' + }, + ms: { + eventName: 'msvisibilitychange', + propertyName: 'msHidden' + }, + webkit: { + eventName: 'webkitvisibilitychange', + propertyName: 'webkitHidden' + } + }; + + Object.keys(features).some(function(feature) { + if (document[features[feature].propertyName] !== undefined) { + detectedFeature = features[feature]; + return true; + } + }); + + if (detectedFeature) { + $document.on(detectedFeature.eventName, broadcastChangeEvent); + } + + return { + /** + * Is the window currently hidden or not. + */ + isHidden: function() { + if (detectedFeature) { + return document[detectedFeature.propertyName]; + } + } + }; +}]); \ No newline at end of file From 4030b0a470fed640c8f194aee805ab5e732c7b65 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 22 May 2015 15:24:14 -0400 Subject: [PATCH 04/24] - Have the heartbeat fail to update if the worker has timed out - Add additional build component logging for tracking down problems in the future --- buildman/component/buildcomponent.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/buildman/component/buildcomponent.py b/buildman/component/buildcomponent.py index f5703bb65..a59cb421a 100644 --- a/buildman/component/buildcomponent.py +++ b/buildman/component/buildcomponent.py @@ -55,6 +55,7 @@ class BuildComponent(BaseComponent): def onConnect(self): self.join(self.builder_realm) + @trollius.coroutine def onJoin(self, details): logger.debug('Registering methods and listeners for component %s', self.builder_realm) yield trollius.From(self.register(self._on_ready, u'io.quay.buildworker.ready')) @@ -277,6 +278,9 @@ class BuildComponent(BaseComponent): # Send the notification that the build has completed successfully. self._current_job.send_notification('build_success', image_id=kwargs.get('image_id')) except ApplicationError as aex: + build_id = self._current_job.repo_build.uuid + logger.exception('Got remote exception for build: %s', build_id) + worker_error = WorkerError(aex.error, aex.kwargs.get('base_error')) # Write the error to the log. @@ -310,6 +314,7 @@ class BuildComponent(BaseComponent): @trollius.coroutine def _on_ready(self, token, version): + logger.debug('On ready called (token "%s")', token) self._worker_version = version if not version in SUPPORTED_WORKER_VERSIONS: @@ -343,6 +348,10 @@ class BuildComponent(BaseComponent): def _on_heartbeat(self): """ Updates the last known heartbeat. """ + if not self._current_job or self._component_status == ComponentStatus.TIMED_OUT: + return + + logger.debug('Got heartbeat for build %s', self._current_job.repo_build.uuid) self._last_heartbeat = datetime.datetime.utcnow() @trollius.coroutine @@ -374,9 +383,15 @@ class BuildComponent(BaseComponent): logger.debug('Checking heartbeat on realm %s', self.builder_realm) if (self._last_heartbeat and self._last_heartbeat < datetime.datetime.utcnow() - HEARTBEAT_DELTA): + logger.debug('Heartbeat on realm %s has expired: %s', self.builder_realm, + self._last_heartbeat) + yield trollius.From(self._timeout()) raise trollius.Return() + logger.debug('Heartbeat on realm %s is valid: %s.', self.builder_realm, + self._last_heartbeat) + yield trollius.From(trollius.sleep(HEARTBEAT_TIMEOUT)) @trollius.coroutine From 88ece113eef60fdfadf1b4f1921b333535663296 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 20 May 2015 14:53:31 -0400 Subject: [PATCH 05/24] Explicitly enable LDAP referrals Note: The mock LDAP system doesn't support referrals, so we can't add a unit test for this. --- data/users.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data/users.py b/data/users.py index a50d462b9..22c51273e 100644 --- a/data/users.py +++ b/data/users.py @@ -43,6 +43,7 @@ class LDAPConnection(object): def __enter__(self): trace_level = 2 if os.environ.get('LDAP_DEBUG') == '1' else 0 self._conn = ldap.initialize(self._ldap_uri, trace_level=trace_level) + self._conn.set_option(ldap.OPT_REFERRALS, 1) self._conn.simple_bind_s(self._user_dn, self._user_pw) return self._conn From f6fea27c12d738a80de6822c4b5fb3863944a3e8 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 20 May 2015 16:37:09 -0400 Subject: [PATCH 06/24] Fix encrypted password generator to use the LDAP username, not the Quay username. Currently, we use the Quay username via `verify_user` when we go to create the encrypted password. This is only correct if Quay has not generated its own different username for the LDAP user, and fails if it has. We therefore add a new method `confirm_existing_user`, which looks up the federated login for the LDAP user and then runs the auth flow using that username. --- data/model/legacy.py | 6 ++++++ data/users.py | 28 +++++++++++++++++++++++++++- endpoints/api/user.py | 3 +-- test/test_ldap.py | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 3 deletions(-) diff --git a/data/model/legacy.py b/data/model/legacy.py index 67daaa540..16191b8a6 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -566,6 +566,12 @@ def list_federated_logins(user): FederatedLogin.user == user) +def lookup_federated_login(user, service_name): + try: + return list_federated_logins(user).where(LoginService.name == service_name).get() + except FederatedLogin.DoesNotExist: + return None + def create_confirm_email_code(user, new_email=None): if new_email: if not validate_email(new_email): diff --git a/data/users.py b/data/users.py index 22c51273e..1accd1fdc 100644 --- a/data/users.py +++ b/data/users.py @@ -28,6 +28,8 @@ class DatabaseUsers(object): return (result, None) + def confirm_existing_user(self, username, password): + return self.verify_user(username, password) def user_exists(self, username): return model.get_user(username) is not None @@ -86,7 +88,21 @@ class LDAPUsers(object): return None - def verify_user(self, username_or_email, password): + def confirm_existing_user(self, username, password): + """ Verify the username and password by looking up the *LDAP* username and confirming the + password. + """ + db_user = model.get_user(username) + if not db_user: + return (None, 'Invalid user') + + federated_login = model.lookup_federated_login(db_user, 'ldap') + if not federated_login: + return (None, 'Invalid user') + + return self.verify_user(federated_login.service_ident, password, create_new_user=False) + + def verify_user(self, username_or_email, password, create_new_user=True): """ Verify the credentials with LDAP and if they are valid, create or update the user in our database. """ @@ -122,6 +138,9 @@ class LDAPUsers(object): db_user = model.verify_federated_login('ldap', username) if not db_user: + if not create_new_user: + return (None, 'Invalid user') + # We must create the user in our db valid_username = None for valid_username in generate_valid_usernames(username): @@ -233,6 +252,13 @@ class UserAuthentication(object): return data.get('password', encrypted) + def confirm_existing_user(self, username, password): + """ Verifies that the given password matches to the given DB username. Unlike verify_user, this + call first translates the DB user via the FederatedLogin table (where applicable). + """ + return self.state.confirm_existing_user(username, password) + + def verify_user(self, username_or_email, password, basic_auth=False): # First try to decode the password as a signed token. if basic_auth: diff --git a/endpoints/api/user.py b/endpoints/api/user.py index b03c5f87b..161c97c88 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -370,8 +370,7 @@ class ClientKey(ApiResource): """ Return's the user's private client key. """ username = get_authenticated_user().username password = request.get_json()['password'] - - (result, error_message) = authentication.verify_user(username, password) + (result, error_message) = authentication.confirm_existing_user(username, password) if not result: raise request_error(message=error_message) diff --git a/test/test_ldap.py b/test/test_ldap.py index 323a87a4e..49fd5979c 100644 --- a/test/test_ldap.py +++ b/test/test_ldap.py @@ -38,6 +38,13 @@ class TestLDAP(unittest.TestCase): 'ou': 'employees', 'uid': ['nomail'], 'userPassword': ['somepass'] + }, + 'uid=cool.user,ou=employees,dc=quay,dc=io': { + 'dc': ['quay', 'io'], + 'ou': 'employees', + 'uid': ['cool.user'], + 'userPassword': ['somepass'], + 'mail': ['foo@bar.com'] } }) @@ -59,9 +66,14 @@ class TestLDAP(unittest.TestCase): ldap = LDAPUsers('ldap://localhost', base_dn, admin_dn, admin_passwd, user_rdn, uid_attr, email_attr) + # Verify we can login. (response, _) = ldap.verify_user('someuser', 'somepass') self.assertEquals(response.username, 'someuser') + # Verify we can confirm the user. + (response, _) = ldap.confirm_existing_user('someuser', 'somepass') + self.assertEquals(response.username, 'someuser') + def test_missing_mail(self): base_dn = ['dc=quay', 'dc=io'] admin_dn = 'uid=testy,ou=employees,dc=quay,dc=io' @@ -77,6 +89,29 @@ class TestLDAP(unittest.TestCase): self.assertIsNone(response) self.assertEquals('Missing mail field "mail" in user record', err_msg) + def test_confirm_different_username(self): + base_dn = ['dc=quay', 'dc=io'] + admin_dn = 'uid=testy,ou=employees,dc=quay,dc=io' + admin_passwd = 'password' + user_rdn = ['ou=employees'] + uid_attr = 'uid' + email_attr = 'mail' + + ldap = LDAPUsers('ldap://localhost', base_dn, admin_dn, admin_passwd, user_rdn, + uid_attr, email_attr) + + # Verify that the user is logged in and their username was adjusted. + (response, _) = ldap.verify_user('cool.user', 'somepass') + self.assertEquals(response.username, 'cool_user') + + # Verify we can confirm the user's quay username. + (response, _) = ldap.confirm_existing_user('cool_user', 'somepass') + self.assertEquals(response.username, 'cool_user') + + # Verify that we *cannot* confirm the LDAP username. + (response, _) = ldap.confirm_existing_user('cool.user', 'somepass') + self.assertIsNone(response) + if __name__ == '__main__': unittest.main() From 4f6234ea8fe59f50659af2a2e0155542eabdbcec Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 20 May 2015 16:31:00 -0400 Subject: [PATCH 07/24] nginx: enable Strict Transport Security --- conf/nginx.conf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conf/nginx.conf b/conf/nginx.conf index 77a78f70e..8375febd0 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -10,8 +10,9 @@ http { server { include server-base.conf; - listen 443 default; + add_header Strict-Transport-Security "max-age=63072000; preload"; + listen 443 default; ssl on; ssl_certificate ./stack/ssl.cert; ssl_certificate_key ./stack/ssl.key; @@ -25,8 +26,9 @@ http { include proxy-protocol.conf; include server-base.conf; - listen 8443 default proxy_protocol; + add_header Strict-Transport-Security "max-age=63072000; preload"; + listen 8443 default proxy_protocol; ssl on; ssl_certificate ./stack/ssl.cert; ssl_certificate_key ./stack/ssl.key; From 2a03f4d070f0e53adf7b77c3df05fe8cd843653b Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 20 May 2015 16:31:32 -0400 Subject: [PATCH 08/24] nginx: drop SSLv3, support TLS 1.1 & 1.2 --- conf/nginx.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/nginx.conf b/conf/nginx.conf index 8375febd0..9e3aead80 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -17,7 +17,7 @@ http { ssl_certificate ./stack/ssl.cert; ssl_certificate_key ./stack/ssl.key; ssl_session_timeout 5m; - ssl_protocols SSLv3 TLSv1; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; ssl_prefer_server_ciphers on; } @@ -33,7 +33,7 @@ http { ssl_certificate ./stack/ssl.cert; ssl_certificate_key ./stack/ssl.key; ssl_session_timeout 5m; - ssl_protocols SSLv3 TLSv1; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; ssl_prefer_server_ciphers on; } From ccfebdf22b642ea1112e62296f3682b871eb1843 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 20 May 2015 16:32:12 -0400 Subject: [PATCH 09/24] nginx: support OCSP Stapling --- conf/nginx.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conf/nginx.conf b/conf/nginx.conf index 9e3aead80..f04ed663c 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -16,6 +16,8 @@ http { ssl on; ssl_certificate ./stack/ssl.cert; ssl_certificate_key ./stack/ssl.key; + ssl_stapling on; + ssl_stapling_verify on; ssl_session_timeout 5m; ssl_protocols TLSv1 TLSv1.1 TLSv1.2; ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; @@ -32,6 +34,8 @@ http { ssl on; ssl_certificate ./stack/ssl.cert; ssl_certificate_key ./stack/ssl.key; + ssl_stapling on; + ssl_stapling_verify on; ssl_session_timeout 5m; ssl_protocols TLSv1 TLSv1.1 TLSv1.2; ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; From 0c15c2888d9fd3b0b1f1ff0b36a2fe8d24762fe5 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 22 May 2015 13:35:49 -0400 Subject: [PATCH 10/24] nginx: update cipher suite, HSTS, X-Frame-Options --- conf/nginx.conf | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/conf/nginx.conf b/conf/nginx.conf index f04ed663c..ca872b224 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -10,35 +10,49 @@ http { server { include server-base.conf; - add_header Strict-Transport-Security "max-age=63072000; preload"; - listen 443 default; + ssl on; ssl_certificate ./stack/ssl.cert; ssl_certificate_key ./stack/ssl.key; + + ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4"; + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 5m; + ssl_stapling on; ssl_stapling_verify on; - ssl_session_timeout 5m; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; + ssl_prefer_server_ciphers on; + + add_header Strict-Transport-Security "max-age=63072000; preload"; + add_header X-Frame-Options DENY; } server { include proxy-protocol.conf; include server-base.conf; - add_header Strict-Transport-Security "max-age=63072000; preload"; - listen 8443 default proxy_protocol; + ssl on; ssl_certificate ./stack/ssl.cert; ssl_certificate_key ./stack/ssl.key; + + ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4"; + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 5m; + ssl_stapling on; ssl_stapling_verify on; - ssl_session_timeout 5m; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; + ssl_prefer_server_ciphers on; + + add_header Strict-Transport-Security "max-age=63072000; preload"; + add_header X-Frame-Options DENY; } } From 5db4e58e16cde6d5dc6f8da9bf68db320f1ecb25 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 22 May 2015 13:54:43 -0400 Subject: [PATCH 11/24] nginx: SSL config into server-base.conf --- conf/nginx.conf | 30 ++---------------------------- conf/server-base.conf | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/conf/nginx.conf b/conf/nginx.conf index ca872b224..860ddae51 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -13,22 +13,9 @@ http { listen 443 default; ssl on; - ssl_certificate ./stack/ssl.cert; - ssl_certificate_key ./stack/ssl.key; - - ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4"; - - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_session_cache shared:SSL:10m; - ssl_session_timeout 5m; - - ssl_stapling on; - ssl_stapling_verify on; - - ssl_prefer_server_ciphers on; + # This header must be set only for HTTPS add_header Strict-Transport-Security "max-age=63072000; preload"; - add_header X-Frame-Options DENY; } server { @@ -38,21 +25,8 @@ http { listen 8443 default proxy_protocol; ssl on; - ssl_certificate ./stack/ssl.cert; - ssl_certificate_key ./stack/ssl.key; - - ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4"; - - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_session_cache shared:SSL:10m; - ssl_session_timeout 5m; - - ssl_stapling on; - ssl_stapling_verify on; - - ssl_prefer_server_ciphers on; + # This header must be set only for HTTPS add_header Strict-Transport-Security "max-age=63072000; preload"; - add_header X-Frame-Options DENY; } } diff --git a/conf/server-base.conf b/conf/server-base.conf index 3853fbccf..1ff261e6b 100644 --- a/conf/server-base.conf +++ b/conf/server-base.conf @@ -8,6 +8,20 @@ if ($args ~ "_escaped_fragment_") { rewrite ^ /snapshot$uri; } +# SSL +ssl_certificate ./stack/ssl.cert; +ssl_certificate_key ./stack/ssl.key; +ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4"; +ssl_protocols TLSv1 TLSv1.1 TLSv1.2; +ssl_session_cache shared:SSL:10m; +ssl_session_timeout 5m; +ssl_stapling on; +ssl_stapling_verify on; +ssl_prefer_server_ciphers on; +add_header X-Frame-Options DENY; + + +# Proxy Headers proxy_set_header X-Forwarded-For $proper_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Host $http_host; From 5db790bb30f098238c0655921ef07d45c57633d2 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 22 May 2015 16:09:11 -0400 Subject: [PATCH 12/24] setup-tool: add HSTS info box --- static/directives/config/config-setup-tool.html | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/static/directives/config/config-setup-tool.html b/static/directives/config/config-setup-tool.html index a3a344286..065d55134 100644 --- a/static/directives/config/config-setup-tool.html +++ b/static/directives/config/config-setup-tool.html @@ -98,6 +98,11 @@ A valid SSL certificate and private key files are required to use this option.
+
+ Enabling SSL also enables HTTP Strict Transport Security.
+ This prevents downgrade attacks and cookie theft, but browsers will reject all future insecure connections on this hostname. +
+ @@ -841,4 +846,4 @@ - \ No newline at end of file + From 0359d3f3799d48316919c3b214cf21bf77ca2845 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 22 May 2015 16:25:28 -0400 Subject: [PATCH 13/24] nginx: move ssl config out of server-base --- conf/nginx.conf | 11 +++++++++++ conf/server-base.conf | 11 +---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/conf/nginx.conf b/conf/nginx.conf index 860ddae51..5e49b1977 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -7,6 +7,16 @@ http { include hosted-http-base.conf; include rate-limiting.conf; + ssl_certificate ./stack/ssl.cert; + ssl_certificate_key ./stack/ssl.key; + ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4"; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 5m; + ssl_stapling on; + ssl_stapling_verify on; + ssl_prefer_server_ciphers on; + server { include server-base.conf; @@ -16,6 +26,7 @@ http { # This header must be set only for HTTPS add_header Strict-Transport-Security "max-age=63072000; preload"; + } server { diff --git a/conf/server-base.conf b/conf/server-base.conf index 1ff261e6b..bfa6c012f 100644 --- a/conf/server-base.conf +++ b/conf/server-base.conf @@ -8,16 +8,7 @@ if ($args ~ "_escaped_fragment_") { rewrite ^ /snapshot$uri; } -# SSL -ssl_certificate ./stack/ssl.cert; -ssl_certificate_key ./stack/ssl.key; -ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4"; -ssl_protocols TLSv1 TLSv1.1 TLSv1.2; -ssl_session_cache shared:SSL:10m; -ssl_session_timeout 5m; -ssl_stapling on; -ssl_stapling_verify on; -ssl_prefer_server_ciphers on; +# Disable the ability to be embedded into iframes add_header X-Frame-Options DENY; From 162dcf05e31e7aeda024e270ed7004f7caee650f Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 22 May 2015 16:26:26 -0400 Subject: [PATCH 14/24] Have the verifyUser endpoint use the same confirm_existing_user method This will prevent us from encountering the same problem as the generated encrypted password issue when using LDAP --- endpoints/api/user.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 161c97c88..2d17073aa 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -537,7 +537,17 @@ class VerifyUser(ApiResource): """ Verifies the signed in the user with the specified credentials. """ signin_data = request.get_json() password = signin_data['password'] - return conduct_signin(get_authenticated_user().username, password) + + username = get_authenticated_user().username + (result, error_message) = authentication.confirm_existing_user(username, password) + if not result: + return { + 'message': error_message, + 'invalidCredentials': True, + }, 403 + + common_login(result) + return {'success': True} @resource('/v1/signout') From 597a86f50114305468f54f964509c508e018e797 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 26 May 2015 13:40:21 -0400 Subject: [PATCH 15/24] Fix case where the auth token was not written properly for BitBucket --- endpoints/trigger.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/endpoints/trigger.py b/endpoints/trigger.py index 2bcdc6242..311c6aaf5 100644 --- a/endpoints/trigger.py +++ b/endpoints/trigger.py @@ -244,7 +244,11 @@ class BitbucketBuildTrigger(BuildTriggerHandler): def _get_authorized_client(self): base_client = self._get_client() auth_token = self.auth_token or 'invalid:invalid' - (access_token, access_token_secret) = auth_token.split(':') + token_parts = auth_token.split(':') + if len(token_parts) != 2: + token_parts = ['invalid', 'invalid'] + + (access_token, access_token_secret) = token_parts return base_client.get_authorized_client(access_token, access_token_secret) def _get_repository_client(self): From d1fa155eee2976ec09327f2eae809db033dd7bb0 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 26 May 2015 13:43:51 -0400 Subject: [PATCH 16/24] Fix NPE --- static/js/pages/new-organization.js | 1 + 1 file changed, 1 insertion(+) diff --git a/static/js/pages/new-organization.js b/static/js/pages/new-organization.js index 85c451115..37613c1e7 100644 --- a/static/js/pages/new-organization.js +++ b/static/js/pages/new-organization.js @@ -45,6 +45,7 @@ $scope.signinStarted = function() { if (Features.BILLING) { PlanService.getMinimumPlan(1, true, function(plan) { + if (!plan) { return; } PlanService.notePlan(plan.stripeId); }); } From 58685f02cd06558253572f5743a1dcac19df6925 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 26 May 2015 13:46:41 -0400 Subject: [PATCH 17/24] Fix NPE in notifications service --- static/js/services/notification-service.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/static/js/services/notification-service.js b/static/js/services/notification-service.js index ade256a63..47eddfd2b 100644 --- a/static/js/services/notification-service.js +++ b/static/js/services/notification-service.js @@ -196,6 +196,10 @@ function($rootScope, $interval, UserService, ApiService, StringBuilderService, P }; notificationService.getClasses = function(notifications) { + if (!notifications.length) { + return ''; + } + var classes = []; for (var i = 0; i < notifications.length; ++i) { var notification = notifications[i]; From 2e4893dce09b4f18597e9d640163f0fc1945d166 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 26 May 2015 13:49:58 -0400 Subject: [PATCH 18/24] We only add the build to the build list if present, not if missing --- static/js/directives/repo-view/repo-panel-builds.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/js/directives/repo-view/repo-panel-builds.js b/static/js/directives/repo-view/repo-panel-builds.js index a1d8fdd31..72b3c05f5 100644 --- a/static/js/directives/repo-view/repo-panel-builds.js +++ b/static/js/directives/repo-view/repo-panel-builds.js @@ -260,7 +260,7 @@ angular.module('quay').directive('repoPanelBuilds', function () { }; $scope.handleBuildStarted = function(build) { - if (!$scope.allBuilds) { + if ($scope.allBuilds) { $scope.allBuilds.push(build); } updateBuilds(); From 7001fb05bf74350cf668293d7a2bb808f5ff4f5c Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 26 May 2015 16:34:59 -0400 Subject: [PATCH 19/24] Add further comments on the TODO in get_direct_download_url --- storage/swift.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/storage/swift.py b/storage/swift.py index 243202223..5fcc98a96 100644 --- a/storage/swift.py +++ b/storage/swift.py @@ -124,7 +124,10 @@ class SwiftStorage(BaseStorage): if requires_cors: return None - # TODO: http://docs.openstack.org/juno/config-reference/content/object-storage-tempurl.html + # TODO(jschorr): This method is not strictly necessary but would result in faster operations + # when using this storage engine. However, the implementation (as seen in the link below) + # is not clean, so we punt on this for now. + # http://docs.openstack.org/juno/config-reference/content/object-storage-tempurl.html return None def get_content(self, path): From 375d7670a8179d99cc229b8991d69ad8e3e5ea96 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 26 May 2015 16:35:12 -0400 Subject: [PATCH 20/24] Explain why we re-raise ClientException in the swift storage engine --- storage/swift.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/swift.py b/storage/swift.py index 5fcc98a96..ddeae9105 100644 --- a/storage/swift.py +++ b/storage/swift.py @@ -106,6 +106,8 @@ class SwiftStorage(BaseStorage): chunk_size=chunk, content_type=content_type, headers=headers) except ClientException: + # We re-raise client exception here so that validation of config during setup can see + # the client exception messages. raise except Exception: logger.exception('Could not put object: %s', path) From 9888c3ad9b6c4e35bf3d3cb319dc165f0157965d Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 26 May 2015 17:22:30 -0400 Subject: [PATCH 21/24] Add an endpoint for downloading the logs of a build. --- endpoints/web.py | 31 ++++++++++++++++++-- static/css/directives/ui/build-logs-view.css | 18 ++++++++++++ static/directives/build-logs-view.html | 6 ++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/endpoints/web.py b/endpoints/web.py index 884f4e6ea..6166a7956 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -9,10 +9,11 @@ from health.healthcheck import get_healthchecker from data import model from data.model.oauth import DatabaseAuthorizationProvider -from app import app, billing as stripe, build_logs, avatar, signer +from app import app, billing as stripe, build_logs, avatar, signer, log_archive from auth.auth import require_session_login, process_oauth from auth.permissions import (AdministerOrganizationPermission, ReadRepositoryPermission, - SuperUserPermission, AdministerRepositoryPermission) + SuperUserPermission, AdministerRepositoryPermission, + ModifyRepositoryPermission) from util.invoice import renderInvoiceToPdf from util.seo import render_snapshot @@ -248,6 +249,32 @@ def robots(): return send_from_directory('static', 'robots.txt') +@web.route('/buildlogs/', methods=['GET']) +@route_show_if(features.BUILD_SUPPORT) +@require_session_login +def buildlogs(build_uuid): + build = model.get_repository_build(build_uuid) + if not build: + abort(403) + + repo = build.repository + if not ModifyRepositoryPermission(repo.namespace_user.username, repo.name).can(): + abort(403) + + # If the logs have been archived, just return a URL of the completed archive + if build.logs_archived: + return redirect(log_archive.get_file_url(build.uuid)) + + _, logs = build_logs.get_log_entries(build.uuid, 0) + response = jsonify({ + 'logs': [log for log in logs] + }) + + response.mimetype = "application/json" + response.headers["Content-Disposition"] = "attachment;filename=" + build.uuid + ".json" + return response + + @web.route('/receipt', methods=['GET']) @route_show_if(features.BILLING) @require_session_login diff --git a/static/css/directives/ui/build-logs-view.css b/static/css/directives/ui/build-logs-view.css index e69a3ad57..746a6c1d7 100644 --- a/static/css/directives/ui/build-logs-view.css +++ b/static/css/directives/ui/build-logs-view.css @@ -157,6 +157,24 @@ transition: all 0.15s ease-in-out; } +.build-logs-view .download-button i.fa { + margin-right: 10px; +} + +.build-logs-view .download-button { + position: absolute; + top: 6px; + right: 124px; + z-index: 2; + transition: all 0.15s ease-in-out; +} + +.build-logs-view .download-button:not(:hover) { + background: transparent; + border: 1px solid transparent; + color: #ddd; +} + .build-logs-view .copy-button:not(.zeroclipboard-is-hover) { background: transparent; border: 1px solid transparent; diff --git a/static/directives/build-logs-view.html b/static/directives/build-logs-view.html index f99a7eb00..aa79cec8a 100644 --- a/static/directives/build-logs-view.html +++ b/static/directives/build-logs-view.html @@ -3,6 +3,12 @@ Copy Logs + + Download Logs + + From b3ea4ecaa25e61f411cedcb09b7945cd3830aa4b Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 26 May 2015 17:30:10 -0400 Subject: [PATCH 22/24] Remove unneeded mime type set; jsonify does this for us --- endpoints/web.py | 1 - 1 file changed, 1 deletion(-) diff --git a/endpoints/web.py b/endpoints/web.py index 6166a7956..7e61589af 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -270,7 +270,6 @@ def buildlogs(build_uuid): 'logs': [log for log in logs] }) - response.mimetype = "application/json" response.headers["Content-Disposition"] = "attachment;filename=" + build.uuid + ".json" return response From bd262bbb3f37e0b07475fd0e0eb2c2a01fd23b42 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 26 May 2015 18:29:04 -0400 Subject: [PATCH 23/24] Make sure there is always a way to create a repo notification Before this change, the button was hidden on small sizes, but the link was only shown on extra-small sizes, leaving a small window where there was no way to create a new notification --- static/directives/repository-events-table.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/static/directives/repository-events-table.html b/static/directives/repository-events-table.html index 5ec0a02db..a2331e7fe 100644 --- a/static/directives/repository-events-table.html +++ b/static/directives/repository-events-table.html @@ -15,10 +15,10 @@
No notifications have been setup for this repository.
-
Certificate: