From 5845e37e321915f63e84e2a46d93080413cb83ca Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 21 May 2015 15:22:59 -0400 Subject: [PATCH] Add Swift storage library --- requirements-nover.txt | 3 +- requirements.txt | 81 +++++--- static/css/core-ui.css | 23 +++ .../directives/config/config-map-field.html | 20 ++ .../directives/config/config-setup-tool.html | 8 +- .../config/config-string-field.html | 2 +- static/js/core-config-setup.js | 52 ++++- storage/__init__.py | 2 + storage/swift.py | 183 ++++++++++++++++++ 9 files changed, 341 insertions(+), 33 deletions(-) create mode 100644 static/directives/config/config-map-field.html create mode 100644 storage/swift.py diff --git a/requirements-nover.txt b/requirements-nover.txt index d76e671b5..20df379e2 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -21,7 +21,6 @@ paramiko xhtml2pdf redis hiredis -docker-py flask-restful==0.2.12 jsonschema git+https://github.com/NateFerrero/oauth2lib.git @@ -52,3 +51,5 @@ mock psutil stringscore mockldap +python-swiftclient +python-keystoneclient \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 08eddb442..2a950f9dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,68 +1,91 @@ -APScheduler==3.0.1 +APScheduler==3.0.3 +Babel==1.3 Flask==0.10.1 Flask-Login==0.2.11 Flask-Mail==0.9.1 Flask-Principal==0.4.0 Flask-RESTful==0.2.12 Jinja2==2.7.3 -LogentriesLogger==0.2.1 -Mako==1.0.0 +Logentries==0.7 +Mako==1.0.1 MarkupSafe==0.23 -Pillow==2.7.0 -PyMySQL==0.6.3 +Pillow==2.8.1 +PyMySQL==0.6.6 PyPDF2==1.24 PyYAML==3.11 -SQLAlchemy==0.9.8 -WebOb==1.4 -Werkzeug==0.9.6 -aiowsgi==0.3 -alembic==0.7.4 +SQLAlchemy==1.0.3 +WebOb==1.4.1 +Werkzeug==0.10.4 +aiowsgi==0.5 +alembic==0.7.5.post2 +argparse==1.3.0 autobahn==0.9.3-3 backports.ssl-match-hostname==3.4.0.2 beautifulsoup4==4.3.2 blinker==1.3 -boto==2.35.1 +boto==2.38.0 cachetools==1.0.0 -docker-py==0.7.1 -ecdsa==0.11 +certifi==2015.04.28 +cffi==0.9.2 +cryptography==0.8.2 +ecdsa==0.13 +enum34==1.0.4 +funcparserlib==0.3.6 futures==2.2.0 gevent==1.0.1 gipc==0.5.0 greenlet==0.4.5 gunicorn==18.0 -hiredis==0.1.5 -html5lib==0.999 +hiredis==0.2.0 +html5lib==0.99999 +iso8601==0.1.10 itsdangerous==0.24 jsonschema==2.4.0 -marisa-trie==0.7 -mixpanel-py==3.2.1 +marisa-trie==0.7.2 +mixpanel-py==4.0.2 mock==1.0.1 mockldap==0.2.4 +msgpack-python==0.4.6 +netaddr==0.7.14 +netifaces==0.10.4 +oauthlib==0.7.2 +oslo.config==1.11.0 +oslo.i18n==1.6.0 +oslo.serialization==1.5.0 +oslo.utils==1.5.0 paramiko==1.15.2 -peewee==2.4.7 +pbr==0.11.0 +peewee==2.6.0 +prettytable==0.7.2 psutil==2.2.1 -psycopg2==2.5.4 +psycopg2==2.6 py-bcrypt==0.4 +pyOpenSSL==0.15.1 +pyasn1==0.1.7 +pycparser==2.12 pycrypto==2.6.1 -python-dateutil==2.4.0 +pygpgme==0.3 +python-dateutil==2.4.2 +python-keystoneclient==1.4.0 python-ldap==2.4.19 python-magic==0.4.6 -pygpgme==0.3 -pytz==2014.10 -pyOpenSSL==0.14 -raven==5.1.1 +python-swiftclient==2.4.0 +pytz==2015.2 +raven==5.3.0 redis==2.10.3 reportlab==2.7 -requests==2.5.1 +requests==2.6.2 requests-oauthlib==0.4.2 +simplejson==3.7.1 six==1.9.0 +stevedore==1.4.0 stringscore==0.1.0 -stripe==1.20.1 +stripe==1.22.2 trollius==1.0.4 -tzlocal==1.1.2 -urllib3==1.10.2 +tzlocal==1.1.3 +urllib3==1.10.3 waitress==0.8.9 -websocket-client==0.23.0 +websocket-client==0.30.0 wsgiref==0.1.2 xhtml2pdf==0.0.6 git+https://github.com/DevTable/aniso8601-fake.git diff --git a/static/css/core-ui.css b/static/css/core-ui.css index 58ab72b81..adf6bc084 100644 --- a/static/css/core-ui.css +++ b/static/css/core-ui.css @@ -388,6 +388,29 @@ a:focus { width: 400px; } +.config-map-field-element table { + margin-bottom: 10px; +} + +.config-map-field-element .form-control-container { + border-top: 1px solid #eee; + padding-top: 10px; +} + +.config-map-field-element .form-control-container select, .config-map-field-element .form-control-container input { + margin-bottom: 10px; +} + +.config-map-field-element .empty { + color: #ccc; + margin-bottom: 10px; + display: block; +} + +.config-map-field-element .item-title { + font-weight: bold; +} + .config-contact-field { margin-bottom: 4px; } diff --git a/static/directives/config/config-map-field.html b/static/directives/config/config-map-field.html new file mode 100644 index 000000000..7089e2010 --- /dev/null +++ b/static/directives/config/config-map-field.html @@ -0,0 +1,20 @@ +
+ + + + + + +
{{ key }}{{ value }} + Remove +
+ No entries defined +
+ Add Key-Value: + + + +
+
diff --git a/static/directives/config/config-setup-tool.html b/static/directives/config/config-setup-tool.html index 7c2fa2e19..a3a344286 100644 --- a/static/directives/config/config-setup-tool.html +++ b/static/directives/config/config-setup-tool.html @@ -184,6 +184,7 @@ + @@ -192,10 +193,15 @@ {{ field.title }}: + + ng-if="field.kind == 'text'" + is-optional="field.optional">
diff --git a/static/directives/config/config-string-field.html b/static/directives/config/config-string-field.html index 7714fd541..703891f89 100644 --- a/static/directives/config/config-string-field.html +++ b/static/directives/config/config-string-field.html @@ -2,7 +2,7 @@
+ ng-pattern="getRegexp(pattern)" ng-required="!isOptional">
{{ errorMessage }}
diff --git a/static/js/core-config-setup.js b/static/js/core-config-setup.js index 033bbfbb6..93c07c7a6 100644 --- a/static/js/core-config-setup.js +++ b/static/js/core-config-setup.js @@ -78,6 +78,19 @@ angular.module("core-config-setup", ['angularFileUpload']) {'name': 'secret_key', 'title': 'Secret Key', 'placeholder': 'secretkeyhere', 'kind': 'text'}, {'name': 'bucket_name', 'title': 'Bucket Name', 'placeholder': 'my-cool-bucket', 'kind': 'text'}, {'name': 'storage_path', 'title': 'Storage Directory', 'placeholder': '/path/inside/bucket', 'kind': 'text'} + ], + + 'SwiftStorage': [ + {'name': 'auth_url', 'title': 'Swift Auth URL', 'placeholder': '', 'kind': 'text'}, + {'name': 'swift_container', 'title': 'Swift Container Name', 'placeholder': 'mycontainer', 'kind': 'text'}, + {'name': 'storage_path', 'title': 'Storage Path', 'placeholder': '/path/inside/container', 'kind': 'text'}, + + {'name': 'swift_user', 'title': 'Username', 'placeholder': 'accesskeyhere', 'kind': 'text'}, + {'name': 'swift_password', 'title': 'Password/Key', 'placeholder': 'secretkeyhere', 'kind': 'text'}, + + {'name': 'ca_cert_path', 'title': 'CA Cert Filename', 'placeholder': 'conf/stack/swift.cert', 'kind': 'text', 'optional': true}, + {'name': 'os_options', 'title': 'OS Options', 'kind': 'map', + 'keys': ['tenant_id', 'auth_token', 'service_type', 'endpoint_type', 'tenant_name', 'object_storage_url', 'region_name']} ] }; @@ -760,6 +773,42 @@ angular.module("core-config-setup", ['angularFileUpload']) return directiveDefinitionObject; }) + .directive('configMapField', function () { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/config/config-map-field.html', + replace: false, + transclude: false, + restrict: 'C', + scope: { + 'binding': '=binding', + 'keys': '=keys' + }, + controller: function($scope, $element) { + $scope.newKey = null; + $scope.newValue = null; + + $scope.hasValues = function(binding) { + return binding && Object.keys(binding).length; + }; + + $scope.removeKey = function(key) { + delete $scope.binding[key]; + }; + + $scope.addEntry = function() { + if (!$scope.newKey || !$scope.newValue) { return; } + + $scope.binding = $scope.binding || {}; + $scope.binding[$scope.newKey] = $scope.newValue; + $scope.newKey = null; + $scope.newValue = null; + } + } + }; + return directiveDefinitionObject; + }) + .directive('configStringField', function () { var directiveDefinitionObject = { priority: 0, @@ -772,7 +821,8 @@ angular.module("core-config-setup", ['angularFileUpload']) 'placeholder': '@placeholder', 'pattern': '@pattern', 'defaultValue': '@defaultValue', - 'validator': '&validator' + 'validator': '&validator', + 'isOptional': '=isOptional' }, controller: function($scope, $element) { $scope.getRegexp = function(pattern) { diff --git a/storage/__init__.py b/storage/__init__.py index 7893343c2..69f26def4 100644 --- a/storage/__init__.py +++ b/storage/__init__.py @@ -2,6 +2,7 @@ from storage.local import LocalStorage from storage.cloud import S3Storage, GoogleCloudStorage, RadosGWStorage from storage.fakestorage import FakeStorage from storage.distributedstorage import DistributedStorage +from storage.swift import SwiftStorage STORAGE_DRIVER_CLASSES = { @@ -9,6 +10,7 @@ STORAGE_DRIVER_CLASSES = { 'S3Storage': S3Storage, 'GoogleCloudStorage': GoogleCloudStorage, 'RadosGWStorage': RadosGWStorage, + 'SwiftStorage': SwiftStorage, } def get_storage_driver(storage_params): diff --git a/storage/swift.py b/storage/swift.py new file mode 100644 index 000000000..243202223 --- /dev/null +++ b/storage/swift.py @@ -0,0 +1,183 @@ +""" Swift storage driver. Based on: github.com/bacongobbler/docker-registry-driver-swift/ """ +from swiftclient.client import Connection, ClientException +from storage.basestorage import BaseStorage + +from random import SystemRandom +import string +import logging + +logger = logging.getLogger(__name__) + +class SwiftStorage(BaseStorage): + def __init__(self, swift_container, storage_path, auth_url, swift_user, + swift_password, auth_version=None, os_options=None, ca_cert_path=None): + self._swift_container = swift_container + self._storage_path = storage_path + + self._auth_url = auth_url + self._ca_cert_path = ca_cert_path + + self._swift_user = swift_user + self._swift_password = swift_password + + self._auth_version = auth_version or 2 + self._os_options = os_options or {} + + self._initialized = False + self._swift_connection = None + + def _initialize(self): + if self._initialized: + return + + self._initialized = True + self._swift_connection = self._get_connection() + + def _get_connection(self): + return Connection( + authurl=self._auth_url, + cacert=self._ca_cert_path, + + user=self._swift_user, + key=self._swift_password, + + auth_version=self._auth_version, + os_options=self._os_options) + + def _get_relative_path(self, path): + if path.startswith(self._storage_path): + path = path[len(self._storage_path)] + + if path.endswith('/'): + path = path[:-1] + + return path + + def _normalize_path(self, path=None): + path = self._storage_path + (path or '') + + # Openstack does not like paths starting with '/' and we always normalize + # to remove trailing '/' + if path.startswith('/'): + path = path[1:] + + if path.endswith('/'): + path = path[:-1] + + return path + + def _get_container(self, path): + self._initialize() + path = self._normalize_path(path) + + if path and not path.endswith('/'): + path += '/' + + try: + _, container = self._swift_connection.get_container( + container=self._swift_container, + prefix=path, delimiter='/') + return container + except: + logger.exception('Could not get container: %s', path) + raise IOError('Unknown path: %s' % path) + + def _get_object(self, path, chunk_size=None): + self._initialize() + path = self._normalize_path(path) + try: + _, obj = self._swift_connection.get_object(self._swift_container, path, + resp_chunk_size=chunk_size) + return obj + except Exception: + logger.exception('Could not get object: %s', path) + raise IOError('Path %s not found' % path) + + def _put_object(self, path, content, chunk=None, content_type=None, content_encoding=None): + self._initialize() + path = self._normalize_path(path) + headers = {} + + if content_encoding is not None: + headers['Content-Encoding'] = content_encoding + + try: + self._swift_connection.put_object(self._swift_container, path, content, + chunk_size=chunk, content_type=content_type, + headers=headers) + except ClientException: + raise + except Exception: + logger.exception('Could not put object: %s', path) + raise IOError("Could not put content: %s" % path) + + def _head_object(self, path): + self._initialize() + path = self._normalize_path(path) + try: + return self._swift_connection.head_object(self._swift_container, path) + except Exception: + logger.exception('Could not head object: %s', path) + return None + + def get_direct_download_url(self, path, expires_in=60, requires_cors=False): + if requires_cors: + return None + + # TODO: http://docs.openstack.org/juno/config-reference/content/object-storage-tempurl.html + return None + + def get_content(self, path): + return self._get_object(path) + + def put_content(self, path, content): + self._put_object(path, content) + + def stream_read(self, path): + for data in self._get_object(path, self.buffer_size): + yield data + + def stream_read_file(self, path): + raise NotImplementedError + + def stream_write(self, path, fp, content_type=None, content_encoding=None): + self._put_object(path, fp, self.buffer_size, content_type=content_type, + content_encoding=content_encoding) + + def list_directory(self, path=None): + container = self._get_container(path) + if not container: + raise OSError('Unknown path: %s' % path) + + for entry in container: + param = None + if 'name' in entry: + param = 'name' + elif 'subdir' in entry: + param = 'subdir' + else: + continue + + yield self._get_relative_path(entry[param]) + + def exists(self, path): + return bool(self._head_object(path)) + + def remove(self, path): + self._initialize() + path = self._normalize_path(path) + try: + self._swift_connection.delete_object(self._swift_container, path) + except Exception: + raise IOError('Cannot delete path: %s' % path) + + def _random_checksum(self, count): + chars = string.ascii_uppercase + string.digits + return ''.join(SystemRandom().choice(chars) for _ in range(count)) + + def get_checksum(self, path): + headers = self._head_object(path) + if not headers: + raise IOError('Cannot lookup path: %s' % path) + + return headers.get('etag', '')[1:-1][:7] or self._random_checksum(7)