Merge pull request #38 from coreos-inc/swift

Add Swift Storage

requirements-nover.txt

@@ -21,7 +21,6 @@ paramiko
 xhtml2pdf
 redis
 hiredis
-docker-py
 flask-restful==0.2.12
 jsonschema
 git+https://github.com/NateFerrero/oauth2lib.git
@@ -52,3 +51,5 @@ mock
 psutil
 stringscore
 mockldap
+python-swiftclient
+python-keystoneclient

requirements.txt

@@ -1,68 +1,91 @@
-APScheduler==3.0.1
+APScheduler==3.0.3
+Babel==1.3
 Flask==0.10.1
 Flask-Login==0.2.11
 Flask-Mail==0.9.1
 Flask-Principal==0.4.0
 Flask-RESTful==0.2.12
 Jinja2==2.7.3
-LogentriesLogger==0.2.1
-Mako==1.0.0
+Logentries==0.7
+Mako==1.0.1
 MarkupSafe==0.23
-Pillow==2.7.0
-PyMySQL==0.6.3
+Pillow==2.8.1
+PyMySQL==0.6.6
 PyPDF2==1.24
 PyYAML==3.11
-SQLAlchemy==0.9.8
-WebOb==1.4
-Werkzeug==0.9.6
-aiowsgi==0.3
-alembic==0.7.4
+SQLAlchemy==1.0.3
+WebOb==1.4.1
+Werkzeug==0.10.4
+aiowsgi==0.5
+alembic==0.7.5.post2
+argparse==1.3.0
 autobahn==0.9.3-3
 backports.ssl-match-hostname==3.4.0.2
 beautifulsoup4==4.3.2
 blinker==1.3
-boto==2.35.1
+boto==2.38.0
 cachetools==1.0.0
-docker-py==0.7.1
-ecdsa==0.11
+certifi==2015.04.28
+cffi==0.9.2
+cryptography==0.8.2
+ecdsa==0.13
+enum34==1.0.4
+funcparserlib==0.3.6
 futures==2.2.0
 gevent==1.0.1
 gipc==0.5.0
 greenlet==0.4.5
 gunicorn==18.0
-hiredis==0.1.5
-html5lib==0.999
+hiredis==0.2.0
+html5lib==0.99999
+iso8601==0.1.10
 itsdangerous==0.24
 jsonschema==2.4.0
-marisa-trie==0.7
-mixpanel-py==3.2.1
+marisa-trie==0.7.2
+mixpanel-py==4.0.2
 mock==1.0.1
 mockldap==0.2.4
+msgpack-python==0.4.6
+netaddr==0.7.14
+netifaces==0.10.4
+oauthlib==0.7.2
+oslo.config==1.11.0
+oslo.i18n==1.6.0
+oslo.serialization==1.5.0
+oslo.utils==1.5.0
 paramiko==1.15.2
-peewee==2.4.7
+pbr==0.11.0
+peewee==2.6.0
+prettytable==0.7.2
 psutil==2.2.1
-psycopg2==2.5.4
+psycopg2==2.6
 py-bcrypt==0.4
+pyOpenSSL==0.15.1
+pyasn1==0.1.7
+pycparser==2.12
 pycrypto==2.6.1
-python-dateutil==2.4.0
+pygpgme==0.3
+python-dateutil==2.4.2
+python-keystoneclient==1.4.0
 python-ldap==2.4.19
 python-magic==0.4.6
-pygpgme==0.3
-pytz==2014.10
-pyOpenSSL==0.14
-raven==5.1.1
+python-swiftclient==2.4.0
+pytz==2015.2
+raven==5.3.0
 redis==2.10.3
 reportlab==2.7
-requests==2.5.1
+requests==2.6.2
 requests-oauthlib==0.4.2
+simplejson==3.7.1
 six==1.9.0
+stevedore==1.4.0
 stringscore==0.1.0
-stripe==1.20.1
+stripe==1.22.2
 trollius==1.0.4
-tzlocal==1.1.2
-urllib3==1.10.2
+tzlocal==1.1.3
+urllib3==1.10.3
 waitress==0.8.9
-websocket-client==0.23.0
+websocket-client==0.30.0
 wsgiref==0.1.2
 xhtml2pdf==0.0.6
 git+https://github.com/DevTable/aniso8601-fake.git

static/css/core-ui.css

@@ -388,6 +388,29 @@ a:focus {
   width: 400px;
 }
 
+.config-map-field-element table {
+  margin-bottom: 10px;
+}
+
+.config-map-field-element .form-control-container {
+  border-top: 1px solid #eee;
+  padding-top: 10px;
+}
+
+.config-map-field-element .form-control-container select, .config-map-field-element .form-control-container input {
+  margin-bottom: 10px;
+}
+
+.config-map-field-element .empty {
+  color: #ccc;
+  margin-bottom: 10px;
+  display: block;
+}
+
+.config-map-field-element .item-title {
+  font-weight: bold;
+}
+
 .config-contact-field {
   margin-bottom: 4px;
 }

static/directives/config/config-map-field.html

@@ -0,0 +1,20 @@
+<div class="config-map-field-element">
+  <table class="table" ng-show="hasValues(binding)">
+    <tr class="item" ng-repeat="(key, value) in binding">
+      <td class="item-title">{{ key }}</td>
+      <td class="item-value">{{ value }}</td>
+      <td class="item-delete">
+        <a href="javascript:void(0)" ng-click="removeKey(key)">Remove</a>
+      </td>
+    </tr>
+  </table>
+  <span class="empty" ng-if="!hasValues(binding)">No entries defined</span>
+  <form class="form-control-container" ng-submit="addEntry()">
+    Add Key-Value:
+    <select ng-model="newKey">
+      <option ng-repeat="key in keys" value="{{ key }}">{{ key }}</option>
+    </select>
+    <input type="text" class="form-control" placeholder="Value" ng-model="newValue">
+    <button class="btn btn-default" style="display: inline-block">Add Entry</button>
+  </form>
+</div>

static/directives/config/config-setup-tool.html

@@ -189,6 +189,7 @@
                   <option value="S3Storage">Amazon S3</option>
                   <option value="GoogleCloudStorage">Google Cloud Storage</option>
                   <option value="RadosGWStorage">Ceph Object Gateway (RADOS)</option>
+                  <option value="SwiftStorage">OpenStack Storage (Swift)</option>
                 </select>
               </td>
             </tr>
@@ -197,10 +198,15 @@
             <tr ng-repeat="field in STORAGE_CONFIG_FIELDS[config.DISTRIBUTED_STORAGE_CONFIG.local[0]]">
               <td>{{ field.title }}:</td>
               <td>
+                <span class="config-map-field"
+                      binding="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]"
+                      ng-if="field.kind == 'map'"
+                      keys="field.keys"></span>
                 <span class="config-string-field"
                       binding="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]"
                       placeholder="{{ field.placeholder }}"
-                      ng-if="field.kind == 'text'"></span>
+                      ng-if="field.kind == 'text'"
+                      is-optional="field.optional"></span>
                 <div class="co-checkbox" ng-if="field.kind == 'bool'">
                   <input id="dsc-{{ field.name }}" type="checkbox"
                          ng-model="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]">

static/directives/config/config-string-field.html

@@ -2,7 +2,7 @@
   <form name="fieldform" novalidate>
     <input type="text" class="form-control" placeholder="{{ placeholder || '' }}"
            ng-model="binding" ng-trim="false" ng-minlength="1"
-           ng-pattern="getRegexp(pattern)" required>
+           ng-pattern="getRegexp(pattern)" ng-required="!isOptional">
     <div class="alert alert-danger" ng-show="errorMessage">
         {{ errorMessage }}
     </div>

static/js/core-config-setup.js

@@ -78,6 +78,19 @@ angular.module("core-config-setup", ['angularFileUpload'])
             {'name': 'secret_key', 'title': 'Secret Key', 'placeholder': 'secretkeyhere', 'kind': 'text'},
             {'name': 'bucket_name', 'title': 'Bucket Name',  'placeholder': 'my-cool-bucket', 'kind': 'text'},
             {'name': 'storage_path', 'title': 'Storage Directory', 'placeholder': '/path/inside/bucket', 'kind': 'text'}
+          ],
+
+          'SwiftStorage': [
+            {'name': 'auth_url', 'title': 'Swift Auth URL', 'placeholder': '', 'kind': 'text'},
+            {'name': 'swift_container', 'title': 'Swift Container Name', 'placeholder': 'mycontainer', 'kind': 'text'},
+            {'name': 'storage_path', 'title': 'Storage Path', 'placeholder': '/path/inside/container', 'kind': 'text'},
+
+            {'name': 'swift_user', 'title': 'Username', 'placeholder': 'accesskeyhere', 'kind': 'text'},
+            {'name': 'swift_password', 'title': 'Password/Key', 'placeholder': 'secretkeyhere', 'kind': 'text'},
+
+            {'name': 'ca_cert_path', 'title': 'CA Cert Filename', 'placeholder': 'conf/stack/swift.cert', 'kind': 'text', 'optional': true},
+            {'name': 'os_options', 'title': 'OS Options', 'kind': 'map',
+             'keys': ['tenant_id', 'auth_token', 'service_type', 'endpoint_type', 'tenant_name', 'object_storage_url', 'region_name']}
           ]
         };
 
@@ -760,6 +773,42 @@ angular.module("core-config-setup", ['angularFileUpload'])
     return directiveDefinitionObject;
   })
 
+ .directive('configMapField', function () {
+    var directiveDefinitionObject = {
+      priority: 0,
+      templateUrl: '/static/directives/config/config-map-field.html',
+      replace: false,
+      transclude: false,
+      restrict: 'C',
+      scope: {
+        'binding': '=binding',
+        'keys': '=keys'
+      },
+      controller: function($scope, $element) {
+        $scope.newKey = null;
+        $scope.newValue = null;
+
+        $scope.hasValues = function(binding) {
+          return binding && Object.keys(binding).length;
+        };
+
+        $scope.removeKey = function(key) {
+          delete $scope.binding[key];
+        };
+
+        $scope.addEntry = function() {
+          if (!$scope.newKey || !$scope.newValue) { return; }
+
+          $scope.binding = $scope.binding || {};
+          $scope.binding[$scope.newKey] = $scope.newValue;
+          $scope.newKey = null;
+          $scope.newValue = null;
+        }
+      }
+    };
+    return directiveDefinitionObject;
+  })
+
   .directive('configStringField', function () {
     var directiveDefinitionObject = {
       priority: 0,
@@ -772,7 +821,8 @@ angular.module("core-config-setup", ['angularFileUpload'])
         'placeholder': '@placeholder',
         'pattern': '@pattern',
         'defaultValue': '@defaultValue',
-        'validator': '&validator'
+        'validator': '&validator',
+        'isOptional': '=isOptional'
       },
       controller: function($scope, $element) {
         $scope.getRegexp = function(pattern) {

storage/__init__.py

@@ -2,6 +2,7 @@ from storage.local import LocalStorage
 from storage.cloud import S3Storage, GoogleCloudStorage, RadosGWStorage
 from storage.fakestorage import FakeStorage
 from storage.distributedstorage import DistributedStorage
+from storage.swift import SwiftStorage
 
 
 STORAGE_DRIVER_CLASSES = {
@@ -9,6 +10,7 @@ STORAGE_DRIVER_CLASSES = {
   'S3Storage': S3Storage,
   'GoogleCloudStorage': GoogleCloudStorage,
   'RadosGWStorage': RadosGWStorage,
+  'SwiftStorage': SwiftStorage,
 }
 
 def get_storage_driver(storage_params):

storage/swift.py

@@ -0,0 +1,188 @@
+""" Swift storage driver. Based on: github.com/bacongobbler/docker-registry-driver-swift/ """
+from swiftclient.client import Connection, ClientException
+from storage.basestorage import BaseStorage
+
+from random import SystemRandom
+import string
+import logging
+
+logger = logging.getLogger(__name__)
+
+class SwiftStorage(BaseStorage):
+  def __init__(self, swift_container, storage_path, auth_url, swift_user,
+               swift_password, auth_version=None, os_options=None, ca_cert_path=None):
+    self._swift_container = swift_container
+    self._storage_path = storage_path
+
+    self._auth_url = auth_url
+    self._ca_cert_path = ca_cert_path
+
+    self._swift_user = swift_user
+    self._swift_password = swift_password
+
+    self._auth_version = auth_version or 2
+    self._os_options = os_options or {}
+
+    self._initialized = False
+    self._swift_connection = None
+
+  def _initialize(self):
+    if self._initialized:
+      return
+
+    self._initialized = True
+    self._swift_connection = self._get_connection()
+
+  def _get_connection(self):
+    return Connection(
+        authurl=self._auth_url,
+        cacert=self._ca_cert_path,
+
+        user=self._swift_user,
+        key=self._swift_password,
+
+        auth_version=self._auth_version,
+        os_options=self._os_options)
+
+  def _get_relative_path(self, path):
+    if path.startswith(self._storage_path):
+      path = path[len(self._storage_path)]
+
+    if path.endswith('/'):
+      path = path[:-1]
+
+    return path
+
+  def _normalize_path(self, path=None):
+    path = self._storage_path + (path or '')
+
+    # Openstack does not like paths starting with '/' and we always normalize
+    # to remove trailing '/'
+    if path.startswith('/'):
+      path = path[1:]
+
+    if path.endswith('/'):
+      path = path[:-1]
+
+    return path
+
+  def _get_container(self, path):
+    self._initialize()
+    path = self._normalize_path(path)
+
+    if path and not path.endswith('/'):
+        path += '/'
+
+    try:
+       _, container = self._swift_connection.get_container(
+            container=self._swift_container,
+            prefix=path, delimiter='/')
+       return container
+    except:
+      logger.exception('Could not get container: %s', path)
+      raise IOError('Unknown path: %s' % path)
+
+  def _get_object(self, path, chunk_size=None):
+    self._initialize()
+    path = self._normalize_path(path)
+    try:
+      _, obj = self._swift_connection.get_object(self._swift_container, path,
+          resp_chunk_size=chunk_size)
+      return obj
+    except Exception:
+      logger.exception('Could not get object: %s', path)
+      raise IOError('Path %s not found' % path)
+
+  def _put_object(self, path, content, chunk=None, content_type=None, content_encoding=None):
+    self._initialize()
+    path = self._normalize_path(path)
+    headers = {}
+
+    if content_encoding is not None:
+      headers['Content-Encoding'] = content_encoding
+
+    try:
+      self._swift_connection.put_object(self._swift_container, path, content,
+                                        chunk_size=chunk, content_type=content_type,
+                                        headers=headers)
+    except ClientException:
+      # We re-raise client exception here so that validation of config during setup can see
+      # the client exception messages.
+      raise
+    except Exception:
+      logger.exception('Could not put object: %s', path)
+      raise IOError("Could not put content: %s" % path)
+
+  def _head_object(self, path):
+    self._initialize()
+    path = self._normalize_path(path)
+    try:
+      return self._swift_connection.head_object(self._swift_container, path)
+    except Exception:
+      logger.exception('Could not head object: %s', path)
+      return None
+
+  def get_direct_download_url(self, path, expires_in=60, requires_cors=False):
+    if requires_cors:
+      return None
+
+    # TODO(jschorr): This method is not strictly necessary but would result in faster operations
+    # when using this storage engine. However, the implementation (as seen in the link below)
+    # is not clean, so we punt on this for now.
+    # http://docs.openstack.org/juno/config-reference/content/object-storage-tempurl.html
+    return None
+
+  def get_content(self, path):
+    return self._get_object(path)
+
+  def put_content(self, path, content):
+    self._put_object(path, content)
+
+  def stream_read(self, path):
+    for data in self._get_object(path, self.buffer_size):
+      yield data
+
+  def stream_read_file(self, path):
+    raise NotImplementedError
+
+  def stream_write(self, path, fp, content_type=None, content_encoding=None):
+    self._put_object(path, fp, self.buffer_size, content_type=content_type,
+                     content_encoding=content_encoding)
+
+  def list_directory(self, path=None):
+    container = self._get_container(path)
+    if not container:
+      raise OSError('Unknown path: %s' % path)
+
+    for entry in container:
+      param = None
+      if 'name' in entry:
+        param = 'name'
+      elif 'subdir' in entry:
+        param = 'subdir'
+      else:
+        continue
+
+      yield self._get_relative_path(entry[param])
+
+  def exists(self, path):
+    return bool(self._head_object(path))
+
+  def remove(self, path):
+    self._initialize()
+    path = self._normalize_path(path)
+    try:
+      self._swift_connection.delete_object(self._swift_container, path)
+    except Exception:
+      raise IOError('Cannot delete path: %s' % path)
+
+  def _random_checksum(self, count):
+    chars = string.ascii_uppercase + string.digits
+    return ''.join(SystemRandom().choice(chars) for _ in range(count))
+
+  def get_checksum(self, path):
+    headers = self._head_object(path)
+    if not headers:
+      raise IOError('Cannot lookup path: %s' % path)
+
+    return headers.get('etag', '')[1:-1][:7] or self._random_checksum(7)