vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add Swift storage library

Browse source
This commit is contained in:
Joseph Schorr 2015-05-21 15:22:59 -04:00
parent 417c77f4d9
commit 5845e37e32
9 changed files with 341 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
requirements-nover.txt
View file

@ -21,7 +21,6 @@ paramiko
xhtml2pdf
redis
hiredis
docker-py
flask-restful==0.2.12
jsonschema
git+https://github.com/NateFerrero/oauth2lib.git
@ -52,3 +51,5 @@ mock
psutil
stringscore
mockldap
python-swiftclient
python-keystoneclient

81
requirements.txt
View file

@ -1,68 +1,91 @@
APScheduler==3.0.1
APScheduler==3.0.3
Babel==1.3
Flask==0.10.1
Flask-Login==0.2.11
Flask-Mail==0.9.1
Flask-Principal==0.4.0
Flask-RESTful==0.2.12
Jinja2==2.7.3
LogentriesLogger==0.2.1
Mako==1.0.0
Logentries==0.7
Mako==1.0.1
MarkupSafe==0.23
Pillow==2.7.0
PyMySQL==0.6.3
Pillow==2.8.1
PyMySQL==0.6.6
PyPDF2==1.24
PyYAML==3.11
SQLAlchemy==0.9.8
WebOb==1.4
Werkzeug==0.9.6
aiowsgi==0.3
alembic==0.7.4
SQLAlchemy==1.0.3
WebOb==1.4.1
Werkzeug==0.10.4
aiowsgi==0.5
alembic==0.7.5.post2
argparse==1.3.0
autobahn==0.9.3-3
backports.ssl-match-hostname==3.4.0.2
beautifulsoup4==4.3.2
blinker==1.3
boto==2.35.1
boto==2.38.0
cachetools==1.0.0
docker-py==0.7.1
ecdsa==0.11
certifi==2015.04.28
cffi==0.9.2
cryptography==0.8.2
ecdsa==0.13
enum34==1.0.4
funcparserlib==0.3.6
futures==2.2.0
gevent==1.0.1
gipc==0.5.0
greenlet==0.4.5
gunicorn==18.0
hiredis==0.1.5
html5lib==0.999
hiredis==0.2.0
html5lib==0.99999
iso8601==0.1.10
itsdangerous==0.24
jsonschema==2.4.0
marisa-trie==0.7
mixpanel-py==3.2.1
marisa-trie==0.7.2
mixpanel-py==4.0.2
mock==1.0.1
mockldap==0.2.4
msgpack-python==0.4.6
netaddr==0.7.14
netifaces==0.10.4
oauthlib==0.7.2
oslo.config==1.11.0
oslo.i18n==1.6.0
oslo.serialization==1.5.0
oslo.utils==1.5.0
paramiko==1.15.2
peewee==2.4.7
pbr==0.11.0
peewee==2.6.0
prettytable==0.7.2
psutil==2.2.1
psycopg2==2.5.4
psycopg2==2.6
py-bcrypt==0.4
pyOpenSSL==0.15.1
pyasn1==0.1.7
pycparser==2.12
pycrypto==2.6.1
python-dateutil==2.4.0
pygpgme==0.3
python-dateutil==2.4.2
python-keystoneclient==1.4.0
python-ldap==2.4.19
python-magic==0.4.6
pygpgme==0.3
pytz==2014.10
pyOpenSSL==0.14
raven==5.1.1
python-swiftclient==2.4.0
pytz==2015.2
raven==5.3.0
redis==2.10.3
reportlab==2.7
requests==2.5.1
requests==2.6.2
requests-oauthlib==0.4.2
simplejson==3.7.1
six==1.9.0
stevedore==1.4.0
stringscore==0.1.0
stripe==1.20.1
stripe==1.22.2
trollius==1.0.4
tzlocal==1.1.2
urllib3==1.10.2
tzlocal==1.1.3
urllib3==1.10.3
waitress==0.8.9
websocket-client==0.23.0
websocket-client==0.30.0
wsgiref==0.1.2
xhtml2pdf==0.0.6
git+https://github.com/DevTable/aniso8601-fake.git

23
static/css/core-ui.css
View file

@ -388,6 +388,29 @@ a:focus {
width: 400px;
}
.config-map-field-element table {
margin-bottom: 10px;
}
.config-map-field-element .form-control-container {
border-top: 1px solid #eee;
padding-top: 10px;
}
.config-map-field-element .form-control-container select, .config-map-field-element .form-control-container input {
margin-bottom: 10px;
}
.config-map-field-element .empty {
color: #ccc;
margin-bottom: 10px;
display: block;
}
.config-map-field-element .item-title {
font-weight: bold;
}
.config-contact-field {
margin-bottom: 4px;
}

20
static/directives/config/config-map-field.html Normal file
View file

@ -0,0 +1,20 @@
<div class="config-map-field-element">
<table class="table" ng-show="hasValues(binding)">
<tr class="item" ng-repeat="(key, value) in binding">
<td class="item-title">{{ key }}</td>
<td class="item-value">{{ value }}</td>
<td class="item-delete">
<a href="javascript:void(0)" ng-click="removeKey(key)">Remove</a>
</td>
</tr>
</table>
<span class="empty" ng-if="!hasValues(binding)">No entries defined</span>
<form class="form-control-container" ng-submit="addEntry()">
Add Key-Value:
<select ng-model="newKey">
<option ng-repeat="key in keys" value="{{ key }}">{{ key }}</option>
</select>
<input type="text" class="form-control" placeholder="Value" ng-model="newValue">
<button class="btn btn-default" style="display: inline-block">Add Entry</button>
</form>
</div>

8
static/directives/config/config-setup-tool.html
View file

@ -184,6 +184,7 @@
<option value="S3Storage">Amazon S3</option>
<option value="GoogleCloudStorage">Google Cloud Storage</option>
<option value="RadosGWStorage">Ceph Object Gateway (RADOS)</option>
<option value="SwiftStorage">OpenStack Storage (Swift)</option>
</select>
</td>
</tr>
@ -192,10 +193,15 @@
<tr ng-repeat="field in STORAGE_CONFIG_FIELDS[config.DISTRIBUTED_STORAGE_CONFIG.local[0]]">
<td>{{ field.title }}:</td>
<td>
<span class="config-map-field"
binding="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]"
ng-if="field.kind == 'map'"
keys="field.keys"></span>
<span class="config-string-field"
binding="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]"
placeholder="{{ field.placeholder }}"
ng-if="field.kind == 'text'"></span>
ng-if="field.kind == 'text'"
is-optional="field.optional"></span>
<div class="co-checkbox" ng-if="field.kind == 'bool'">
<input id="dsc-{{ field.name }}" type="checkbox"
ng-model="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]">

2
static/directives/config/config-string-field.html
View file

@ -2,7 +2,7 @@
<form name="fieldform" novalidate>
<input type="text" class="form-control" placeholder="{{ placeholder || '' }}"
ng-model="binding" ng-trim="false" ng-minlength="1"
ng-pattern="getRegexp(pattern)" required>
ng-pattern="getRegexp(pattern)" ng-required="!isOptional">
<div class="alert alert-danger" ng-show="errorMessage">
{{ errorMessage }}
</div>

52
static/js/core-config-setup.js
View file

@ -78,6 +78,19 @@ angular.module("core-config-setup", ['angularFileUpload'])
{'name': 'secret_key', 'title': 'Secret Key', 'placeholder': 'secretkeyhere', 'kind': 'text'},
{'name': 'bucket_name', 'title': 'Bucket Name', 'placeholder': 'my-cool-bucket', 'kind': 'text'},
{'name': 'storage_path', 'title': 'Storage Directory', 'placeholder': '/path/inside/bucket', 'kind': 'text'}
],
'SwiftStorage': [
{'name': 'auth_url', 'title': 'Swift Auth URL', 'placeholder': '', 'kind': 'text'},
{'name': 'swift_container', 'title': 'Swift Container Name', 'placeholder': 'mycontainer', 'kind': 'text'},
{'name': 'storage_path', 'title': 'Storage Path', 'placeholder': '/path/inside/container', 'kind': 'text'},
{'name': 'swift_user', 'title': 'Username', 'placeholder': 'accesskeyhere', 'kind': 'text'},
{'name': 'swift_password', 'title': 'Password/Key', 'placeholder': 'secretkeyhere', 'kind': 'text'},
{'name': 'ca_cert_path', 'title': 'CA Cert Filename', 'placeholder': 'conf/stack/swift.cert', 'kind': 'text', 'optional': true},
{'name': 'os_options', 'title': 'OS Options', 'kind': 'map',
'keys': ['tenant_id', 'auth_token', 'service_type', 'endpoint_type', 'tenant_name', 'object_storage_url', 'region_name']}
]
};
@ -760,6 +773,42 @@ angular.module("core-config-setup", ['angularFileUpload'])
return directiveDefinitionObject;
})
.directive('configMapField', function () {
var directiveDefinitionObject = {
priority: 0,
templateUrl: '/static/directives/config/config-map-field.html',
replace: false,
transclude: false,
restrict: 'C',
scope: {
'binding': '=binding',
'keys': '=keys'
},
controller: function($scope, $element) {
$scope.newKey = null;
$scope.newValue = null;
$scope.hasValues = function(binding) {
return binding && Object.keys(binding).length;
};
$scope.removeKey = function(key) {
delete $scope.binding[key];
};
$scope.addEntry = function() {
if (!$scope.newKey || !$scope.newValue) { return; }
$scope.binding = $scope.binding || {};
$scope.binding[$scope.newKey] = $scope.newValue;
$scope.newKey = null;
$scope.newValue = null;
}
}
};
return directiveDefinitionObject;
})
.directive('configStringField', function () {
var directiveDefinitionObject = {
priority: 0,
@ -772,7 +821,8 @@ angular.module("core-config-setup", ['angularFileUpload'])
'placeholder': '@placeholder',
'pattern': '@pattern',
'defaultValue': '@defaultValue',
'validator': '&validator'
'validator': '&validator',
'isOptional': '=isOptional'
},
controller: function($scope, $element) {
$scope.getRegexp = function(pattern) {

2
storage/__init__.py
View file

@ -2,6 +2,7 @@ from storage.local import LocalStorage
from storage.cloud import S3Storage, GoogleCloudStorage, RadosGWStorage
from storage.fakestorage import FakeStorage
from storage.distributedstorage import DistributedStorage
from storage.swift import SwiftStorage
STORAGE_DRIVER_CLASSES = {
@ -9,6 +10,7 @@ STORAGE_DRIVER_CLASSES = {
'S3Storage': S3Storage,
'GoogleCloudStorage': GoogleCloudStorage,
'RadosGWStorage': RadosGWStorage,
'SwiftStorage': SwiftStorage,
}
def get_storage_driver(storage_params):

183
storage/swift.py Normal file
View file

@ -0,0 +1,183 @@
""" Swift storage driver. Based on: github.com/bacongobbler/docker-registry-driver-swift/ """
from swiftclient.client import Connection, ClientException
from storage.basestorage import BaseStorage
from random import SystemRandom
import string
import logging
logger = logging.getLogger(__name__)
class SwiftStorage(BaseStorage):
def __init__(self, swift_container, storage_path, auth_url, swift_user,
swift_password, auth_version=None, os_options=None, ca_cert_path=None):
self._swift_container = swift_container
self._storage_path = storage_path
self._auth_url = auth_url
self._ca_cert_path = ca_cert_path
self._swift_user = swift_user
self._swift_password = swift_password
self._auth_version = auth_version or 2
self._os_options = os_options or {}
self._initialized = False
self._swift_connection = None
def _initialize(self):
if self._initialized:
return
self._initialized = True
self._swift_connection = self._get_connection()
def _get_connection(self):
return Connection(
authurl=self._auth_url,
cacert=self._ca_cert_path,
user=self._swift_user,
key=self._swift_password,
auth_version=self._auth_version,
os_options=self._os_options)
def _get_relative_path(self, path):
if path.startswith(self._storage_path):
path = path[len(self._storage_path)]
if path.endswith('/'):
path = path[:-1]
return path
def _normalize_path(self, path=None):
path = self._storage_path + (path or '')
# Openstack does not like paths starting with '/' and we always normalize
# to remove trailing '/'
if path.startswith('/'):
path = path[1:]
if path.endswith('/'):
path = path[:-1]
return path
def _get_container(self, path):
self._initialize()
path = self._normalize_path(path)
if path and not path.endswith('/'):
path += '/'
try:
_, container = self._swift_connection.get_container(
container=self._swift_container,
prefix=path, delimiter='/')
return container
except:
logger.exception('Could not get container: %s', path)
raise IOError('Unknown path: %s' % path)
def _get_object(self, path, chunk_size=None):
self._initialize()
path = self._normalize_path(path)
try:
_, obj = self._swift_connection.get_object(self._swift_container, path,
resp_chunk_size=chunk_size)
return obj
except Exception:
logger.exception('Could not get object: %s', path)
raise IOError('Path %s not found' % path)
def _put_object(self, path, content, chunk=None, content_type=None, content_encoding=None):
self._initialize()
path = self._normalize_path(path)
headers = {}
if content_encoding is not None:
headers['Content-Encoding'] = content_encoding
try:
self._swift_connection.put_object(self._swift_container, path, content,
chunk_size=chunk, content_type=content_type,
headers=headers)
except ClientException:
raise
except Exception:
logger.exception('Could not put object: %s', path)
raise IOError("Could not put content: %s" % path)
def _head_object(self, path):
self._initialize()
path = self._normalize_path(path)
try:
return self._swift_connection.head_object(self._swift_container, path)
except Exception:
logger.exception('Could not head object: %s', path)
return None
def get_direct_download_url(self, path, expires_in=60, requires_cors=False):
if requires_cors:
return None
# TODO: http://docs.openstack.org/juno/config-reference/content/object-storage-tempurl.html
return None
def get_content(self, path):
return self._get_object(path)
def put_content(self, path, content):
self._put_object(path, content)
def stream_read(self, path):
for data in self._get_object(path, self.buffer_size):
yield data
def stream_read_file(self, path):
raise NotImplementedError
def stream_write(self, path, fp, content_type=None, content_encoding=None):
self._put_object(path, fp, self.buffer_size, content_type=content_type,
content_encoding=content_encoding)
def list_directory(self, path=None):
container = self._get_container(path)
if not container:
raise OSError('Unknown path: %s' % path)
for entry in container:
param = None
if 'name' in entry:
param = 'name'
elif 'subdir' in entry:
param = 'subdir'
else:
continue
yield self._get_relative_path(entry[param])
def exists(self, path):
return bool(self._head_object(path))
def remove(self, path):
self._initialize()
path = self._normalize_path(path)
try:
self._swift_connection.delete_object(self._swift_container, path)
except Exception:
raise IOError('Cannot delete path: %s' % path)
def _random_checksum(self, count):
chars = string.ascii_uppercase + string.digits
return ''.join(SystemRandom().choice(chars) for _ in range(count))
def get_checksum(self, path):
headers = self._head_object(path)
if not headers:
raise IOError('Cannot lookup path: %s' % path)
return headers.get('etag', '')[1:-1][:7] or self._random_checksum(7)