Fix paths in swift driver to be more intelligible
This commit is contained in:
Jake Moshenko
parent
0b1951a4a4
commit
c6d7eba98d
1 changed files with 38 additions and 43 deletions
|
|
@ -2,22 +2,25 @@
|
|||
|
||||
Uses: http://docs.openstack.org/developer/swift/overview_large_objects.html
|
||||
"""
|
||||
from swiftclient.client import Connection, ClientException
|
||||
from storage.basestorage import BaseStorage
|
||||
from util.registry.generatorfile import GeneratorFile
|
||||
from urlparse import urlparse
|
||||
from random import SystemRandom
|
||||
from hashlib import sha1
|
||||
from time import time
|
||||
from collections import namedtuple
|
||||
from util.registry import filelike
|
||||
|
||||
import os.path
|
||||
import copy
|
||||
import hmac
|
||||
import string
|
||||
import logging
|
||||
from uuid import uuid4
|
||||
|
||||
from swiftclient.client import Connection, ClientException
|
||||
from urlparse import urlparse
|
||||
from random import SystemRandom
|
||||
from hashlib import sha1
|
||||
from time import time
|
||||
from collections import namedtuple
|
||||
|
||||
from util.registry import filelike
|
||||
from storage.basestorage import BaseStorage
|
||||
from util.registry.generatorfile import GeneratorFile
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_PartUploadMetadata = namedtuple('_PartUploadMetadata', ['path', 'offset', 'length'])
|
||||
|
|
@ -28,10 +31,12 @@ _MAXIMUM_SEGMENT_SIZE = 5000000000 # 5 GB
|
|||
class SwiftStorage(BaseStorage):
|
||||
def __init__(self, swift_container, storage_path, auth_url, swift_user,
|
||||
swift_password, auth_version=None, os_options=None, ca_cert_path=None,
|
||||
temp_url_key=None):
|
||||
temp_url_key=None, simple_path_concat=False):
|
||||
super(SwiftStorage, self).__init__()
|
||||
self._swift_container = swift_container
|
||||
self._storage_path = storage_path
|
||||
|
||||
self._storage_path = storage_path.lstrip('/')
|
||||
self._simple_path_concat = simple_path_concat
|
||||
|
||||
self._auth_url = auth_url
|
||||
self._ca_cert_path = ca_cert_path
|
||||
|
|
@ -52,38 +57,30 @@ class SwiftStorage(BaseStorage):
|
|||
|
||||
def _get_connection(self):
|
||||
return Connection(
|
||||
authurl=self._auth_url,
|
||||
cacert=self._ca_cert_path,
|
||||
authurl=self._auth_url,
|
||||
cacert=self._ca_cert_path,
|
||||
|
||||
user=self._swift_user,
|
||||
key=self._swift_password,
|
||||
user=self._swift_user,
|
||||
key=self._swift_password,
|
||||
|
||||
auth_version=self._auth_version,
|
||||
os_options=self._os_options)
|
||||
|
||||
def _normalize_path(self, object_path=None):
|
||||
path = self._storage_path
|
||||
if not path.endswith('/'):
|
||||
path = path + '/'
|
||||
|
||||
path = path + (object_path or '')
|
||||
|
||||
# Openstack does not like paths starting with '/' and we always normalize
|
||||
# to remove trailing '/'
|
||||
if path.startswith('/'):
|
||||
path = path[1:]
|
||||
|
||||
if path.endswith('/'):
|
||||
path = path[:-1]
|
||||
|
||||
return path
|
||||
auth_version=self._auth_version,
|
||||
os_options=self._os_options
|
||||
)
|
||||
|
||||
def _normalize_path(self, object_path):
|
||||
""" No matter what inputs we get, we are going to return a path without a leading or trailing
|
||||
'/'
|
||||
"""
|
||||
if self._simple_path_concat:
|
||||
return (self._storage_path + object_path).rstrip('/')
|
||||
else:
|
||||
return os.path.join(self._storage_path, object_path).rstrip('/')
|
||||
|
||||
def _get_object(self, path, chunk_size=None):
|
||||
path = self._normalize_path(path)
|
||||
try:
|
||||
_, obj = self._get_connection().get_object(self._swift_container, path,
|
||||
resp_chunk_size=chunk_size)
|
||||
resp_chunk_size=chunk_size)
|
||||
return obj
|
||||
except Exception:
|
||||
logger.exception('Could not get object: %s', path)
|
||||
|
|
@ -134,18 +131,15 @@ class SwiftStorage(BaseStorage):
|
|||
|
||||
object_url = urlparse(object_url_value)
|
||||
scheme = object_url.scheme
|
||||
path = object_url.path
|
||||
path = object_url.path.rstrip('/')
|
||||
hostname = object_url.netloc
|
||||
|
||||
if not path.endswith('/'):
|
||||
path = path + '/'
|
||||
|
||||
object_path = self._normalize_path(object_path)
|
||||
|
||||
# Generate the signed HMAC body.
|
||||
method = 'GET'
|
||||
expires = int(time() + expires_in)
|
||||
full_path = '%s%s/%s' % (path, self._swift_container, object_path)
|
||||
full_path = '%s/%s/%s' % (path, self._swift_container, object_path)
|
||||
|
||||
hmac_body = '%s\n%s\n%s' % (method, expires, full_path)
|
||||
sig = hmac.new(self._temp_url_key.encode('utf-8'), hmac_body.encode('utf-8'), sha1).hexdigest()
|
||||
|
|
@ -241,7 +235,8 @@ class SwiftStorage(BaseStorage):
|
|||
while True:
|
||||
try:
|
||||
bytes_written, storage_metadata = self._stream_upload_segment(uuid, offset, length, in_fp,
|
||||
storage_metadata, content_type)
|
||||
storage_metadata,
|
||||
content_type)
|
||||
except IOError as ex:
|
||||
logger.warn('stream write error: %s', ex)
|
||||
error = ex
|
||||
|
|
@ -288,7 +283,7 @@ class SwiftStorage(BaseStorage):
|
|||
"""
|
||||
# Finally, we write an empty file at the proper location with a X-Object-Manifest
|
||||
# header pointing to the prefix for the segments.
|
||||
segments_prefix_path = self._normalize_path('%s/%s/' % (_SEGMENT_DIRECTORY, uuid))
|
||||
segments_prefix_path = self._normalize_path('%s/%s' % (_SEGMENT_DIRECTORY, uuid))
|
||||
contained_segments_prefix_path = '%s/%s' % (self._swift_container, segments_prefix_path)
|
||||
|
||||
self._put_object(final_path, '', headers={'X-Object-Manifest': contained_segments_prefix_path})
|
||||
|
|
|
|||
Reference in a new issue