From c6d7eba98dd9576b48f334c2c922e6c4289168ab Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Thu, 14 Jan 2016 15:30:01 -0500 Subject: [PATCH] Fix paths in swift driver to be more intelligible --- storage/swift.py | 81 +++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 43 deletions(-) diff --git a/storage/swift.py b/storage/swift.py index 5a0a5b92a..2618c6731 100644 --- a/storage/swift.py +++ b/storage/swift.py @@ -2,22 +2,25 @@ Uses: http://docs.openstack.org/developer/swift/overview_large_objects.html """ -from swiftclient.client import Connection, ClientException -from storage.basestorage import BaseStorage -from util.registry.generatorfile import GeneratorFile -from urlparse import urlparse -from random import SystemRandom -from hashlib import sha1 -from time import time -from collections import namedtuple -from util.registry import filelike - +import os.path import copy import hmac import string import logging from uuid import uuid4 +from swiftclient.client import Connection, ClientException +from urlparse import urlparse +from random import SystemRandom +from hashlib import sha1 +from time import time +from collections import namedtuple + +from util.registry import filelike +from storage.basestorage import BaseStorage +from util.registry.generatorfile import GeneratorFile + + logger = logging.getLogger(__name__) _PartUploadMetadata = namedtuple('_PartUploadMetadata', ['path', 'offset', 'length']) @@ -28,10 +31,12 @@ _MAXIMUM_SEGMENT_SIZE = 5000000000 # 5 GB class SwiftStorage(BaseStorage): def __init__(self, swift_container, storage_path, auth_url, swift_user, swift_password, auth_version=None, os_options=None, ca_cert_path=None, - temp_url_key=None): + temp_url_key=None, simple_path_concat=False): super(SwiftStorage, self).__init__() self._swift_container = swift_container - self._storage_path = storage_path + + self._storage_path = storage_path.lstrip('/') + self._simple_path_concat = simple_path_concat self._auth_url = auth_url self._ca_cert_path = ca_cert_path @@ -52,38 +57,30 @@ class SwiftStorage(BaseStorage): def _get_connection(self): return Connection( - authurl=self._auth_url, - cacert=self._ca_cert_path, + authurl=self._auth_url, + cacert=self._ca_cert_path, - user=self._swift_user, - key=self._swift_password, + user=self._swift_user, + key=self._swift_password, - auth_version=self._auth_version, - os_options=self._os_options) - - def _normalize_path(self, object_path=None): - path = self._storage_path - if not path.endswith('/'): - path = path + '/' - - path = path + (object_path or '') - - # Openstack does not like paths starting with '/' and we always normalize - # to remove trailing '/' - if path.startswith('/'): - path = path[1:] - - if path.endswith('/'): - path = path[:-1] - - return path + auth_version=self._auth_version, + os_options=self._os_options + ) + def _normalize_path(self, object_path): + """ No matter what inputs we get, we are going to return a path without a leading or trailing + '/' + """ + if self._simple_path_concat: + return (self._storage_path + object_path).rstrip('/') + else: + return os.path.join(self._storage_path, object_path).rstrip('/') def _get_object(self, path, chunk_size=None): path = self._normalize_path(path) try: _, obj = self._get_connection().get_object(self._swift_container, path, - resp_chunk_size=chunk_size) + resp_chunk_size=chunk_size) return obj except Exception: logger.exception('Could not get object: %s', path) @@ -134,18 +131,15 @@ class SwiftStorage(BaseStorage): object_url = urlparse(object_url_value) scheme = object_url.scheme - path = object_url.path + path = object_url.path.rstrip('/') hostname = object_url.netloc - if not path.endswith('/'): - path = path + '/' - object_path = self._normalize_path(object_path) # Generate the signed HMAC body. method = 'GET' expires = int(time() + expires_in) - full_path = '%s%s/%s' % (path, self._swift_container, object_path) + full_path = '%s/%s/%s' % (path, self._swift_container, object_path) hmac_body = '%s\n%s\n%s' % (method, expires, full_path) sig = hmac.new(self._temp_url_key.encode('utf-8'), hmac_body.encode('utf-8'), sha1).hexdigest() @@ -241,7 +235,8 @@ class SwiftStorage(BaseStorage): while True: try: bytes_written, storage_metadata = self._stream_upload_segment(uuid, offset, length, in_fp, - storage_metadata, content_type) + storage_metadata, + content_type) except IOError as ex: logger.warn('stream write error: %s', ex) error = ex @@ -288,7 +283,7 @@ class SwiftStorage(BaseStorage): """ # Finally, we write an empty file at the proper location with a X-Object-Manifest # header pointing to the prefix for the segments. - segments_prefix_path = self._normalize_path('%s/%s/' % (_SEGMENT_DIRECTORY, uuid)) + segments_prefix_path = self._normalize_path('%s/%s' % (_SEGMENT_DIRECTORY, uuid)) contained_segments_prefix_path = '%s/%s' % (self._swift_container, segments_prefix_path) self._put_object(final_path, '', headers={'X-Object-Manifest': contained_segments_prefix_path})