vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Fix paths in swift driver to be more intelligible

Browse source
This commit is contained in:
Jake Moshenko 2016-01-14 15:30:01 -05:00
parent 0b1951a4a4
commit c6d7eba98d
1 changed files with 38 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

81
storage/swift.py
View file

@ -2,22 +2,25 @@
Uses: http://docs.openstack.org/developer/swift/overview_large_objects.html Uses: http://docs.openstack.org/developer/swift/overview_large_objects.html
""" """
from swiftclient.client import Connection, ClientException import os.path
from storage.basestorage import BaseStorage
from util.registry.generatorfile import GeneratorFile
from urlparse import urlparse
from random import SystemRandom
from hashlib import sha1
from time import time
from collections import namedtuple
from util.registry import filelike
import copy import copy
import hmac import hmac
import string import string
import logging import logging
from uuid import uuid4 from uuid import uuid4
from swiftclient.client import Connection, ClientException
from urlparse import urlparse
from random import SystemRandom
from hashlib import sha1
from time import time
from collections import namedtuple
from util.registry import filelike
from storage.basestorage import BaseStorage
from util.registry.generatorfile import GeneratorFile
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_PartUploadMetadata = namedtuple('_PartUploadMetadata', ['path', 'offset', 'length']) _PartUploadMetadata = namedtuple('_PartUploadMetadata', ['path', 'offset', 'length'])
@ -28,10 +31,12 @@ _MAXIMUM_SEGMENT_SIZE = 5000000000 # 5 GB
class SwiftStorage(BaseStorage): class SwiftStorage(BaseStorage):
def __init__(self, swift_container, storage_path, auth_url, swift_user, def __init__(self, swift_container, storage_path, auth_url, swift_user,
swift_password, auth_version=None, os_options=None, ca_cert_path=None, swift_password, auth_version=None, os_options=None, ca_cert_path=None,
temp_url_key=None): temp_url_key=None, simple_path_concat=False):
super(SwiftStorage, self).__init__() super(SwiftStorage, self).__init__()
self._swift_container = swift_container self._swift_container = swift_container
self._storage_path = storage_path
self._storage_path = storage_path.lstrip('/')
self._simple_path_concat = simple_path_concat
self._auth_url = auth_url self._auth_url = auth_url
self._ca_cert_path = ca_cert_path self._ca_cert_path = ca_cert_path
@ -52,38 +57,30 @@ class SwiftStorage(BaseStorage):
def _get_connection(self): def _get_connection(self):
return Connection( return Connection(
authurl=self._auth_url, authurl=self._auth_url,
cacert=self._ca_cert_path, cacert=self._ca_cert_path,
user=self._swift_user, user=self._swift_user,
key=self._swift_password, key=self._swift_password,
auth_version=self._auth_version, auth_version=self._auth_version,
os_options=self._os_options) os_options=self._os_options
)
def _normalize_path(self, object_path=None):
path = self._storage_path
if not path.endswith('/'):
path = path + '/'
path = path + (object_path or '')
# Openstack does not like paths starting with '/' and we always normalize
# to remove trailing '/'
if path.startswith('/'):
path = path[1:]
if path.endswith('/'):
path = path[:-1]
return path
def _normalize_path(self, object_path):
""" No matter what inputs we get, we are going to return a path without a leading or trailing
'/'
"""
if self._simple_path_concat:
return (self._storage_path + object_path).rstrip('/')
else:
return os.path.join(self._storage_path, object_path).rstrip('/')
def _get_object(self, path, chunk_size=None): def _get_object(self, path, chunk_size=None):
path = self._normalize_path(path) path = self._normalize_path(path)
try: try:
_, obj = self._get_connection().get_object(self._swift_container, path, _, obj = self._get_connection().get_object(self._swift_container, path,
resp_chunk_size=chunk_size) resp_chunk_size=chunk_size)
return obj return obj
except Exception: except Exception:
logger.exception('Could not get object: %s', path) logger.exception('Could not get object: %s', path)
@ -134,18 +131,15 @@ class SwiftStorage(BaseStorage):
object_url = urlparse(object_url_value) object_url = urlparse(object_url_value)
scheme = object_url.scheme scheme = object_url.scheme
path = object_url.path path = object_url.path.rstrip('/')
hostname = object_url.netloc hostname = object_url.netloc
if not path.endswith('/'):
path = path + '/'
object_path = self._normalize_path(object_path) object_path = self._normalize_path(object_path)
# Generate the signed HMAC body. # Generate the signed HMAC body.
method = 'GET' method = 'GET'
expires = int(time() + expires_in) expires = int(time() + expires_in)
full_path = '%s%s/%s' % (path, self._swift_container, object_path) full_path = '%s/%s/%s' % (path, self._swift_container, object_path)
hmac_body = '%s\n%s\n%s' % (method, expires, full_path) hmac_body = '%s\n%s\n%s' % (method, expires, full_path)
sig = hmac.new(self._temp_url_key.encode('utf-8'), hmac_body.encode('utf-8'), sha1).hexdigest() sig = hmac.new(self._temp_url_key.encode('utf-8'), hmac_body.encode('utf-8'), sha1).hexdigest()
@ -241,7 +235,8 @@ class SwiftStorage(BaseStorage):
while True: while True:
try: try:
bytes_written, storage_metadata = self._stream_upload_segment(uuid, offset, length, in_fp, bytes_written, storage_metadata = self._stream_upload_segment(uuid, offset, length, in_fp,
storage_metadata, content_type) storage_metadata,
content_type)
except IOError as ex: except IOError as ex:
logger.warn('stream write error: %s', ex) logger.warn('stream write error: %s', ex)
error = ex error = ex
@ -288,7 +283,7 @@ class SwiftStorage(BaseStorage):
""" """
# Finally, we write an empty file at the proper location with a X-Object-Manifest # Finally, we write an empty file at the proper location with a X-Object-Manifest
# header pointing to the prefix for the segments. # header pointing to the prefix for the segments.
segments_prefix_path = self._normalize_path('%s/%s/' % (_SEGMENT_DIRECTORY, uuid)) segments_prefix_path = self._normalize_path('%s/%s' % (_SEGMENT_DIRECTORY, uuid))
contained_segments_prefix_path = '%s/%s' % (self._swift_container, segments_prefix_path) contained_segments_prefix_path = '%s/%s' % (self._swift_container, segments_prefix_path)
self._put_object(final_path, '', headers={'X-Object-Manifest': contained_segments_prefix_path}) self._put_object(final_path, '', headers={'X-Object-Manifest': contained_segments_prefix_path})