""" Swift storage driver. Uses: http://docs.openstack.org/developer/swift/overview_large_objects.html """ import os.path import copy import hmac import string import logging import json from _pyio import BufferedReader from collections import namedtuple from hashlib import sha1 from random import SystemRandom from time import time from urlparse import urlparse from uuid import uuid4 from cachetools import lru_cache from swiftclient.client import Connection, ClientException, ReadableToIterable from storage.basestorage import BaseStorage from util.registry import filelike from util.registry.generatorfile import GeneratorFile logger = logging.getLogger(__name__) _PartUploadMetadata = namedtuple('_PartUploadMetadata', ['path', 'offset', 'length']) _SEGMENTS_KEY = 'segments' _EMPTY_SEGMENTS_KEY = 'emptysegments' _SEGMENT_DIRECTORY = 'segments' _MAXIMUM_SEGMENT_SIZE = 200000000 # ~200 MB _DEFAULT_SWIFT_CONNECT_TIMEOUT = 5 # seconds _CHUNK_CLEANUP_DELAY = 30 # seconds class SwiftStorage(BaseStorage): def __init__(self, context, swift_container, storage_path, auth_url, swift_user, swift_password, auth_version=None, os_options=None, ca_cert_path=None, temp_url_key=None, simple_path_concat=False, connect_timeout=None, retry_count=None, retry_on_ratelimit=True): super(SwiftStorage, self).__init__() self._swift_container = swift_container self._context = context self._storage_path = storage_path.lstrip('/') self._simple_path_concat = simple_path_concat self._auth_url = auth_url self._ca_cert_path = ca_cert_path self._swift_user = swift_user self._swift_password = swift_password self._temp_url_key = temp_url_key self._connect_timeout = connect_timeout self._retry_count = retry_count self._retry_on_ratelimit = retry_on_ratelimit try: self._auth_version = int(auth_version or '2') except ValueError: self._auth_version = 2 self._os_options = os_options or {} self._initialized = False def _get_connection(self): return Connection( authurl=self._auth_url, cacert=self._ca_cert_path, user=self._swift_user, key=self._swift_password, auth_version=self._auth_version, os_options=self._os_options, retry_on_ratelimit=self._retry_on_ratelimit, timeout=self._connect_timeout or _DEFAULT_SWIFT_CONNECT_TIMEOUT, retries=self._retry_count or 5, ) def _normalize_path(self, object_path): """ No matter what inputs we get, we are going to return a path without a leading or trailing '/' """ if self._simple_path_concat: return (self._storage_path + object_path).rstrip('/') else: return os.path.join(self._storage_path, object_path).rstrip('/') def _get_object(self, path, chunk_size=None): path = self._normalize_path(path) try: _, obj = self._get_connection().get_object(self._swift_container, path, resp_chunk_size=chunk_size) return obj except ClientException as ex: logger.exception('Could not get object at path %s: %s', path, ex) raise IOError('Path %s not found' % path) def _put_object(self, path, content, chunk=None, content_type=None, content_encoding=None, headers=None): path = self._normalize_path(path) headers = headers or {} if content_encoding is not None: headers['Content-Encoding'] = content_encoding is_filelike = hasattr(content, 'read') if is_filelike: content = ReadableToIterable(content, md5=True) try: etag = self._get_connection().put_object(self._swift_container, path, content, chunk_size=chunk, content_type=content_type, headers=headers) except ClientException: # We re-raise client exception here so that validation of config during setup can see # the client exception messages. raise # If we wrapped the content in a ReadableToIterable, compare its MD5 to the etag returned. If # they don't match, raise an IOError indicating a write failure. if is_filelike: if etag != content.get_md5sum(): logger.error('Got mismatch in md5 etag for path %s: Expected %s, but server has %s', path, content.get_md5sum(), etag) raise IOError('upload verification failed for path {0}:' 'md5 mismatch, local {1} != remote {2}' .format(path, content.get_md5sum(), etag)) def _head_object(self, path): path = self._normalize_path(path) try: return self._get_connection().head_object(self._swift_container, path) except ClientException as ce: if ce.http_status != 404: logger.exception('Could not head object at path %s: %s', path, ce) return None @lru_cache(maxsize=1) def _get_root_storage_url(self): """ Returns the root storage URL for this Swift storage. Note that since this requires a call to Swift, we cache the result of this function call. """ storage_url, _ = self._get_connection().get_auth() return storage_url def get_direct_download_url(self, object_path, request_ip=None, expires_in=60, requires_cors=False, head=False): if requires_cors: return None # Reference: http://docs.openstack.org/juno/config-reference/content/object-storage-tempurl.html if not self._temp_url_key: return None # Retrieve the root storage URL for the connection. try: root_storage_url = self._get_root_storage_url() except ClientException: logger.exception('Got client exception when trying to load Swift auth') return None parsed_storage_url = urlparse(root_storage_url) scheme = parsed_storage_url.scheme path = parsed_storage_url.path.rstrip('/') hostname = parsed_storage_url.netloc object_path = self._normalize_path(object_path) # Generate the signed HMAC body. method = 'HEAD' if head else 'GET' expires = int(time() + expires_in) full_path = '%s/%s/%s' % (path, self._swift_container, object_path) hmac_body = '%s\n%s\n%s' % (method, expires, full_path) sig = hmac.new(self._temp_url_key.encode('utf-8'), hmac_body.encode('utf-8'), sha1).hexdigest() surl = '{scheme}://{host}{full_path}?temp_url_sig={sig}&temp_url_expires={expires}' return surl.format(scheme=scheme, host=hostname, full_path=full_path, sig=sig, expires=expires) def validate(self, client): super(SwiftStorage, self).validate(client) if self._temp_url_key: # Generate a direct download URL. dd_url = self.get_direct_download_url('_verify') if not dd_url: raise Exception('Could not validate direct download URL; the token may be invalid.') # Try to retrieve the direct download URL. response = client.get(dd_url, timeout=2) if response.status_code != 200: logger.debug('Direct download failure: %s => %s with body %s', dd_url, response.status_code, response.text) msg = 'Direct download URL failed with status code %s. Please check your temp-url-key.' raise Exception(msg % response.status_code) def get_content(self, path): return self._get_object(path) def put_content(self, path, content): self._put_object(path, content) def stream_read(self, path): for data in self._get_object(path, self.buffer_size): yield data def stream_read_file(self, path): return GeneratorFile(self.stream_read(path)) def stream_write(self, path, fp, content_type=None, content_encoding=None): self._put_object(path, fp, self.buffer_size, content_type=content_type, content_encoding=content_encoding) def exists(self, path): return bool(self._head_object(path)) def remove(self, path): # Retrieve the object so we can see if it is segmented. If so, we'll delete its segments after # removing the object. try: headers = self._head_object(path) except ClientException as ex: logger.warning('Could not head for delete of path %s: %s', path, str(ex)) raise IOError('Cannot delete path: %s' % path) logger.debug('Found headers for path %s to delete: %s', path, headers) # Delete the path itself. path = self._normalize_path(path) try: self._get_connection().delete_object(self._swift_container, path) except ClientException as ex: logger.warning('Could not delete path %s: %s', path, str(ex)) raise IOError('Cannot delete path: %s' % path) # Delete the segments. object_manifest = headers.get('x-object-manifest', headers.get('X-Object-Manifest')) if object_manifest is not None: logger.debug('Found DLO for path %s: %s', path, object_manifest) # Remove the container name from the beginning. container_name, prefix_path = object_manifest.split('/', 1) if container_name != self._swift_container: logger.error('Expected container name %s, found path %s', self._swift_container, prefix_path) raise Exception("How did we end up with an invalid container name?") logger.debug('Loading Dynamic Large Object segments for path prefix %s', prefix_path) try: _, container_objects = self._get_connection().get_container(self._swift_container, full_listing=True, prefix=prefix_path) except ClientException as ex: logger.warning('Could not load objects with prefix path %s: %s', prefix_path, str(ex)) raise IOError('Cannot load path: %s' % prefix_path) logger.debug('Found Dynamic Large Object segments for path prefix %s: %s', prefix_path, len(container_objects)) for obj in container_objects: try: logger.debug('Deleting Dynamic Large Object segment %s for path prefix %s', obj['name'], prefix_path) self._get_connection().delete_object(self._swift_container, obj['name']) except ClientException as ex: logger.warning('Could not delete object with path %s: %s', obj['name'], str(ex)) raise IOError('Cannot delete path: %s' % obj['name']) def _random_checksum(self, count): chars = string.ascii_uppercase + string.digits return ''.join(SystemRandom().choice(chars) for _ in range(count)) def get_checksum(self, path): headers = self._head_object(path) if not headers: raise IOError('Cannot lookup path: %s' % path) return headers.get('etag', '')[1:-1][:7] or self._random_checksum(7) @staticmethod def _segment_list_from_metadata(storage_metadata, key=_SEGMENTS_KEY): return [_PartUploadMetadata(*segment_args) for segment_args in storage_metadata[key]] def initiate_chunked_upload(self): random_uuid = str(uuid4()) metadata = { _SEGMENTS_KEY: [], _EMPTY_SEGMENTS_KEY: [], } return random_uuid, metadata def stream_upload_chunk(self, uuid, offset, length, in_fp, storage_metadata, content_type=None): if length == 0: return 0, storage_metadata, None # Note: Swift limits segments in size, so we need to sub-divide chunks into segments # based on the configured maximum. total_bytes_written = 0 upload_error = None read_until_end = length == filelike.READ_UNTIL_END while True: try: bytes_written, storage_metadata = self._stream_upload_segment(uuid, offset, length, in_fp, storage_metadata, content_type) except IOError as ex: message = ('Error writing to stream in stream_upload_chunk for uuid %s (offset %s' + ', length %s, metadata: %s): %s') logger.exception(message, uuid, offset, length, storage_metadata, ex) upload_error = ex break if not read_until_end: length = length - bytes_written offset = offset + bytes_written total_bytes_written = total_bytes_written + bytes_written if bytes_written == 0 or (not read_until_end and length <= 0): return total_bytes_written, storage_metadata, upload_error return total_bytes_written, storage_metadata, upload_error def _stream_upload_segment(self, uuid, offset, length, in_fp, storage_metadata, content_type): updated_metadata = copy.deepcopy(storage_metadata) segment_count = len(updated_metadata[_SEGMENTS_KEY]) segment_path = '%s/%s/%s' % (_SEGMENT_DIRECTORY, uuid, '%09d' % segment_count) # Track the number of bytes read and if an explicit length is specified, limit the # file stream to that length. if length == filelike.READ_UNTIL_END: length = _MAXIMUM_SEGMENT_SIZE else: length = min(_MAXIMUM_SEGMENT_SIZE, length) limiting_fp = filelike.LimitingStream(in_fp, length) # If retries are requested, then we need to use a buffered reader to allow for calls to # seek() on retries from within the Swift client. if self._retry_count > 0: limiting_fp = BufferedReader(limiting_fp, buffer_size=length) # Write the segment to Swift. self.stream_write(segment_path, limiting_fp, content_type) # We are only going to track keys to which data was confirmed written. bytes_written = limiting_fp.tell() if bytes_written > 0: updated_metadata[_SEGMENTS_KEY].append(_PartUploadMetadata(segment_path, offset, bytes_written)) else: updated_metadata[_EMPTY_SEGMENTS_KEY].append(_PartUploadMetadata(segment_path, offset, bytes_written)) return bytes_written, updated_metadata def complete_chunked_upload(self, uuid, final_path, storage_metadata): """ Complete the chunked upload and store the final results in the path indicated. Returns nothing. """ # Check all potentially empty segments against the segments that were uploaded; if the path # is still empty, then we queue the segment to be deleted. if self._context.chunk_cleanup_queue is not None: nonempty_segments = SwiftStorage._segment_list_from_metadata(storage_metadata, key=_SEGMENTS_KEY) potentially_empty_segments = SwiftStorage._segment_list_from_metadata(storage_metadata, key=_EMPTY_SEGMENTS_KEY) nonempty_paths = set([segment.path for segment in nonempty_segments]) for segment in potentially_empty_segments: if segment.path in nonempty_paths: continue # Queue the chunk to be deleted, as it is empty and therefore unused. self._context.chunk_cleanup_queue.put( ['segment/%s/%s' % (self._context.location, uuid)], json.dumps({ 'location': self._context.location, 'uuid': uuid, 'path': segment.path, }), available_after=_CHUNK_CLEANUP_DELAY) # Finally, we write an empty file at the proper location with a X-Object-Manifest # header pointing to the prefix for the segments. segments_prefix_path = self._normalize_path('%s/%s' % (_SEGMENT_DIRECTORY, uuid)) contained_segments_prefix_path = '%s/%s' % (self._swift_container, segments_prefix_path) self._put_object(final_path, '', headers={'X-Object-Manifest': contained_segments_prefix_path}) def cancel_chunked_upload(self, uuid, storage_metadata): """ Cancel the chunked upload and clean up any outstanding partially uploaded data. Returns nothing. """ # Delete all the uploaded segments. for segment in SwiftStorage._segment_list_from_metadata(storage_metadata, key=_SEGMENTS_KEY): self.remove(segment.path) for segment in SwiftStorage._segment_list_from_metadata(storage_metadata, key=_EMPTY_SEGMENTS_KEY): self.remove(segment.path) def copy_to(self, destination, path): if (self.__class__ == destination.__class__ and self._swift_user == destination._swift_user and self._swift_password == destination._swift_password and self._auth_url == destination._auth_url and self._auth_version == destination._auth_version): logger.debug('Copying file from swift %s to swift %s via a Swift copy', self._swift_container, destination) normalized_path = self._normalize_path(path) target = '/%s/%s' % (destination._swift_container, normalized_path) try: self._get_connection().copy_object(self._swift_container, normalized_path, target) except ClientException as ex: logger.exception('Could not swift copy path %s: %s', path, ex) raise IOError('Failed to swift copy path %s' % path) return # Fallback to a slower, default copy. logger.debug('Copying file from swift %s to %s via a streamed copy', self._swift_container, destination) with self.stream_read_file(path) as fp: destination.stream_write(path, fp)