Implement some new methods on the storage engines.

storage/basestorage.py

@@ -42,18 +42,9 @@ class StoragePaths(object):
 
 
 class BaseStorage(StoragePaths):
-  """Storage is organized as follow:
-  $ROOT/images/<image_id>/json
-  $ROOT/images/<image_id>/layer
-  $ROOT/repositories/<namespace>/<repository_name>/<tag_name>
-  """
-
-  # Useful if we want to change those locations later without rewriting
-  # the code which uses Storage
-  repositories = 'repositories'
-  images = 'images'
-  # Set the IO buffer to 64kB
-  buffer_size = 64 * 1024
+  def __init__(self):
+    # Set the IO buffer to 64kB
+    self.buffer_size = 64 * 1024
 
   def setup(self):
     """ Called to perform any storage system setup. """
@@ -99,31 +90,55 @@ class BaseStorage(StoragePaths):
   def get_checksum(self, path):
     raise NotImplementedError
 
+  def stream_write_to_fp(self, in_fp, out_fp, num_bytes=-1):
+    """ Copy the specified number of bytes from the input file stream to the output stream. If
+        num_bytes < 0 copy until the stream ends.
+    """
+    bytes_copied = 0
+    while bytes_copied < num_bytes or num_bytes == -1:
+      size_to_read = min(num_bytes - bytes_copied, self.buffer_size)
+      if size_to_read < 0:
+        size_to_read = self.buffer_size
+
+      try:
+        buf = in_fp.read(size_to_read)
+        if not buf:
+          break
+        out_fp.write(buf)
+        bytes_copied += len(buf)
+      except IOError:
+        break
+
+    return bytes_copied
+
 
 class InvalidChunkException(RuntimeError):
   pass
 
 
 class BaseStorageV2(BaseStorage):
-  def initiate_chunked_upload(self, upload_uuid):
-    """ Start a new chunked upload
+  def initiate_chunked_upload(self):
+    """ Start a new chunked upload, returning the uuid and any associated storage metadata
     """
     raise NotImplementedError
 
-  def stream_upload_chunk(self, uuid, offset, length, in_fp, hash_obj):
+  def stream_upload_chunk(self, uuid, offset, length, in_fp, storage_metadata):
     """ Upload the specified amount of data from the given file pointer to the chunked destination
-        specified, starting at the given offset. Raises InvalidChunkException if the offset or
-        length can not be accepted.
+        specified, starting at the given offset. Returns the number of bytes uploaded, and a new
+        version of the storage_metadata. Raises InvalidChunkException if the offset or length can
+        not be accepted.
     """
     raise NotImplementedError
 
-  def complete_chunked_upload(self, uuid, final_path):
+  def complete_chunked_upload(self, uuid, final_path, storage_metadata):
     """ Complete the chunked upload and store the final results in the path indicated.
+        Returns nothing.
     """
     raise NotImplementedError
 
-  def cancel_chunked_upload(self, uuid):
+  def cancel_chunked_upload(self, uuid, storage_metadata):
     """ Cancel the chunked upload and clean up any outstanding partially uploaded data.
+        Returns nothing.
     """
     raise NotImplementedError