- Handle missing images properly

- Add support for deleting directories
- Add a slew of tests for deletion of directories and other kinds of deletion and layering

util/aufs.py

@@ -4,31 +4,28 @@ AUFS_METADATA = u'.wh..wh.'
 AUFS_WHITEOUT = u'.wh.'
 AUFS_WHITEOUT_PREFIX_LENGTH = len(AUFS_WHITEOUT)
 
-def is_aufs_metadata(filepath):
-  """ Returns whether the given filepath references an AUFS metadata file. """
-  filename = os.path.basename(filepath)
-  return filename.startswith(AUFS_METADATA) or filepath.startswith(AUFS_METADATA)
+def is_aufs_metadata(absolute):
+  """ Returns whether the given absolute references an AUFS metadata file. """
+  filename = os.path.basename(absolute)
+  return filename.startswith(AUFS_METADATA) or absolute.startswith(AUFS_METADATA)
 
-def get_deleted_filename(filepath):
+def get_deleted_filename(absolute):
   """ Returns the name of the deleted file referenced by the AUFS whiteout file at
       the given path or None if the file path does not reference a whiteout file.
   """
-  filename = os.path.basename(filepath)
+  filename = os.path.basename(absolute)
   if not filename.startswith(AUFS_WHITEOUT):
     return None
 
   return filename[AUFS_WHITEOUT_PREFIX_LENGTH:]
 
-def get_deleted_prefix(filepath):
+def get_deleted_prefix(absolute):
   """ Returns the path prefix of the deleted file referenced by the AUFS whiteout file at
      the given path or None if the file path does not reference a whiteout file.
   """
-  deleted_filename = get_deleted_filename(filepath)
+  deleted_filename = get_deleted_filename(absolute)
   if deleted_filename is None:
     return None
 
-  dirname = os.path.dirname(filepath)
-  if not dirname:
-    return deleted_filename
-
-  return os.path.join('/', dirname, deleted_filename)
+  dirname = os.path.dirname(absolute)
+  return os.path.join('/', dirname, deleted_filename)[1:]

util/streamlayerformat.py

@@ -12,9 +12,13 @@ AUFS_WHITEOUT_PREFIX_LENGTH = len(AUFS_WHITEOUT)
 class StreamLayerMerger(object):
   """ Class which creates a generator of the combined TAR data for a set of Docker layers. """
   def __init__(self, layer_iterator):
-    self.trie = marisa_trie.Trie()
+    self.path_trie = marisa_trie.Trie()
+    self.path_encountered = []
+
+    self.prefix_trie = marisa_trie.Trie()
+    self.prefix_encountered = []
+
     self.layer_iterator = layer_iterator
-    self.encountered = []
 
   def get_generator(self):
     for current_layer in self.layer_iterator():
@@ -60,8 +64,9 @@ class StreamLayerMerger(object):
       # Close the layer stream now that we're done with it.
       tar_file.close()
 
-      # Update the trie with the new encountered entries.
-      self.trie = marisa_trie.Trie(self.encountered)
+      # Update the tries.
+      self.path_trie = marisa_trie.Trie(self.path_encountered)
+      self.prefix_trie = marisa_trie.Trie(self.prefix_encountered)
       
     # Last two records are empty in TAR spec.
     yield '\0' * 512
@@ -76,16 +81,17 @@ class StreamLayerMerger(object):
       return False
 
     # Add any prefix of deleted paths to the prefix list.
-    deleted_prefix = get_deleted_prefix(absolute)
+    deleted_prefix = get_deleted_prefix(absolute)    
     if deleted_prefix is not None:
-      self.encountered.append(deleted_prefix)
+      self.prefix_encountered.append(deleted_prefix)
       return False
 
     # Check if this file has already been encountered somewhere. If so,
     # skip it.
-    if unicode(absolute) in self.trie:
+    ubsolute = unicode(absolute)
+    if ubsolute in self.path_trie or any(self.prefix_trie.iter_prefixes(ubsolute)):
       return False
 
     # Otherwise, add the path to the encountered list and return it.
-    self.encountered.append(absolute)
+    self.path_encountered.append(absolute)
     return True