archive, chrootarchive: split out decompression

In `ApplyLayer` and `Untar`, the stream is magically decompressed. Since
this is not able to be toggled, rather than break this ./pkg/ API, add
an `ApplyUncompressedLayer` and `UntarUncompressed` that does not
magically decompress the layer stream.

Signed-off-by: Vincent Batts <vbatts@redhat.com>

archive/archive.go

@@ -633,8 +633,20 @@ loop:
 // The archive may be compressed with one of the following algorithms:
 //  identity (uncompressed), gzip, bzip2, xz.
 // FIXME: specify behavior when target path exists vs. doesn't exist.
-func Untar(archive io.Reader, dest string, options *TarOptions) error {
+func Untar(tarArchive io.Reader, dest string, options *TarOptions) error {
-	if archive == nil {
+	return untarHandler(tarArchive, dest, options, true)
+}
+
+// Untar reads a stream of bytes from `archive`, parses it as a tar archive,
+// and unpacks it into the directory at `dest`.
+// The archive must be an uncompressed stream.
+func UntarUncompressed(tarArchive io.Reader, dest string, options *TarOptions) error {
+	return untarHandler(tarArchive, dest, options, false)
+}
+
+// Handler for teasing out the automatic decompression
+func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decompress bool) error {
+	if tarArchive == nil {
 		return fmt.Errorf("Empty archive")
 	}
 	dest = filepath.Clean(dest)
@@ -644,12 +656,18 @@ func Untar(archive io.Reader, dest string, options *TarOptions) error {
 	if options.ExcludePatterns == nil {
 		options.ExcludePatterns = []string{}
 	}
-	decompressedArchive, err := DecompressStream(archive)
+
-	if err != nil {
+	var r io.Reader = tarArchive
-		return err
+	if decompress {
+		decompressedArchive, err := DecompressStream(tarArchive)
+		if err != nil {
+			return err
+		}
+		defer decompressedArchive.Close()
+		r = decompressedArchive
 	}
-	defer decompressedArchive.Close()
+
-	return Unpack(decompressedArchive, dest, options)
+	return Unpack(r, dest, options)
 }
 
 func (archiver *Archiver) TarUntar(src, dst string) error {

archive/diff.go

@@ -173,10 +173,24 @@ func UnpackLayer(dest string, layer ArchiveReader) (size int64, err error) {
 	return size, nil
 }
 
-// ApplyLayer parses a diff in the standard layer format from `layer`, and
+// ApplyLayer parses a diff in the standard layer format from `layer`,
-// applies it to the directory `dest`. Returns the size in bytes of the
+// and applies it to the directory `dest`. The stream `layer` can be
-// contents of the layer.
+// compressed or uncompressed.
+// Returns the size in bytes of the contents of the layer.
 func ApplyLayer(dest string, layer ArchiveReader) (int64, error) {
+	return applyLayerHandler(dest, layer, true)
+}
+
+// ApplyUncompressedLayer parses a diff in the standard layer format from
+// `layer`, and applies it to the directory `dest`. The stream `layer`
+// can only be uncompressed.
+// Returns the size in bytes of the contents of the layer.
+func ApplyUncompressedLayer(dest string, layer ArchiveReader) (int64, error) {
+	return applyLayerHandler(dest, layer, false)
+}
+
+// do the bulk load of ApplyLayer, but allow for not calling DecompressStream
+func applyLayerHandler(dest string, layer ArchiveReader, decompress bool) (int64, error) {
 	dest = filepath.Clean(dest)
 
 	// We need to be able to set any perms
@@ -186,9 +200,11 @@ func ApplyLayer(dest string, layer ArchiveReader) (int64, error) {
 	}
 	defer system.Umask(oldmask) // ignore err, ErrNotSupportedPlatform
 
-	layer, err = DecompressStream(layer)
+	if decompress {
-	if err != nil {
+		layer, err = DecompressStream(layer)
-		return 0, err
+		if err != nil {
+			return 0, err
+		}
 	}
 	return UnpackLayer(dest, layer)
 }

chrootarchive/archive.go

@@ -3,6 +3,7 @@ package chrootarchive
 import (
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -17,6 +18,18 @@ var chrootArchiver = &archive.Archiver{Untar: Untar}
 // The archive may be compressed with one of the following algorithms:
 //  identity (uncompressed), gzip, bzip2, xz.
 func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
+	return untarHandler(tarArchive, dest, options, true)
+}
+
+// UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive,
+// and unpacks it into the directory at `dest`.
+// The archive must be an uncompressed stream.
+func UntarUncompressed(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
+	return untarHandler(tarArchive, dest, options, false)
+}
+
+// Handler for teasing out the automatic decompression
+func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions, decompress bool) error {
 
 	if tarArchive == nil {
 		return fmt.Errorf("Empty archive")
@@ -35,13 +48,17 @@ func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error
 		}
 	}
 
-	decompressedArchive, err := archive.DecompressStream(tarArchive)
+	r := ioutil.NopCloser(tarArchive)
-	if err != nil {
+	if decompress {
-		return err
+		decompressedArchive, err := archive.DecompressStream(tarArchive)
+		if err != nil {
+			return err
+		}
+		defer decompressedArchive.Close()
+		r = decompressedArchive
 	}
-	defer decompressedArchive.Close()
 
-	return invokeUnpack(decompressedArchive, dest, options)
+	return invokeUnpack(r, dest, options)
 }
 
 // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other.

chrootarchive/archive_unix.go

@@ -49,7 +49,7 @@ func untar() {
 	os.Exit(0)
 }
 
-func invokeUnpack(decompressedArchive io.ReadCloser, dest string, options *archive.TarOptions) error {
+func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.TarOptions) error {
 
 	// We can't pass a potentially large exclude list directly via cmd line
 	// because we easily overrun the kernel's max argument/environment size

chrootarchive/diff_unix.go

@@ -65,20 +65,36 @@ func applyLayer() {
 	os.Exit(0)
 }
 
-// ApplyLayer parses a diff in the standard layer format from `layer`, and
+// ApplyLayer parses a diff in the standard layer format from `layer`,
-// applies it to the directory `dest`. Returns the size in bytes of the
+// and applies it to the directory `dest`. The stream `layer` can only be
-// contents of the layer.
+// uncompressed.
+// Returns the size in bytes of the contents of the layer.
 func ApplyLayer(dest string, layer archive.ArchiveReader) (size int64, err error) {
+	return applyLayerHandler(dest, layer, true)
+}
+
+// ApplyUncompressedLayer parses a diff in the standard layer format from
+// `layer`, and applies it to the directory `dest`. The stream `layer`
+// can only be uncompressed.
+// Returns the size in bytes of the contents of the layer.
+func ApplyUncompressedLayer(dest string, layer archive.ArchiveReader) (int64, error) {
+	return applyLayerHandler(dest, layer, false)
+}
+
+func applyLayerHandler(dest string, layer archive.ArchiveReader, decompress bool) (size int64, err error) {
 	dest = filepath.Clean(dest)
-	decompressed, err := archive.DecompressStream(layer)
+	if decompress {
-	if err != nil {
+		decompressed, err := archive.DecompressStream(layer)
-		return 0, err
+		if err != nil {
+			return 0, err
+		}
+		defer decompressed.Close()
+
+		layer = decompressed
 	}
 
-	defer decompressed.Close()
-
 	cmd := reexec.Command("docker-applyLayer", dest)
-	cmd.Stdin = decompressed
+	cmd.Stdin = layer
 
 	outBuf, errBuf := new(bytes.Buffer), new(bytes.Buffer)
 	cmd.Stdout, cmd.Stderr = outBuf, errBuf