Move archive package into pkg/archive

Now that the archive package does not depend on any docker-specific
packages, only those in pkg and vendor, it can be safely moved into pkg.

Signed-off-by: Rafe Colton <rafael.colton@gmail.com>

archive/MAINTAINERS

@@ -0,0 +1,2 @@
+Cristian Staretu <cristian.staretu@gmail.com> (@unclejack)
+Tibor Vass <teabee89@gmail.com> (@tiborvass)

archive/README.md

@@ -0,0 +1 @@
+This code provides helper functions for dealing with archive files.

archive/archive.go

@@ -0,0 +1,706 @@
+package archive
+
+import (
+	"bufio"
+	"bytes"
+	"compress/bzip2"
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar"
+
+	"github.com/docker/docker/pkg/fileutils"
+	"github.com/docker/docker/pkg/log"
+	"github.com/docker/docker/pkg/pools"
+	"github.com/docker/docker/pkg/promise"
+	"github.com/docker/docker/pkg/system"
+)
+
+type (
+	Archive       io.ReadCloser
+	ArchiveReader io.Reader
+	Compression   int
+	TarOptions    struct {
+		Includes    []string
+		Excludes    []string
+		Compression Compression
+		NoLchown    bool
+	}
+)
+
+var (
+	ErrNotImplemented = errors.New("Function not implemented")
+)
+
+const (
+	Uncompressed Compression = iota
+	Bzip2
+	Gzip
+	Xz
+)
+
+func IsArchive(header []byte) bool {
+	compression := DetectCompression(header)
+	if compression != Uncompressed {
+		return true
+	}
+	r := tar.NewReader(bytes.NewBuffer(header))
+	_, err := r.Next()
+	return err == nil
+}
+
+func DetectCompression(source []byte) Compression {
+	for compression, m := range map[Compression][]byte{
+		Bzip2: {0x42, 0x5A, 0x68},
+		Gzip:  {0x1F, 0x8B, 0x08},
+		Xz:    {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
+	} {
+		if len(source) < len(m) {
+			log.Debugf("Len too short")
+			continue
+		}
+		if bytes.Compare(m, source[:len(m)]) == 0 {
+			return compression
+		}
+	}
+	return Uncompressed
+}
+
+func xzDecompress(archive io.Reader) (io.ReadCloser, error) {
+	args := []string{"xz", "-d", "-c", "-q"}
+
+	return CmdStream(exec.Command(args[0], args[1:]...), archive)
+}
+
+func DecompressStream(archive io.Reader) (io.ReadCloser, error) {
+	p := pools.BufioReader32KPool
+	buf := p.Get(archive)
+	bs, err := buf.Peek(10)
+	if err != nil {
+		return nil, err
+	}
+	log.Debugf("[tar autodetect] n: %v", bs)
+
+	compression := DetectCompression(bs)
+	switch compression {
+	case Uncompressed:
+		readBufWrapper := p.NewReadCloserWrapper(buf, buf)
+		return readBufWrapper, nil
+	case Gzip:
+		gzReader, err := gzip.NewReader(buf)
+		if err != nil {
+			return nil, err
+		}
+		readBufWrapper := p.NewReadCloserWrapper(buf, gzReader)
+		return readBufWrapper, nil
+	case Bzip2:
+		bz2Reader := bzip2.NewReader(buf)
+		readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader)
+		return readBufWrapper, nil
+	case Xz:
+		xzReader, err := xzDecompress(buf)
+		if err != nil {
+			return nil, err
+		}
+		readBufWrapper := p.NewReadCloserWrapper(buf, xzReader)
+		return readBufWrapper, nil
+	default:
+		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
+	}
+}
+
+func CompressStream(dest io.WriteCloser, compression Compression) (io.WriteCloser, error) {
+	p := pools.BufioWriter32KPool
+	buf := p.Get(dest)
+	switch compression {
+	case Uncompressed:
+		writeBufWrapper := p.NewWriteCloserWrapper(buf, buf)
+		return writeBufWrapper, nil
+	case Gzip:
+		gzWriter := gzip.NewWriter(dest)
+		writeBufWrapper := p.NewWriteCloserWrapper(buf, gzWriter)
+		return writeBufWrapper, nil
+	case Bzip2, Xz:
+		// archive/bzip2 does not support writing, and there is no xz support at all
+		// However, this is not a problem as docker only currently generates gzipped tars
+		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
+	default:
+		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
+	}
+}
+
+func (compression *Compression) Extension() string {
+	switch *compression {
+	case Uncompressed:
+		return "tar"
+	case Bzip2:
+		return "tar.bz2"
+	case Gzip:
+		return "tar.gz"
+	case Xz:
+		return "tar.xz"
+	}
+	return ""
+}
+
+func addTarFile(path, name string, tw *tar.Writer, twBuf *bufio.Writer) error {
+	fi, err := os.Lstat(path)
+	if err != nil {
+		return err
+	}
+
+	link := ""
+	if fi.Mode()&os.ModeSymlink != 0 {
+		if link, err = os.Readlink(path); err != nil {
+			return err
+		}
+	}
+
+	hdr, err := tar.FileInfoHeader(fi, link)
+	if err != nil {
+		return err
+	}
+
+	if fi.IsDir() && !strings.HasSuffix(name, "/") {
+		name = name + "/"
+	}
+
+	hdr.Name = name
+
+	stat, ok := fi.Sys().(*syscall.Stat_t)
+	if ok {
+		// Currently go does not fill in the major/minors
+		if stat.Mode&syscall.S_IFBLK == syscall.S_IFBLK ||
+			stat.Mode&syscall.S_IFCHR == syscall.S_IFCHR {
+			hdr.Devmajor = int64(major(uint64(stat.Rdev)))
+			hdr.Devminor = int64(minor(uint64(stat.Rdev)))
+		}
+
+	}
+
+	capability, _ := system.Lgetxattr(path, "security.capability")
+	if capability != nil {
+		hdr.Xattrs = make(map[string]string)
+		hdr.Xattrs["security.capability"] = string(capability)
+	}
+
+	if err := tw.WriteHeader(hdr); err != nil {
+		return err
+	}
+
+	if hdr.Typeflag == tar.TypeReg {
+		file, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+
+		twBuf.Reset(tw)
+		_, err = io.Copy(twBuf, file)
+		file.Close()
+		if err != nil {
+			return err
+		}
+		err = twBuf.Flush()
+		if err != nil {
+			return err
+		}
+		twBuf.Reset(nil)
+	}
+
+	return nil
+}
+
+func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool) error {
+	// hdr.Mode is in linux format, which we can use for sycalls,
+	// but for os.Foo() calls we need the mode converted to os.FileMode,
+	// so use hdrInfo.Mode() (they differ for e.g. setuid bits)
+	hdrInfo := hdr.FileInfo()
+
+	switch hdr.Typeflag {
+	case tar.TypeDir:
+		// Create directory unless it exists as a directory already.
+		// In that case we just want to merge the two
+		if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) {
+			if err := os.Mkdir(path, hdrInfo.Mode()); err != nil {
+				return err
+			}
+		}
+
+	case tar.TypeReg, tar.TypeRegA:
+		// Source is regular file
+		file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode())
+		if err != nil {
+			return err
+		}
+		if _, err := io.Copy(file, reader); err != nil {
+			file.Close()
+			return err
+		}
+		file.Close()
+
+	case tar.TypeBlock, tar.TypeChar, tar.TypeFifo:
+		mode := uint32(hdr.Mode & 07777)
+		switch hdr.Typeflag {
+		case tar.TypeBlock:
+			mode |= syscall.S_IFBLK
+		case tar.TypeChar:
+			mode |= syscall.S_IFCHR
+		case tar.TypeFifo:
+			mode |= syscall.S_IFIFO
+		}
+
+		if err := syscall.Mknod(path, mode, int(mkdev(hdr.Devmajor, hdr.Devminor))); err != nil {
+			return err
+		}
+
+	case tar.TypeLink:
+		if err := os.Link(filepath.Join(extractDir, hdr.Linkname), path); err != nil {
+			return err
+		}
+
+	case tar.TypeSymlink:
+		if err := os.Symlink(hdr.Linkname, path); err != nil {
+			return err
+		}
+
+	case tar.TypeXGlobalHeader:
+		log.Debugf("PAX Global Extended Headers found and ignored")
+		return nil
+
+	default:
+		return fmt.Errorf("Unhandled tar header type %d\n", hdr.Typeflag)
+	}
+
+	if err := os.Lchown(path, hdr.Uid, hdr.Gid); err != nil && Lchown {
+		return err
+	}
+
+	for key, value := range hdr.Xattrs {
+		if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil {
+			return err
+		}
+	}
+
+	// There is no LChmod, so ignore mode for symlink. Also, this
+	// must happen after chown, as that can modify the file mode
+	if hdr.Typeflag != tar.TypeSymlink {
+		if err := os.Chmod(path, hdrInfo.Mode()); err != nil {
+			return err
+		}
+	}
+
+	ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)}
+	// syscall.UtimesNano doesn't support a NOFOLLOW flag atm, and
+	if hdr.Typeflag != tar.TypeSymlink {
+		if err := system.UtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform {
+			return err
+		}
+	} else {
+		if err := system.LUtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform {
+			return err
+		}
+	}
+	return nil
+}
+
+// Tar creates an archive from the directory at `path`, and returns it as a
+// stream of bytes.
+func Tar(path string, compression Compression) (io.ReadCloser, error) {
+	return TarWithOptions(path, &TarOptions{Compression: compression})
+}
+
+func escapeName(name string) string {
+	escaped := make([]byte, 0)
+	for i, c := range []byte(name) {
+		if i == 0 && c == '/' {
+			continue
+		}
+		// all printable chars except "-" which is 0x2d
+		if (0x20 <= c && c <= 0x7E) && c != 0x2d {
+			escaped = append(escaped, c)
+		} else {
+			escaped = append(escaped, fmt.Sprintf("\\%03o", c)...)
+		}
+	}
+	return string(escaped)
+}
+
+// TarWithOptions creates an archive from the directory at `path`, only including files whose relative
+// paths are included in `options.Includes` (if non-nil) or not in `options.Excludes`.
+func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) {
+	pipeReader, pipeWriter := io.Pipe()
+
+	compressWriter, err := CompressStream(pipeWriter, options.Compression)
+	if err != nil {
+		return nil, err
+	}
+
+	tw := tar.NewWriter(compressWriter)
+
+	go func() {
+		// In general we log errors here but ignore them because
+		// during e.g. a diff operation the container can continue
+		// mutating the filesystem and we can see transient errors
+		// from this
+
+		if options.Includes == nil {
+			options.Includes = []string{"."}
+		}
+
+		twBuf := pools.BufioWriter32KPool.Get(nil)
+		defer pools.BufioWriter32KPool.Put(twBuf)
+
+		for _, include := range options.Includes {
+			filepath.Walk(filepath.Join(srcPath, include), func(filePath string, f os.FileInfo, err error) error {
+				if err != nil {
+					log.Debugf("Tar: Can't stat file %s to tar: %s", srcPath, err)
+					return nil
+				}
+
+				relFilePath, err := filepath.Rel(srcPath, filePath)
+				if err != nil {
+					return nil
+				}
+
+				skip, err := fileutils.Matches(relFilePath, options.Excludes)
+				if err != nil {
+					log.Debugf("Error matching %s", relFilePath, err)
+					return err
+				}
+
+				if skip {
+					if f.IsDir() {
+						return filepath.SkipDir
+					}
+					return nil
+				}
+
+				if err := addTarFile(filePath, relFilePath, tw, twBuf); err != nil {
+					log.Debugf("Can't add file %s to tar: %s", srcPath, err)
+				}
+				return nil
+			})
+		}
+
+		// Make sure to check the error on Close.
+		if err := tw.Close(); err != nil {
+			log.Debugf("Can't close tar writer: %s", err)
+		}
+		if err := compressWriter.Close(); err != nil {
+			log.Debugf("Can't close compress writer: %s", err)
+		}
+		if err := pipeWriter.Close(); err != nil {
+			log.Debugf("Can't close pipe writer: %s", err)
+		}
+	}()
+
+	return pipeReader, nil
+}
+
+// Untar reads a stream of bytes from `archive`, parses it as a tar archive,
+// and unpacks it into the directory at `path`.
+// The archive may be compressed with one of the following algorithms:
+//  identity (uncompressed), gzip, bzip2, xz.
+// FIXME: specify behavior when target path exists vs. doesn't exist.
+func Untar(archive io.Reader, dest string, options *TarOptions) error {
+	if options == nil {
+		options = &TarOptions{}
+	}
+
+	if archive == nil {
+		return fmt.Errorf("Empty archive")
+	}
+
+	if options.Excludes == nil {
+		options.Excludes = []string{}
+	}
+
+	decompressedArchive, err := DecompressStream(archive)
+	if err != nil {
+		return err
+	}
+	defer decompressedArchive.Close()
+
+	tr := tar.NewReader(decompressedArchive)
+	trBuf := pools.BufioReader32KPool.Get(nil)
+	defer pools.BufioReader32KPool.Put(trBuf)
+
+	var dirs []*tar.Header
+
+	// Iterate through the files in the archive.
+loop:
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		// Normalize name, for safety and for a simple is-root check
+		hdr.Name = filepath.Clean(hdr.Name)
+
+		for _, exclude := range options.Excludes {
+			if strings.HasPrefix(hdr.Name, exclude) {
+				continue loop
+			}
+		}
+
+		if !strings.HasSuffix(hdr.Name, "/") {
+			// Not the root directory, ensure that the parent directory exists
+			parent := filepath.Dir(hdr.Name)
+			parentPath := filepath.Join(dest, parent)
+			if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
+				err = os.MkdirAll(parentPath, 0777)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		path := filepath.Join(dest, hdr.Name)
+
+		// If path exits we almost always just want to remove and replace it
+		// The only exception is when it is a directory *and* the file from
+		// the layer is also a directory. Then we want to merge them (i.e.
+		// just apply the metadata from the layer).
+		if fi, err := os.Lstat(path); err == nil {
+			if fi.IsDir() && hdr.Name == "." {
+				continue
+			}
+			if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
+				if err := os.RemoveAll(path); err != nil {
+					return err
+				}
+			}
+		}
+		trBuf.Reset(tr)
+		if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown); err != nil {
+			return err
+		}
+
+		// Directory mtimes must be handled at the end to avoid further
+		// file creation in them to modify the directory mtime
+		if hdr.Typeflag == tar.TypeDir {
+			dirs = append(dirs, hdr)
+		}
+	}
+
+	for _, hdr := range dirs {
+		path := filepath.Join(dest, hdr.Name)
+		ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)}
+		if err := syscall.UtimesNano(path, ts); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// TarUntar is a convenience function which calls Tar and Untar, with
+// the output of one piped into the other. If either Tar or Untar fails,
+// TarUntar aborts and returns the error.
+func TarUntar(src string, dst string) error {
+	log.Debugf("TarUntar(%s %s)", src, dst)
+	archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed})
+	if err != nil {
+		return err
+	}
+	defer archive.Close()
+	return Untar(archive, dst, nil)
+}
+
+// UntarPath is a convenience function which looks for an archive
+// at filesystem path `src`, and unpacks it at `dst`.
+func UntarPath(src, dst string) error {
+	archive, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer archive.Close()
+	if err := Untar(archive, dst, nil); err != nil {
+		return err
+	}
+	return nil
+}
+
+// CopyWithTar creates a tar archive of filesystem path `src`, and
+// unpacks it at filesystem path `dst`.
+// The archive is streamed directly with fixed buffering and no
+// intermediary disk IO.
+//
+func CopyWithTar(src, dst string) error {
+	srcSt, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+	if !srcSt.IsDir() {
+		return CopyFileWithTar(src, dst)
+	}
+	// Create dst, copy src's content into it
+	log.Debugf("Creating dest directory: %s", dst)
+	if err := os.MkdirAll(dst, 0755); err != nil && !os.IsExist(err) {
+		return err
+	}
+	log.Debugf("Calling TarUntar(%s, %s)", src, dst)
+	return TarUntar(src, dst)
+}
+
+// CopyFileWithTar emulates the behavior of the 'cp' command-line
+// for a single file. It copies a regular file from path `src` to
+// path `dst`, and preserves all its metadata.
+//
+// If `dst` ends with a trailing slash '/', the final destination path
+// will be `dst/base(src)`.
+func CopyFileWithTar(src, dst string) (err error) {
+	log.Debugf("CopyFileWithTar(%s, %s)", src, dst)
+	srcSt, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+	if srcSt.IsDir() {
+		return fmt.Errorf("Can't copy a directory")
+	}
+	// Clean up the trailing /
+	if dst[len(dst)-1] == '/' {
+		dst = path.Join(dst, filepath.Base(src))
+	}
+	// Create the holding directory if necessary
+	if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil && !os.IsExist(err) {
+		return err
+	}
+
+	r, w := io.Pipe()
+	errC := promise.Go(func() error {
+		defer w.Close()
+
+		srcF, err := os.Open(src)
+		if err != nil {
+			return err
+		}
+		defer srcF.Close()
+
+		hdr, err := tar.FileInfoHeader(srcSt, "")
+		if err != nil {
+			return err
+		}
+		hdr.Name = filepath.Base(dst)
+		tw := tar.NewWriter(w)
+		defer tw.Close()
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
+		}
+		if _, err := io.Copy(tw, srcF); err != nil {
+			return err
+		}
+		return nil
+	})
+	defer func() {
+		if er := <-errC; err != nil {
+			err = er
+		}
+	}()
+	return Untar(r, filepath.Dir(dst), nil)
+}
+
+// CmdStream executes a command, and returns its stdout as a stream.
+// If the command fails to run or doesn't complete successfully, an error
+// will be returned, including anything written on stderr.
+func CmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, error) {
+	if input != nil {
+		stdin, err := cmd.StdinPipe()
+		if err != nil {
+			return nil, err
+		}
+		// Write stdin if any
+		go func() {
+			io.Copy(stdin, input)
+			stdin.Close()
+		}()
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, err
+	}
+	pipeR, pipeW := io.Pipe()
+	errChan := make(chan []byte)
+	// Collect stderr, we will use it in case of an error
+	go func() {
+		errText, e := ioutil.ReadAll(stderr)
+		if e != nil {
+			errText = []byte("(...couldn't fetch stderr: " + e.Error() + ")")
+		}
+		errChan <- errText
+	}()
+	// Copy stdout to the returned pipe
+	go func() {
+		_, err := io.Copy(pipeW, stdout)
+		if err != nil {
+			pipeW.CloseWithError(err)
+		}
+		errText := <-errChan
+		if err := cmd.Wait(); err != nil {
+			pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errText))
+		} else {
+			pipeW.Close()
+		}
+	}()
+	// Run the command and return the pipe
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+	return pipeR, nil
+}
+
+// NewTempArchive reads the content of src into a temporary file, and returns the contents
+// of that file as an archive. The archive can only be read once - as soon as reading completes,
+// the file will be deleted.
+func NewTempArchive(src Archive, dir string) (*TempArchive, error) {
+	f, err := ioutil.TempFile(dir, "")
+	if err != nil {
+		return nil, err
+	}
+	if _, err := io.Copy(f, src); err != nil {
+		return nil, err
+	}
+	if err = f.Sync(); err != nil {
+		return nil, err
+	}
+	if _, err := f.Seek(0, 0); err != nil {
+		return nil, err
+	}
+	st, err := f.Stat()
+	if err != nil {
+		return nil, err
+	}
+	size := st.Size()
+	return &TempArchive{f, size}, nil
+}
+
+type TempArchive struct {
+	*os.File
+	Size int64 // Pre-computed from Stat().Size() as a convenience
+}
+
+func (archive *TempArchive) Read(data []byte) (int, error) {
+	n, err := archive.File.Read(data)
+	if err != nil {
+		os.Remove(archive.File.Name())
+	}
+	return n, err
+}

archive/archive_test.go

@@ -0,0 +1,244 @@
+package archive
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar"
+)
+
+func TestCmdStreamLargeStderr(t *testing.T) {
+	cmd := exec.Command("/bin/sh", "-c", "dd if=/dev/zero bs=1k count=1000 of=/dev/stderr; echo hello")
+	out, err := CmdStream(cmd, nil)
+	if err != nil {
+		t.Fatalf("Failed to start command: %s", err)
+	}
+	errCh := make(chan error)
+	go func() {
+		_, err := io.Copy(ioutil.Discard, out)
+		errCh <- err
+	}()
+	select {
+	case err := <-errCh:
+		if err != nil {
+			t.Fatalf("Command should not have failed (err=%.100s...)", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatalf("Command did not complete in 5 seconds; probable deadlock")
+	}
+}
+
+func TestCmdStreamBad(t *testing.T) {
+	badCmd := exec.Command("/bin/sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1")
+	out, err := CmdStream(badCmd, nil)
+	if err != nil {
+		t.Fatalf("Failed to start command: %s", err)
+	}
+	if output, err := ioutil.ReadAll(out); err == nil {
+		t.Fatalf("Command should have failed")
+	} else if err.Error() != "exit status 1: error couldn't reverse the phase pulser\n" {
+		t.Fatalf("Wrong error value (%s)", err)
+	} else if s := string(output); s != "hello\n" {
+		t.Fatalf("Command output should be '%s', not '%s'", "hello\\n", output)
+	}
+}
+
+func TestCmdStreamGood(t *testing.T) {
+	cmd := exec.Command("/bin/sh", "-c", "echo hello; exit 0")
+	out, err := CmdStream(cmd, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if output, err := ioutil.ReadAll(out); err != nil {
+		t.Fatalf("Command should not have failed (err=%s)", err)
+	} else if s := string(output); s != "hello\n" {
+		t.Fatalf("Command output should be '%s', not '%s'", "hello\\n", output)
+	}
+}
+
+func tarUntar(t *testing.T, origin string, options *TarOptions) ([]Change, error) {
+	archive, err := TarWithOptions(origin, options)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer archive.Close()
+
+	buf := make([]byte, 10)
+	if _, err := archive.Read(buf); err != nil {
+		return nil, err
+	}
+	wrap := io.MultiReader(bytes.NewReader(buf), archive)
+
+	detectedCompression := DetectCompression(buf)
+	compression := options.Compression
+	if detectedCompression.Extension() != compression.Extension() {
+		return nil, fmt.Errorf("Wrong compression detected. Actual compression: %s, found %s", compression.Extension(), detectedCompression.Extension())
+	}
+
+	tmp, err := ioutil.TempDir("", "docker-test-untar")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(tmp)
+	if err := Untar(wrap, tmp, nil); err != nil {
+		return nil, err
+	}
+	if _, err := os.Stat(tmp); err != nil {
+		return nil, err
+	}
+
+	return ChangesDirs(origin, tmp)
+}
+
+func TestTarUntar(t *testing.T) {
+	origin, err := ioutil.TempDir("", "docker-test-untar-origin")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(origin)
+	if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(path.Join(origin, "2"), []byte("welcome!"), 0700); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(path.Join(origin, "3"), []byte("will be ignored"), 0700); err != nil {
+		t.Fatal(err)
+	}
+
+	for _, c := range []Compression{
+		Uncompressed,
+		Gzip,
+	} {
+		changes, err := tarUntar(t, origin, &TarOptions{
+			Compression: c,
+			Excludes:    []string{"3"},
+		})
+
+		if err != nil {
+			t.Fatalf("Error tar/untar for compression %s: %s", c.Extension(), err)
+		}
+
+		if len(changes) != 1 || changes[0].Path != "/3" {
+			t.Fatalf("Unexpected differences after tarUntar: %v", changes)
+		}
+	}
+}
+
+func TestTarWithOptions(t *testing.T) {
+	origin, err := ioutil.TempDir("", "docker-test-untar-origin")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(origin)
+	if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(path.Join(origin, "2"), []byte("welcome!"), 0700); err != nil {
+		t.Fatal(err)
+	}
+
+	cases := []struct {
+		opts       *TarOptions
+		numChanges int
+	}{
+		{&TarOptions{Includes: []string{"1"}}, 1},
+		{&TarOptions{Excludes: []string{"2"}}, 1},
+	}
+	for _, testCase := range cases {
+		changes, err := tarUntar(t, origin, testCase.opts)
+		if err != nil {
+			t.Fatalf("Error tar/untar when testing inclusion/exclusion: %s", err)
+		}
+		if len(changes) != testCase.numChanges {
+			t.Errorf("Expected %d changes, got %d for %+v:",
+				testCase.numChanges, len(changes), testCase.opts)
+		}
+	}
+}
+
+// Some tar archives such as http://haproxy.1wt.eu/download/1.5/src/devel/haproxy-1.5-dev21.tar.gz
+// use PAX Global Extended Headers.
+// Failing prevents the archives from being uncompressed during ADD
+func TestTypeXGlobalHeaderDoesNotFail(t *testing.T) {
+	hdr := tar.Header{Typeflag: tar.TypeXGlobalHeader}
+	err := createTarFile("pax_global_header", "some_dir", &hdr, nil, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Some tar have both GNU specific (huge uid) and Ustar specific (long name) things.
+// Not supposed to happen (should use PAX instead of Ustar for long name) but it does and it should still work.
+func TestUntarUstarGnuConflict(t *testing.T) {
+	f, err := os.Open("testdata/broken.tar")
+	if err != nil {
+		t.Fatal(err)
+	}
+	found := false
+	tr := tar.NewReader(f)
+	// Iterate through the files in the archive.
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		if hdr.Name == "root/.cpanm/work/1395823785.24209/Plack-1.0030/blib/man3/Plack::Middleware::LighttpdScriptNameFix.3pm" {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Fatalf("%s not found in the archive", "root/.cpanm/work/1395823785.24209/Plack-1.0030/blib/man3/Plack::Middleware::LighttpdScriptNameFix.3pm")
+	}
+}
+
+func prepareUntarSourceDirectory(numberOfFiles int, targetPath string) (int, error) {
+	fileData := []byte("fooo")
+	for n := 0; n < numberOfFiles; n++ {
+		fileName := fmt.Sprintf("file-%d", n)
+		if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil {
+			return 0, err
+		}
+	}
+	totalSize := numberOfFiles * len(fileData)
+	return totalSize, nil
+}
+
+func BenchmarkTarUntar(b *testing.B) {
+	origin, err := ioutil.TempDir("", "docker-test-untar-origin")
+	if err != nil {
+		b.Fatal(err)
+	}
+	tempDir, err := ioutil.TempDir("", "docker-test-untar-destination")
+	if err != nil {
+		b.Fatal(err)
+	}
+	target := path.Join(tempDir, "dest")
+	n, err := prepareUntarSourceDirectory(100, origin)
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	b.SetBytes(int64(n))
+	defer os.RemoveAll(origin)
+	defer os.RemoveAll(tempDir)
+	for n := 0; n < b.N; n++ {
+		err := TarUntar(origin, target)
+		if err != nil {
+			b.Fatal(err)
+		}
+		os.RemoveAll(target)
+	}
+}

archive/changes.go

@@ -0,0 +1,411 @@
+package archive
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar"
+
+	"github.com/docker/docker/pkg/log"
+	"github.com/docker/docker/pkg/pools"
+	"github.com/docker/docker/pkg/system"
+)
+
+type ChangeType int
+
+const (
+	ChangeModify = iota
+	ChangeAdd
+	ChangeDelete
+)
+
+type Change struct {
+	Path string
+	Kind ChangeType
+}
+
+func (change *Change) String() string {
+	var kind string
+	switch change.Kind {
+	case ChangeModify:
+		kind = "C"
+	case ChangeAdd:
+		kind = "A"
+	case ChangeDelete:
+		kind = "D"
+	}
+	return fmt.Sprintf("%s %s", kind, change.Path)
+}
+
+// Gnu tar and the go tar writer don't have sub-second mtime
+// precision, which is problematic when we apply changes via tar
+// files, we handle this by comparing for exact times, *or* same
+// second count and either a or b having exactly 0 nanoseconds
+func sameFsTime(a, b time.Time) bool {
+	return a == b ||
+		(a.Unix() == b.Unix() &&
+			(a.Nanosecond() == 0 || b.Nanosecond() == 0))
+}
+
+func sameFsTimeSpec(a, b syscall.Timespec) bool {
+	return a.Sec == b.Sec &&
+		(a.Nsec == b.Nsec || a.Nsec == 0 || b.Nsec == 0)
+}
+
+// Changes walks the path rw and determines changes for the files in the path,
+// with respect to the parent layers
+func Changes(layers []string, rw string) ([]Change, error) {
+	var changes []Change
+	err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Rebase path
+		path, err = filepath.Rel(rw, path)
+		if err != nil {
+			return err
+		}
+		path = filepath.Join("/", path)
+
+		// Skip root
+		if path == "/" {
+			return nil
+		}
+
+		// Skip AUFS metadata
+		if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched {
+			return err
+		}
+
+		change := Change{
+			Path: path,
+		}
+
+		// Find out what kind of modification happened
+		file := filepath.Base(path)
+		// If there is a whiteout, then the file was removed
+		if strings.HasPrefix(file, ".wh.") {
+			originalFile := file[len(".wh."):]
+			change.Path = filepath.Join(filepath.Dir(path), originalFile)
+			change.Kind = ChangeDelete
+		} else {
+			// Otherwise, the file was added
+			change.Kind = ChangeAdd
+
+			// ...Unless it already existed in a top layer, in which case, it's a modification
+			for _, layer := range layers {
+				stat, err := os.Stat(filepath.Join(layer, path))
+				if err != nil && !os.IsNotExist(err) {
+					return err
+				}
+				if err == nil {
+					// The file existed in the top layer, so that's a modification
+
+					// However, if it's a directory, maybe it wasn't actually modified.
+					// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
+					if stat.IsDir() && f.IsDir() {
+						if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) {
+							// Both directories are the same, don't record the change
+							return nil
+						}
+					}
+					change.Kind = ChangeModify
+					break
+				}
+			}
+		}
+
+		// Record change
+		changes = append(changes, change)
+		return nil
+	})
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+	return changes, nil
+}
+
+type FileInfo struct {
+	parent     *FileInfo
+	name       string
+	stat       syscall.Stat_t
+	children   map[string]*FileInfo
+	capability []byte
+	added      bool
+}
+
+func (root *FileInfo) LookUp(path string) *FileInfo {
+	parent := root
+	if path == "/" {
+		return root
+	}
+
+	pathElements := strings.Split(path, "/")
+	for _, elem := range pathElements {
+		if elem != "" {
+			child := parent.children[elem]
+			if child == nil {
+				return nil
+			}
+			parent = child
+		}
+	}
+	return parent
+}
+
+func (info *FileInfo) path() string {
+	if info.parent == nil {
+		return "/"
+	}
+	return filepath.Join(info.parent.path(), info.name)
+}
+
+func (info *FileInfo) isDir() bool {
+	return info.parent == nil || info.stat.Mode&syscall.S_IFDIR == syscall.S_IFDIR
+}
+
+func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
+
+	sizeAtEntry := len(*changes)
+
+	if oldInfo == nil {
+		// add
+		change := Change{
+			Path: info.path(),
+			Kind: ChangeAdd,
+		}
+		*changes = append(*changes, change)
+		info.added = true
+	}
+
+	// We make a copy so we can modify it to detect additions
+	// also, we only recurse on the old dir if the new info is a directory
+	// otherwise any previous delete/change is considered recursive
+	oldChildren := make(map[string]*FileInfo)
+	if oldInfo != nil && info.isDir() {
+		for k, v := range oldInfo.children {
+			oldChildren[k] = v
+		}
+	}
+
+	for name, newChild := range info.children {
+		oldChild, _ := oldChildren[name]
+		if oldChild != nil {
+			// change?
+			oldStat := &oldChild.stat
+			newStat := &newChild.stat
+			// Note: We can't compare inode or ctime or blocksize here, because these change
+			// when copying a file into a container. However, that is not generally a problem
+			// because any content change will change mtime, and any status change should
+			// be visible when actually comparing the stat fields. The only time this
+			// breaks down is if some code intentionally hides a change by setting
+			// back mtime
+			if oldStat.Mode != newStat.Mode ||
+				oldStat.Uid != newStat.Uid ||
+				oldStat.Gid != newStat.Gid ||
+				oldStat.Rdev != newStat.Rdev ||
+				// Don't look at size for dirs, its not a good measure of change
+				(oldStat.Size != newStat.Size && oldStat.Mode&syscall.S_IFDIR != syscall.S_IFDIR) ||
+				!sameFsTimeSpec(system.GetLastModification(oldStat), system.GetLastModification(newStat)) ||
+				bytes.Compare(oldChild.capability, newChild.capability) != 0 {
+				change := Change{
+					Path: newChild.path(),
+					Kind: ChangeModify,
+				}
+				*changes = append(*changes, change)
+				newChild.added = true
+			}
+
+			// Remove from copy so we can detect deletions
+			delete(oldChildren, name)
+		}
+
+		newChild.addChanges(oldChild, changes)
+	}
+	for _, oldChild := range oldChildren {
+		// delete
+		change := Change{
+			Path: oldChild.path(),
+			Kind: ChangeDelete,
+		}
+		*changes = append(*changes, change)
+	}
+
+	// If there were changes inside this directory, we need to add it, even if the directory
+	// itself wasn't changed. This is needed to properly save and restore filesystem permissions.
+	if len(*changes) > sizeAtEntry && info.isDir() && !info.added && info.path() != "/" {
+		change := Change{
+			Path: info.path(),
+			Kind: ChangeModify,
+		}
+		// Let's insert the directory entry before the recently added entries located inside this dir
+		*changes = append(*changes, change) // just to resize the slice, will be overwritten
+		copy((*changes)[sizeAtEntry+1:], (*changes)[sizeAtEntry:])
+		(*changes)[sizeAtEntry] = change
+	}
+
+}
+
+func (info *FileInfo) Changes(oldInfo *FileInfo) []Change {
+	var changes []Change
+
+	info.addChanges(oldInfo, &changes)
+
+	return changes
+}
+
+func newRootFileInfo() *FileInfo {
+	root := &FileInfo{
+		name:     "/",
+		children: make(map[string]*FileInfo),
+	}
+	return root
+}
+
+func collectFileInfo(sourceDir string) (*FileInfo, error) {
+	root := newRootFileInfo()
+
+	err := filepath.Walk(sourceDir, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Rebase path
+		relPath, err := filepath.Rel(sourceDir, path)
+		if err != nil {
+			return err
+		}
+		relPath = filepath.Join("/", relPath)
+
+		if relPath == "/" {
+			return nil
+		}
+
+		parent := root.LookUp(filepath.Dir(relPath))
+		if parent == nil {
+			return fmt.Errorf("collectFileInfo: Unexpectedly no parent for %s", relPath)
+		}
+
+		info := &FileInfo{
+			name:     filepath.Base(relPath),
+			children: make(map[string]*FileInfo),
+			parent:   parent,
+		}
+
+		if err := syscall.Lstat(path, &info.stat); err != nil {
+			return err
+		}
+
+		info.capability, _ = system.Lgetxattr(path, "security.capability")
+
+		parent.children[info.name] = info
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return root, nil
+}
+
+// ChangesDirs compares two directories and generates an array of Change objects describing the changes.
+// If oldDir is "", then all files in newDir will be Add-Changes.
+func ChangesDirs(newDir, oldDir string) ([]Change, error) {
+	var (
+		oldRoot, newRoot *FileInfo
+		err1, err2       error
+		errs             = make(chan error, 2)
+	)
+	go func() {
+		if oldDir != "" {
+			oldRoot, err1 = collectFileInfo(oldDir)
+		}
+		errs <- err1
+	}()
+	go func() {
+		newRoot, err2 = collectFileInfo(newDir)
+		errs <- err2
+	}()
+	for i := 0; i < 2; i++ {
+		if err := <-errs; err != nil {
+			return nil, err
+		}
+	}
+
+	return newRoot.Changes(oldRoot), nil
+}
+
+// ChangesSize calculates the size in bytes of the provided changes, based on newDir.
+func ChangesSize(newDir string, changes []Change) int64 {
+	var size int64
+	for _, change := range changes {
+		if change.Kind == ChangeModify || change.Kind == ChangeAdd {
+			file := filepath.Join(newDir, change.Path)
+			fileInfo, _ := os.Lstat(file)
+			if fileInfo != nil && !fileInfo.IsDir() {
+				size += fileInfo.Size()
+			}
+		}
+	}
+	return size
+}
+
+func major(device uint64) uint64 {
+	return (device >> 8) & 0xfff
+}
+
+func minor(device uint64) uint64 {
+	return (device & 0xff) | ((device >> 12) & 0xfff00)
+}
+
+// ExportChanges produces an Archive from the provided changes, relative to dir.
+func ExportChanges(dir string, changes []Change) (Archive, error) {
+	reader, writer := io.Pipe()
+	tw := tar.NewWriter(writer)
+
+	go func() {
+		twBuf := pools.BufioWriter32KPool.Get(nil)
+		defer pools.BufioWriter32KPool.Put(twBuf)
+		// In general we log errors here but ignore them because
+		// during e.g. a diff operation the container can continue
+		// mutating the filesystem and we can see transient errors
+		// from this
+		for _, change := range changes {
+			if change.Kind == ChangeDelete {
+				whiteOutDir := filepath.Dir(change.Path)
+				whiteOutBase := filepath.Base(change.Path)
+				whiteOut := filepath.Join(whiteOutDir, ".wh."+whiteOutBase)
+				timestamp := time.Now()
+				hdr := &tar.Header{
+					Name:       whiteOut[1:],
+					Size:       0,
+					ModTime:    timestamp,
+					AccessTime: timestamp,
+					ChangeTime: timestamp,
+				}
+				if err := tw.WriteHeader(hdr); err != nil {
+					log.Debugf("Can't write whiteout header: %s", err)
+				}
+			} else {
+				path := filepath.Join(dir, change.Path)
+				if err := addTarFile(path, change.Path[1:], tw, twBuf); err != nil {
+					log.Debugf("Can't add file %s to tar: %s", path, err)
+				}
+			}
+		}
+
+		// Make sure to check the error on Close.
+		if err := tw.Close(); err != nil {
+			log.Debugf("Can't close layer: %s", err)
+		}
+		writer.Close()
+	}()
+	return reader, nil
+}

archive/changes_test.go

@@ -0,0 +1,301 @@
+package archive
+
+import (
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"sort"
+	"testing"
+	"time"
+)
+
+func max(x, y int) int {
+	if x >= y {
+		return x
+	}
+	return y
+}
+
+func copyDir(src, dst string) error {
+	cmd := exec.Command("cp", "-a", src, dst)
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Helper to sort []Change by path
+type byPath struct{ changes []Change }
+
+func (b byPath) Less(i, j int) bool { return b.changes[i].Path < b.changes[j].Path }
+func (b byPath) Len() int           { return len(b.changes) }
+func (b byPath) Swap(i, j int)      { b.changes[i], b.changes[j] = b.changes[j], b.changes[i] }
+
+type FileType uint32
+
+const (
+	Regular FileType = iota
+	Dir
+	Symlink
+)
+
+type FileData struct {
+	filetype    FileType
+	path        string
+	contents    string
+	permissions os.FileMode
+}
+
+func createSampleDir(t *testing.T, root string) {
+	files := []FileData{
+		{Regular, "file1", "file1\n", 0600},
+		{Regular, "file2", "file2\n", 0666},
+		{Regular, "file3", "file3\n", 0404},
+		{Regular, "file4", "file4\n", 0600},
+		{Regular, "file5", "file5\n", 0600},
+		{Regular, "file6", "file6\n", 0600},
+		{Regular, "file7", "file7\n", 0600},
+		{Dir, "dir1", "", 0740},
+		{Regular, "dir1/file1-1", "file1-1\n", 01444},
+		{Regular, "dir1/file1-2", "file1-2\n", 0666},
+		{Dir, "dir2", "", 0700},
+		{Regular, "dir2/file2-1", "file2-1\n", 0666},
+		{Regular, "dir2/file2-2", "file2-2\n", 0666},
+		{Dir, "dir3", "", 0700},
+		{Regular, "dir3/file3-1", "file3-1\n", 0666},
+		{Regular, "dir3/file3-2", "file3-2\n", 0666},
+		{Dir, "dir4", "", 0700},
+		{Regular, "dir4/file3-1", "file4-1\n", 0666},
+		{Regular, "dir4/file3-2", "file4-2\n", 0666},
+		{Symlink, "symlink1", "target1", 0666},
+		{Symlink, "symlink2", "target2", 0666},
+	}
+
+	now := time.Now()
+	for _, info := range files {
+		p := path.Join(root, info.path)
+		if info.filetype == Dir {
+			if err := os.MkdirAll(p, info.permissions); err != nil {
+				t.Fatal(err)
+			}
+		} else if info.filetype == Regular {
+			if err := ioutil.WriteFile(p, []byte(info.contents), info.permissions); err != nil {
+				t.Fatal(err)
+			}
+		} else if info.filetype == Symlink {
+			if err := os.Symlink(info.contents, p); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		if info.filetype != Symlink {
+			// Set a consistent ctime, atime for all files and dirs
+			if err := os.Chtimes(p, now, now); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+}
+
+// Create an directory, copy it, make sure we report no changes between the two
+func TestChangesDirsEmpty(t *testing.T) {
+	src, err := ioutil.TempDir("", "docker-changes-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	createSampleDir(t, src)
+	dst := src + "-copy"
+	if err := copyDir(src, dst); err != nil {
+		t.Fatal(err)
+	}
+	changes, err := ChangesDirs(dst, src)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(changes) != 0 {
+		t.Fatalf("Reported changes for identical dirs: %v", changes)
+	}
+	os.RemoveAll(src)
+	os.RemoveAll(dst)
+}
+
+func mutateSampleDir(t *testing.T, root string) {
+	// Remove a regular file
+	if err := os.RemoveAll(path.Join(root, "file1")); err != nil {
+		t.Fatal(err)
+	}
+
+	// Remove a directory
+	if err := os.RemoveAll(path.Join(root, "dir1")); err != nil {
+		t.Fatal(err)
+	}
+
+	// Remove a symlink
+	if err := os.RemoveAll(path.Join(root, "symlink1")); err != nil {
+		t.Fatal(err)
+	}
+
+	// Rewrite a file
+	if err := ioutil.WriteFile(path.Join(root, "file2"), []byte("fileNN\n"), 0777); err != nil {
+		t.Fatal(err)
+	}
+
+	// Replace a file
+	if err := os.RemoveAll(path.Join(root, "file3")); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(path.Join(root, "file3"), []byte("fileMM\n"), 0404); err != nil {
+		t.Fatal(err)
+	}
+
+	// Touch file
+	if err := os.Chtimes(path.Join(root, "file4"), time.Now().Add(time.Second), time.Now().Add(time.Second)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Replace file with dir
+	if err := os.RemoveAll(path.Join(root, "file5")); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll(path.Join(root, "file5"), 0666); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create new file
+	if err := ioutil.WriteFile(path.Join(root, "filenew"), []byte("filenew\n"), 0777); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create new dir
+	if err := os.MkdirAll(path.Join(root, "dirnew"), 0766); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a new symlink
+	if err := os.Symlink("targetnew", path.Join(root, "symlinknew")); err != nil {
+		t.Fatal(err)
+	}
+
+	// Change a symlink
+	if err := os.RemoveAll(path.Join(root, "symlink2")); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Symlink("target2change", path.Join(root, "symlink2")); err != nil {
+		t.Fatal(err)
+	}
+
+	// Replace dir with file
+	if err := os.RemoveAll(path.Join(root, "dir2")); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(path.Join(root, "dir2"), []byte("dir2\n"), 0777); err != nil {
+		t.Fatal(err)
+	}
+
+	// Touch dir
+	if err := os.Chtimes(path.Join(root, "dir3"), time.Now().Add(time.Second), time.Now().Add(time.Second)); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestChangesDirsMutated(t *testing.T) {
+	src, err := ioutil.TempDir("", "docker-changes-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	createSampleDir(t, src)
+	dst := src + "-copy"
+	if err := copyDir(src, dst); err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(src)
+	defer os.RemoveAll(dst)
+
+	mutateSampleDir(t, dst)
+
+	changes, err := ChangesDirs(dst, src)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sort.Sort(byPath{changes})
+
+	expectedChanges := []Change{
+		{"/dir1", ChangeDelete},
+		{"/dir2", ChangeModify},
+		{"/dir3", ChangeModify},
+		{"/dirnew", ChangeAdd},
+		{"/file1", ChangeDelete},
+		{"/file2", ChangeModify},
+		{"/file3", ChangeModify},
+		{"/file4", ChangeModify},
+		{"/file5", ChangeModify},
+		{"/filenew", ChangeAdd},
+		{"/symlink1", ChangeDelete},
+		{"/symlink2", ChangeModify},
+		{"/symlinknew", ChangeAdd},
+	}
+
+	for i := 0; i < max(len(changes), len(expectedChanges)); i++ {
+		if i >= len(expectedChanges) {
+			t.Fatalf("unexpected change %s\n", changes[i].String())
+		}
+		if i >= len(changes) {
+			t.Fatalf("no change for expected change %s\n", expectedChanges[i].String())
+		}
+		if changes[i].Path == expectedChanges[i].Path {
+			if changes[i] != expectedChanges[i] {
+				t.Fatalf("Wrong change for %s, expected %s, got %s\n", changes[i].Path, changes[i].String(), expectedChanges[i].String())
+			}
+		} else if changes[i].Path < expectedChanges[i].Path {
+			t.Fatalf("unexpected change %s\n", changes[i].String())
+		} else {
+			t.Fatalf("no change for expected change %s != %s\n", expectedChanges[i].String(), changes[i].String())
+		}
+	}
+}
+
+func TestApplyLayer(t *testing.T) {
+	src, err := ioutil.TempDir("", "docker-changes-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	createSampleDir(t, src)
+	defer os.RemoveAll(src)
+	dst := src + "-copy"
+	if err := copyDir(src, dst); err != nil {
+		t.Fatal(err)
+	}
+	mutateSampleDir(t, dst)
+	defer os.RemoveAll(dst)
+
+	changes, err := ChangesDirs(dst, src)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer, err := ExportChanges(dst, changes)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layerCopy, err := NewTempArchive(layer, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ApplyLayer(src, layerCopy); err != nil {
+		t.Fatal(err)
+	}
+
+	changes2, err := ChangesDirs(src, dst)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(changes2) != 0 {
+		t.Fatalf("Unexpected differences after reapplying mutation: %v", changes2)
+	}
+}

archive/diff.go

@@ -0,0 +1,156 @@
+package archive
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar"
+
+	"github.com/docker/docker/pkg/pools"
+)
+
+// Linux device nodes are a bit weird due to backwards compat with 16 bit device nodes.
+// They are, from low to high: the lower 8 bits of the minor, then 12 bits of the major,
+// then the top 12 bits of the minor
+func mkdev(major int64, minor int64) uint32 {
+	return uint32(((minor & 0xfff00) << 12) | ((major & 0xfff) << 8) | (minor & 0xff))
+}
+
+// ApplyLayer parses a diff in the standard layer format from `layer`, and
+// applies it to the directory `dest`.
+func ApplyLayer(dest string, layer ArchiveReader) error {
+	// We need to be able to set any perms
+	oldmask := syscall.Umask(0)
+	defer syscall.Umask(oldmask)
+
+	layer, err := DecompressStream(layer)
+	if err != nil {
+		return err
+	}
+
+	tr := tar.NewReader(layer)
+	trBuf := pools.BufioReader32KPool.Get(tr)
+	defer pools.BufioReader32KPool.Put(trBuf)
+
+	var dirs []*tar.Header
+
+	aufsTempdir := ""
+	aufsHardlinks := make(map[string]*tar.Header)
+
+	// Iterate through the files in the archive.
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		// Normalize name, for safety and for a simple is-root check
+		hdr.Name = filepath.Clean(hdr.Name)
+
+		if !strings.HasSuffix(hdr.Name, "/") {
+			// Not the root directory, ensure that the parent directory exists.
+			// This happened in some tests where an image had a tarfile without any
+			// parent directories.
+			parent := filepath.Dir(hdr.Name)
+			parentPath := filepath.Join(dest, parent)
+			if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
+				err = os.MkdirAll(parentPath, 0600)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		// Skip AUFS metadata dirs
+		if strings.HasPrefix(hdr.Name, ".wh..wh.") {
+			// Regular files inside /.wh..wh.plnk can be used as hardlink targets
+			// We don't want this directory, but we need the files in them so that
+			// such hardlinks can be resolved.
+			if strings.HasPrefix(hdr.Name, ".wh..wh.plnk") && hdr.Typeflag == tar.TypeReg {
+				basename := filepath.Base(hdr.Name)
+				aufsHardlinks[basename] = hdr
+				if aufsTempdir == "" {
+					if aufsTempdir, err = ioutil.TempDir("", "dockerplnk"); err != nil {
+						return err
+					}
+					defer os.RemoveAll(aufsTempdir)
+				}
+				if err := createTarFile(filepath.Join(aufsTempdir, basename), dest, hdr, tr, true); err != nil {
+					return err
+				}
+			}
+			continue
+		}
+
+		path := filepath.Join(dest, hdr.Name)
+		base := filepath.Base(path)
+		if strings.HasPrefix(base, ".wh.") {
+			originalBase := base[len(".wh."):]
+			originalPath := filepath.Join(filepath.Dir(path), originalBase)
+			if err := os.RemoveAll(originalPath); err != nil {
+				return err
+			}
+		} else {
+			// If path exits we almost always just want to remove and replace it.
+			// The only exception is when it is a directory *and* the file from
+			// the layer is also a directory. Then we want to merge them (i.e.
+			// just apply the metadata from the layer).
+			if fi, err := os.Lstat(path); err == nil {
+				if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
+					if err := os.RemoveAll(path); err != nil {
+						return err
+					}
+				}
+			}
+
+			trBuf.Reset(tr)
+			srcData := io.Reader(trBuf)
+			srcHdr := hdr
+
+			// Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so
+			// we manually retarget these into the temporary files we extracted them into
+			if hdr.Typeflag == tar.TypeLink && strings.HasPrefix(filepath.Clean(hdr.Linkname), ".wh..wh.plnk") {
+				linkBasename := filepath.Base(hdr.Linkname)
+				srcHdr = aufsHardlinks[linkBasename]
+				if srcHdr == nil {
+					return fmt.Errorf("Invalid aufs hardlink")
+				}
+				tmpFile, err := os.Open(filepath.Join(aufsTempdir, linkBasename))
+				if err != nil {
+					return err
+				}
+				defer tmpFile.Close()
+				srcData = tmpFile
+			}
+
+			if err := createTarFile(path, dest, srcHdr, srcData, true); err != nil {
+				return err
+			}
+
+			// Directory mtimes must be handled at the end to avoid further
+			// file creation in them to modify the directory mtime
+			if hdr.Typeflag == tar.TypeDir {
+				dirs = append(dirs, hdr)
+			}
+		}
+	}
+
+	for _, hdr := range dirs {
+		path := filepath.Join(dest, hdr.Name)
+		ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)}
+		if err := syscall.UtimesNano(path, ts); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

archive/time_linux.go

@@ -0,0 +1,16 @@
+package archive
+
+import (
+	"syscall"
+	"time"
+)
+
+func timeToTimespec(time time.Time) (ts syscall.Timespec) {
+	if time.IsZero() {
+		// Return UTIME_OMIT special value
+		ts.Sec = 0
+		ts.Nsec = ((1 << 30) - 2)
+		return
+	}
+	return syscall.NsecToTimespec(time.UnixNano())
+}

archive/time_unsupported.go

@@ -0,0 +1,16 @@
+// +build !linux
+
+package archive
+
+import (
+	"syscall"
+	"time"
+)
+
+func timeToTimespec(time time.Time) (ts syscall.Timespec) {
+	nsec := int64(0)
+	if !time.IsZero() {
+		nsec = time.UnixNano()
+	}
+	return syscall.NsecToTimespec(nsec)
+}

archive/wrap.go

@@ -0,0 +1,59 @@
+package archive
+
+import (
+	"bytes"
+	"github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar"
+	"io/ioutil"
+)
+
+// Generate generates a new archive from the content provided
+// as input.
+//
+// `files` is a sequence of path/content pairs. A new file is
+// added to the archive for each pair.
+// If the last pair is incomplete, the file is created with an
+// empty content. For example:
+//
+// Generate("foo.txt", "hello world", "emptyfile")
+//
+// The above call will return an archive with 2 files:
+//  * ./foo.txt with content "hello world"
+//  * ./empty with empty content
+//
+// FIXME: stream content instead of buffering
+// FIXME: specify permissions and other archive metadata
+func Generate(input ...string) (Archive, error) {
+	files := parseStringPairs(input...)
+	buf := new(bytes.Buffer)
+	tw := tar.NewWriter(buf)
+	for _, file := range files {
+		name, content := file[0], file[1]
+		hdr := &tar.Header{
+			Name: name,
+			Size: int64(len(content)),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return nil, err
+		}
+		if _, err := tw.Write([]byte(content)); err != nil {
+			return nil, err
+		}
+	}
+	if err := tw.Close(); err != nil {
+		return nil, err
+	}
+	return ioutil.NopCloser(buf), nil
+}
+
+func parseStringPairs(input ...string) (output [][2]string) {
+	output = make([][2]string, 0, len(input)/2+1)
+	for i := 0; i < len(input); i += 2 {
+		var pair [2]string
+		pair[0] = input[i]
+		if i+1 < len(input) {
+			pair[1] = input[i+1]
+		}
+		output = append(output, pair)
+	}
+	return
+}