Add archive package

Archive package handles generating and applying diff tar streams
based on the OCI diff tar specification.

Signed-off-by: Derek McGowan <derek@mcgstyle.net> (github: dmcgowan)

archive/tar.go

@@ -0,0 +1,497 @@
+package archive
+
+import (
+	"archive/tar"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/containerd/fs"
+	"github.com/pkg/errors"
+)
+
+var (
+	bufferPool = &sync.Pool{
+		New: func() interface{} {
+			return make([]byte, 32*1024)
+		},
+	}
+
+	breakoutError = errors.New("file name outside of root")
+)
+
+// TarFromChanges returns a tar stream of the computed filesystem
+// difference between the provided directories.
+//
+// Produces a tar using OCI style file markers for deletions. Deleted
+// files will be prepended with the prefix ".wh.". This style is
+// based off AUFS whiteouts.
+// See https://github.com/opencontainers/image-spec/blob/master/layer.md
+func DiffTarStream(ctx context.Context, a, b string) io.ReadCloser {
+	r, w := io.Pipe()
+
+	go func() {
+		var err error
+		cw := newChangeWriter(w, b)
+		if err = fs.Changes(ctx, a, b, cw.HandleChange); err != nil {
+			err = errors.Wrap(err, "failed to create diff tar stream")
+		} else {
+			err = cw.Close()
+		}
+		if err = w.CloseWithError(err); err != nil {
+			logrus.Errorf("Error closing tar pipe: %v", err)
+		}
+	}()
+
+	return r
+}
+
+const (
+	// whiteoutPrefix prefix means file is a whiteout. If this is followed by a
+	// filename this means that file has been removed from the base layer.
+	// See https://github.com/opencontainers/image-spec/blob/master/layer.md#whiteouts
+	whiteoutPrefix = ".wh."
+
+	// whiteoutMetaPrefix prefix means whiteout has a special meaning and is not
+	// for removing an actual file. Normally these files are excluded from exported
+	// archives.
+	whiteoutMetaPrefix = whiteoutPrefix + whiteoutPrefix
+
+	// whiteoutLinkDir is a directory AUFS uses for storing hardlink links to other
+	// layers. Normally these should not go into exported archives and all changed
+	// hardlinks should be copied to the top layer.
+	whiteoutLinkDir = whiteoutMetaPrefix + "plnk"
+
+	// whiteoutOpaqueDir file means directory has been made opaque - meaning
+	// readdir calls to this directory do not follow to lower layers.
+	whiteoutOpaqueDir = whiteoutMetaPrefix + ".opq"
+)
+
+// ApplyDiffTar applies a tar stream of an OCI style diff tar.
+// See https://github.com/opencontainers/image-spec/blob/master/layer.md#applying-changesets
+func ApplyDiffTar(ctx context.Context, root string, r io.Reader) (int64, error) {
+	root = filepath.Clean(root)
+	fn := prepareApply()
+	defer fn()
+
+	var (
+		tr   = tar.NewReader(r)
+		size int64
+		dirs []*tar.Header
+
+		// Used for handling opaque directory markers which
+		// may occur out of order
+		unpackedPaths = make(map[string]struct{})
+
+		// Used for aufs plink directory
+		aufsTempdir   = ""
+		aufsHardlinks = make(map[string]*tar.Header)
+	)
+
+	// Iterate through the files in the archive.
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return 0, err
+		}
+
+		size += hdr.Size
+
+		// Normalize name, for safety and for a simple is-root check
+		hdr.Name = filepath.Clean(hdr.Name)
+
+		if skipFile(hdr) {
+			continue
+		}
+
+		// Note as these operations are platform specific, so must the slash be.
+		if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) {
+			// Not the root directory, ensure that the parent directory exists.
+			// This happened in some tests where an image had a tarfile without any
+			// parent directories.
+			parent := filepath.Dir(hdr.Name)
+			parentPath := filepath.Join(root, parent)
+
+			if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
+				err = mkdirAll(parentPath, 0600)
+				if err != nil {
+					return 0, err
+				}
+			}
+		}
+
+		// Skip AUFS metadata dirs
+		if strings.HasPrefix(hdr.Name, whiteoutMetaPrefix) {
+			// Regular files inside /.wh..wh.plnk can be used as hardlink targets
+			// We don't want this directory, but we need the files in them so that
+			// such hardlinks can be resolved.
+			if strings.HasPrefix(hdr.Name, whiteoutLinkDir) && hdr.Typeflag == tar.TypeReg {
+				basename := filepath.Base(hdr.Name)
+				aufsHardlinks[basename] = hdr
+				if aufsTempdir == "" {
+					if aufsTempdir, err = ioutil.TempDir("", "dockerplnk"); err != nil {
+						return 0, err
+					}
+					defer os.RemoveAll(aufsTempdir)
+				}
+				if err := createTarFile(filepath.Join(aufsTempdir, basename), root, hdr, tr); err != nil {
+					return 0, err
+				}
+			}
+
+			if hdr.Name != whiteoutOpaqueDir {
+				continue
+			}
+		}
+
+		path := filepath.Join(root, hdr.Name)
+		rel, err := filepath.Rel(root, path)
+		if err != nil {
+			return 0, err
+		}
+
+		// Note as these operations are platform specific, so must the slash be.
+		if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
+			return 0, errors.Wrapf(breakoutError, "%q is outside of %q", hdr.Name, root)
+		}
+		base := filepath.Base(path)
+
+		if strings.HasPrefix(base, whiteoutPrefix) {
+			dir := filepath.Dir(path)
+			if base == whiteoutOpaqueDir {
+				_, err := os.Lstat(dir)
+				if err != nil {
+					return 0, err
+				}
+				err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+					if err != nil {
+						if os.IsNotExist(err) {
+							err = nil // parent was deleted
+						}
+						return err
+					}
+					if path == dir {
+						return nil
+					}
+					if _, exists := unpackedPaths[path]; !exists {
+						err := os.RemoveAll(path)
+						return err
+					}
+					return nil
+				})
+				if err != nil {
+					return 0, err
+				}
+				continue
+			}
+
+			originalBase := base[len(whiteoutPrefix):]
+			originalPath := filepath.Join(dir, originalBase)
+			if err := os.RemoveAll(originalPath); err != nil {
+				return 0, err
+			}
+			continue
+		}
+		// If path exits we almost always just want to remove and replace it.
+		// The only exception is when it is a directory *and* the file from
+		// the layer is also a directory. Then we want to merge them (i.e.
+		// just apply the metadata from the layer).
+		if fi, err := os.Lstat(path); err == nil {
+			if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
+				if err := os.RemoveAll(path); err != nil {
+					return 0, err
+				}
+			}
+		}
+
+		srcData := io.Reader(tr)
+		srcHdr := hdr
+
+		// Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so
+		// we manually retarget these into the temporary files we extracted them into
+		if hdr.Typeflag == tar.TypeLink && strings.HasPrefix(filepath.Clean(hdr.Linkname), whiteoutLinkDir) {
+			linkBasename := filepath.Base(hdr.Linkname)
+			srcHdr = aufsHardlinks[linkBasename]
+			if srcHdr == nil {
+				return 0, fmt.Errorf("Invalid aufs hardlink")
+			}
+			tmpFile, err := os.Open(filepath.Join(aufsTempdir, linkBasename))
+			if err != nil {
+				return 0, err
+			}
+			defer tmpFile.Close()
+			srcData = tmpFile
+		}
+
+		if err := createTarFile(path, root, srcHdr, srcData); err != nil {
+			return 0, err
+		}
+
+		// Directory mtimes must be handled at the end to avoid further
+		// file creation in them to modify the directory mtime
+		if hdr.Typeflag == tar.TypeDir {
+			dirs = append(dirs, hdr)
+		}
+		unpackedPaths[path] = struct{}{}
+	}
+
+	for _, hdr := range dirs {
+		path := filepath.Join(root, hdr.Name)
+		if err := chtimes(path, boundTime(latestTime(hdr.AccessTime, hdr.ModTime)), boundTime(hdr.ModTime)); err != nil {
+			return 0, err
+		}
+	}
+
+	return size, nil
+}
+
+type changeWriter struct {
+	tw         *tar.Writer
+	source     string
+	whiteoutT  time.Time
+	inodeCache map[uint64]string
+}
+
+func newChangeWriter(w io.Writer, source string) *changeWriter {
+	return &changeWriter{
+		tw:         tar.NewWriter(w),
+		source:     source,
+		whiteoutT:  time.Now(),
+		inodeCache: map[uint64]string{},
+	}
+}
+
+func (cw *changeWriter) HandleChange(k fs.ChangeKind, p string, f os.FileInfo, err error) error {
+	if err != nil {
+		return err
+	}
+	if k == fs.ChangeKindDelete {
+		whiteOutDir := filepath.Dir(p)
+		whiteOutBase := filepath.Base(p)
+		whiteOut := filepath.Join(whiteOutDir, whiteoutPrefix+whiteOutBase)
+		hdr := &tar.Header{
+			Name:       whiteOut[1:],
+			Size:       0,
+			ModTime:    cw.whiteoutT,
+			AccessTime: cw.whiteoutT,
+			ChangeTime: cw.whiteoutT,
+		}
+		if err := cw.tw.WriteHeader(hdr); err != nil {
+			errors.Wrap(err, "failed to write whiteout header")
+		}
+	} else {
+		var (
+			link   string
+			err    error
+			source = filepath.Join(cw.source, p)
+		)
+
+		if f.Mode()&os.ModeSymlink != 0 {
+			if link, err = os.Readlink(source); err != nil {
+				return err
+			}
+		}
+
+		hdr, err := tar.FileInfoHeader(f, link)
+		if err != nil {
+			return err
+		}
+
+		hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
+
+		name := p
+		if strings.HasPrefix(name, string(filepath.Separator)) {
+			name, err = filepath.Rel(string(filepath.Separator), name)
+			if err != nil {
+				return errors.Wrap(err, "failed to make path relative")
+			}
+		}
+		name, err = tarName(name)
+		if err != nil {
+			return errors.Wrap(err, "cannot canonicalize path")
+		}
+		// suffix with '/' for directories
+		if f.IsDir() && !strings.HasSuffix(name, "/") {
+			name += "/"
+		}
+		hdr.Name = name
+
+		if err := setHeaderForSpecialDevice(hdr, name, f); err != nil {
+			return errors.Wrap(err, "failed to set device headers")
+		}
+
+		linkname, err := fs.GetLinkSource(name, f, cw.inodeCache)
+		if err != nil {
+			return errors.Wrap(err, "failed to get hardlink")
+		}
+
+		if linkname != "" {
+			hdr.Typeflag = tar.TypeLink
+			hdr.Linkname = linkname
+			hdr.Size = 0
+		}
+
+		if capability, err := getxattr(source, "security.capability"); err != nil {
+			return errors.Wrap(err, "failed to get capabilities xattr")
+		} else if capability != nil {
+			hdr.Xattrs = map[string]string{
+				"security.capability": string(capability),
+			}
+		}
+
+		if err := cw.tw.WriteHeader(hdr); err != nil {
+			return errors.Wrap(err, "failed to write file header")
+		}
+
+		if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 {
+			file, err := open(source)
+			if err != nil {
+				return errors.Wrapf(err, "failed to open path: %v", source)
+			}
+			defer file.Close()
+
+			buf := bufferPool.Get().([]byte)
+			n, err := io.CopyBuffer(cw.tw, file, buf)
+			bufferPool.Put(buf)
+			if err != nil {
+				return errors.Wrap(err, "failed to copy")
+			}
+			if n != hdr.Size {
+				return errors.New("short write copying file")
+			}
+		}
+	}
+	return nil
+}
+
+func (cw *changeWriter) Close() error {
+	if err := cw.tw.Close(); err != nil {
+		return errors.Wrap(err, "failed to close tar writer")
+	}
+	return nil
+}
+
+func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader) error {
+	// hdr.Mode is in linux format, which we can use for sycalls,
+	// but for os.Foo() calls we need the mode converted to os.FileMode,
+	// so use hdrInfo.Mode() (they differ for e.g. setuid bits)
+	hdrInfo := hdr.FileInfo()
+
+	switch hdr.Typeflag {
+	case tar.TypeDir:
+		// Create directory unless it exists as a directory already.
+		// In that case we just want to merge the two
+		if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) {
+			if err := os.Mkdir(path, hdrInfo.Mode()); err != nil {
+				return err
+			}
+		}
+
+	case tar.TypeReg, tar.TypeRegA:
+		file, err := openFile(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode())
+		if err != nil {
+			return err
+		}
+		buf := bufferPool.Get().([]byte)
+		_, err = io.CopyBuffer(file, reader, buf)
+		if err1 := file.Close(); err == nil {
+			err = err1
+		}
+		if err != nil {
+			return err
+		}
+
+	case tar.TypeBlock, tar.TypeChar:
+		// Handle this is an OS-specific way
+		if err := handleTarTypeBlockCharFifo(hdr, path); err != nil {
+			return err
+		}
+
+	case tar.TypeFifo:
+		// Handle this is an OS-specific way
+		if err := handleTarTypeBlockCharFifo(hdr, path); err != nil {
+			return err
+		}
+
+	case tar.TypeLink:
+		targetPath := filepath.Join(extractDir, hdr.Linkname)
+		// check for hardlink breakout
+		if !strings.HasPrefix(targetPath, extractDir) {
+			return errors.Wrapf(breakoutError, "invalid hardlink %q -> %q", targetPath, hdr.Linkname)
+		}
+		if err := os.Link(targetPath, path); err != nil {
+			return err
+		}
+
+	case tar.TypeSymlink:
+		// 	path 				-> hdr.Linkname = targetPath
+		// e.g. /extractDir/path/to/symlink 	-> ../2/file	= /extractDir/path/2/file
+		targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname)
+
+		// the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because
+		// that symlink would first have to be created, which would be caught earlier, at this very check:
+		if !strings.HasPrefix(targetPath, extractDir) {
+			return errors.Wrapf(breakoutError, "invalid symlink %q -> %q", path, hdr.Linkname)
+		}
+		if err := os.Symlink(hdr.Linkname, path); err != nil {
+			return err
+		}
+
+	case tar.TypeXGlobalHeader:
+		logrus.Debug("PAX Global Extended Headers found and ignored")
+		return nil
+
+	default:
+		return errors.Errorf("unhandled tar header type %d\n", hdr.Typeflag)
+	}
+
+	// Lchown is not supported on Windows.
+	if runtime.GOOS != "windows" {
+		if err := os.Lchown(path, hdr.Uid, hdr.Gid); err != nil {
+			return err
+		}
+	}
+
+	var warnErrors []string
+	for key, value := range hdr.Xattrs {
+		if err := setxattr(path, key, value); err != nil {
+			if errors.Cause(err) == syscall.ENOTSUP {
+				// Collect errors only for filesystem warning
+				warnErrors = append(warnErrors, err.Error())
+				continue
+			}
+			return err
+		}
+	}
+
+	if len(warnErrors) > 0 {
+		logrus.WithFields(logrus.Fields{
+			"errors": warnErrors,
+		}).Warn("ignored xattrs in archive: underlying filesystem doesn't support them")
+	}
+
+	// There is no LChmod, so ignore mode for symlink. Also, this
+	// must happen after chown, as that can modify the file mode
+	if err := handleLChmod(hdr, path, hdrInfo); err != nil {
+		return err
+	}
+
+	if err := chtimes(path, boundTime(latestTime(hdr.AccessTime, hdr.ModTime)), boundTime(hdr.ModTime)); err != nil {
+		return err
+	}
+
+	return nil
+}

archive/tar_linux.go

@@ -0,0 +1,134 @@
+package archive
+
+import (
+	"archive/tar"
+	"os"
+	"sync"
+	"syscall"
+
+	"github.com/opencontainers/runc/libcontainer/system"
+	"github.com/pkg/errors"
+	"github.com/stevvooe/continuity/sysx"
+)
+
+func tarName(p string) (string, error) {
+	return p, nil
+}
+
+func chmodTarEntry(perm os.FileMode) os.FileMode {
+	return perm
+}
+
+func setHeaderForSpecialDevice(hdr *tar.Header, name string, fi os.FileInfo) error {
+	s, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return errors.New("unsupported stat type")
+	}
+
+	// Currently go does not fill in the major/minors
+	if s.Mode&syscall.S_IFBLK != 0 ||
+		s.Mode&syscall.S_IFCHR != 0 {
+		hdr.Devmajor = int64(major(uint64(s.Rdev)))
+		hdr.Devminor = int64(minor(uint64(s.Rdev)))
+	}
+
+	return nil
+}
+
+func major(device uint64) uint64 {
+	return (device >> 8) & 0xfff
+}
+
+func minor(device uint64) uint64 {
+	return (device & 0xff) | ((device >> 12) & 0xfff00)
+}
+
+func mkdev(major int64, minor int64) uint32 {
+	return uint32(((minor & 0xfff00) << 12) | ((major & 0xfff) << 8) | (minor & 0xff))
+}
+
+func open(p string) (*os.File, error) {
+	return os.Open(p)
+}
+
+func openFile(name string, flag int, perm os.FileMode) (*os.File, error) {
+	return os.OpenFile(name, flag, perm)
+}
+
+func mkdirAll(path string, perm os.FileMode) error {
+	return os.MkdirAll(path, perm)
+}
+
+func prepareApply() func() {
+	// Unset unmask before doing an apply operation,
+	// restore unmask when complete
+	oldmask := syscall.Umask(0)
+	return func() {
+		syscall.Umask(oldmask)
+	}
+}
+
+func skipFile(*tar.Header) bool {
+	return false
+}
+
+var (
+	inUserNS bool
+	nsOnce   sync.Once
+)
+
+func setInUserNS() {
+	inUserNS = system.RunningInUserNS()
+}
+
+// handleTarTypeBlockCharFifo is an OS-specific helper function used by
+// createTarFile to handle the following types of header: Block; Char; Fifo
+func handleTarTypeBlockCharFifo(hdr *tar.Header, path string) error {
+	nsOnce.Do(setInUserNS)
+	if inUserNS {
+		// cannot create a device if running in user namespace
+		return nil
+	}
+
+	mode := uint32(hdr.Mode & 07777)
+	switch hdr.Typeflag {
+	case tar.TypeBlock:
+		mode |= syscall.S_IFBLK
+	case tar.TypeChar:
+		mode |= syscall.S_IFCHR
+	case tar.TypeFifo:
+		mode |= syscall.S_IFIFO
+	}
+
+	if err := syscall.Mknod(path, mode, int(mkdev(hdr.Devmajor, hdr.Devminor))); err != nil {
+		return err
+	}
+	return nil
+}
+
+func handleLChmod(hdr *tar.Header, path string, hdrInfo os.FileInfo) error {
+	if hdr.Typeflag == tar.TypeLink {
+		if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) {
+			if err := os.Chmod(path, hdrInfo.Mode()); err != nil {
+				return err
+			}
+		}
+	} else if hdr.Typeflag != tar.TypeSymlink {
+		if err := os.Chmod(path, hdrInfo.Mode()); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func getxattr(path, attr string) ([]byte, error) {
+	b, err := sysx.LGetxattr(path, attr)
+	if err == syscall.ENOTSUP || err == syscall.ENODATA {
+		return nil, nil
+	}
+	return b, err
+}
+
+func setxattr(path, key, value string) error {
+	return sysx.LSetxattr(path, key, []byte(value), 0)
+}

archive/tar_test.go

@@ -0,0 +1,224 @@
+package archive
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"os/user"
+	"testing"
+	"time"
+
+	_ "crypto/sha256"
+
+	"github.com/docker/containerd/fs"
+	"github.com/docker/containerd/fs/fstest"
+	"github.com/pkg/errors"
+)
+
+const tarCmd = "/usr/bin/tar"
+
+// baseApplier creates a basic filesystem layout
+// with multiple types of files for basic tests.
+var baseApplier = fstest.Apply(
+	fstest.CreateDir("/etc/", 0755),
+	fstest.CreateFile("/etc/hosts", []byte("127.0.0.1 localhost"), 0644),
+	fstest.Link("/etc/hosts", "/etc/hosts.allow"),
+	fstest.CreateDir("/usr/local/lib", 0755),
+	fstest.CreateFile("/usr/local/lib/libnothing.so", []byte{0x00, 0x00}, 0755),
+	fstest.Symlink("libnothing.so", "/usr/local/lib/libnothing.so.2"),
+	fstest.CreateDir("/home", 0755),
+	fstest.CreateDir("/home/derek", 0700),
+)
+
+func TestUnpack(t *testing.T) {
+	requireTar(t)
+
+	if err := testApply(baseApplier); err != nil {
+		t.Fatalf("Test apply failed: %+v", err)
+	}
+}
+
+func TestBaseDiff(t *testing.T) {
+	requireTar(t)
+
+	if err := testBaseDiff(baseApplier); err != nil {
+		t.Fatalf("Test base diff failed: %+v", err)
+	}
+}
+
+func TestDiffApply(t *testing.T) {
+	as := []fstest.Applier{
+		baseApplier,
+		fstest.Apply(
+			fstest.CreateFile("/etc/hosts", []byte("127.0.0.1 localhost.localdomain"), 0644),
+			fstest.CreateFile("/etc/fstab", []byte("/dev/sda1\t/\text4\tdefaults 1 1\n"), 0600),
+			fstest.CreateFile("/etc/badfile", []byte(""), 0666),
+			fstest.CreateFile("/home/derek/.zshrc", []byte("#ZSH is just better\n"), 0640),
+		),
+		fstest.Apply(
+			fstest.RemoveFile("/etc/badfile"),
+			fstest.Rename("/home/derek", "/home/notderek"),
+		),
+		fstest.Apply(
+			fstest.RemoveFile("/usr"),
+			fstest.RemoveFile("/etc/hosts.allow"),
+		),
+		fstest.Apply(
+			fstest.RemoveFile("/home"),
+			fstest.CreateDir("/home/derek", 0700),
+			fstest.CreateFile("/home/derek/.bashrc", []byte("#not going away\n"), 0640),
+			// "/etc/hosts" must be touched to be hardlinked in same layer
+			fstest.Chtime("/etc/hosts", time.Now()),
+			fstest.Link("/etc/hosts", "/etc/hosts.allow"),
+		),
+	}
+
+	if err := testDiffApply(as...); err != nil {
+		t.Fatalf("Test diff apply failed: %+v", err)
+	}
+}
+
+func testApply(a fstest.Applier) error {
+	td, err := ioutil.TempDir("", "test-apply-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create temp dir")
+	}
+	defer os.RemoveAll(td)
+	dest, err := ioutil.TempDir("", "test-apply-dest-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create temp dir")
+	}
+	defer os.RemoveAll(dest)
+
+	if err := a.Apply(td); err != nil {
+		return errors.Wrap(err, "failed to apply filesystem changes")
+	}
+
+	tarArgs := []string{"c", "-C", td}
+	names, err := readDirNames(td)
+	if err != nil {
+		return errors.Wrap(err, "failed to read directory names")
+	}
+	tarArgs = append(tarArgs, names...)
+
+	cmd := exec.Command(tarCmd, tarArgs...)
+
+	arch, err := cmd.StdoutPipe()
+	if err != nil {
+		return errors.Wrap(err, "failed to create stdout pipe")
+	}
+
+	if err := cmd.Start(); err != nil {
+		return errors.Wrap(err, "failed to start command")
+	}
+
+	if _, err := ApplyDiffTar(context.Background(), dest, arch); err != nil {
+		return errors.Wrap(err, "failed to apply tar stream")
+	}
+
+	return fstest.CheckDirectoryEqual(td, dest)
+}
+
+func testBaseDiff(a fstest.Applier) error {
+	td, err := ioutil.TempDir("", "test-base-diff-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create temp dir")
+	}
+	defer os.RemoveAll(td)
+	dest, err := ioutil.TempDir("", "test-base-diff-dest-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create temp dir")
+	}
+	defer os.RemoveAll(dest)
+
+	if err := a.Apply(td); err != nil {
+		return errors.Wrap(err, "failed to apply filesystem changes")
+	}
+
+	arch := DiffTarStream(context.Background(), "", td)
+
+	cmd := exec.Command(tarCmd, "x", "-C", dest)
+	cmd.Stdin = arch
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, "tar command failed")
+	}
+
+	return fstest.CheckDirectoryEqual(td, dest)
+}
+
+func testDiffApply(as ...fstest.Applier) error {
+	base, err := ioutil.TempDir("", "test-diff-apply-base-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create temp dir")
+	}
+	defer os.RemoveAll(base)
+	dest, err := ioutil.TempDir("", "test-diff-apply-dest-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create temp dir")
+	}
+	defer os.RemoveAll(dest)
+
+	ctx := context.Background()
+	for i, a := range as {
+		if err := diffApply(ctx, a, base, dest); err != nil {
+			return errors.Wrapf(err, "diff apply failed at layer %d", i)
+		}
+	}
+	return nil
+}
+
+// diffApply applies the given changes on the base and
+// computes the diff and applies to the dest.
+func diffApply(ctx context.Context, a fstest.Applier, base, dest string) error {
+	baseCopy, err := ioutil.TempDir("", "test-diff-apply-copy-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create temp dir")
+	}
+	defer os.RemoveAll(baseCopy)
+	if err := fs.CopyDir(baseCopy, base); err != nil {
+		return errors.Wrap(err, "failed to copy base")
+	}
+
+	if err := a.Apply(base); err != nil {
+		return errors.Wrap(err, "failed to apply changes to base")
+	}
+
+	if _, err := ApplyDiffTar(ctx, dest, DiffTarStream(ctx, baseCopy, base)); err != nil {
+		return errors.Wrap(err, "failed to apply tar stream")
+	}
+
+	return fstest.CheckDirectoryEqual(base, dest)
+}
+
+func readDirNames(p string) ([]string, error) {
+	fis, err := ioutil.ReadDir(p)
+	if err != nil {
+		return nil, err
+	}
+	names := make([]string, len(fis))
+	for i, fi := range fis {
+		names[i] = fi.Name()
+	}
+	return names, nil
+}
+
+func requireTar(t *testing.T) {
+	if _, err := os.Stat(tarCmd); err != nil {
+		if os.IsNotExist(err) {
+			t.Skipf("%s not found, skipping", tarCmd)
+		} else {
+			t.Fatalf("Unable to stat %s: %v", tarCmd, err)
+		}
+	}
+}
+
+func requireRoot(t *testing.T) {
+	u, err := user.Current()
+	if err != nil {
+		t.Fatalf("Unable to get current user: %v", err)
+	}
+	if u.Uid != "0" {
+		t.Skipf("test requires root, skipping")
+	}
+}

archive/tar_windows.go

@@ -0,0 +1,106 @@
+package archive
+
+import (
+	"archive/tar"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/docker/pkg/system"
+)
+
+// canonicalTarNameForPath returns platform-specific filepath
+// to canonical posix-style path for tar archival. p is relative
+// path.
+func tarName(p string) (string, error) {
+	// windows: convert windows style relative path with backslashes
+	// into forward slashes. Since windows does not allow '/' or '\'
+	// in file names, it is mostly safe to replace however we must
+	// check just in case
+	if strings.Contains(p, "/") {
+		return "", fmt.Errorf("Windows path contains forward slash: %s", p)
+	}
+
+	return strings.Replace(p, string(os.PathSeparator), "/", -1), nil
+}
+
+// chmodTarEntry is used to adjust the file permissions used in tar header based
+// on the platform the archival is done.
+func chmodTarEntry(perm os.FileMode) os.FileMode {
+	perm &= 0755
+	// Add the x bit: make everything +x from windows
+	perm |= 0111
+
+	return perm
+}
+
+func setHeaderForSpecialDevice(*tar.Header, string, os.FileInfo) error {
+	// do nothing. no notion of Rdev, Inode, Nlink in stat on Windows
+	return nil
+}
+
+func open(p string) (*os.File, error) {
+	// We use system.OpenSequential to ensure we use sequential file
+	// access on Windows to avoid depleting the standby list.
+	return system.OpenSequential(p)
+}
+
+func openFile(name string, flag int, perm os.FileMode) (*os.File, error) {
+	// Source is regular file. We use system.OpenFileSequential to use sequential
+	// file access to avoid depleting the standby list on Windows.
+	return system.OpenFileSequential(name, flag, perm)
+}
+
+func mkdirAll(path string, perm os.FileMode) error {
+	return system.MkdirAll(path, perm)
+}
+
+func prepareApply() func() {
+	// No umask or filesystem changes needed before apply
+	return func() {}
+}
+
+func skipFile(hdr *tar.Header) bool {
+	// Windows does not support filenames with colons in them. Ignore
+	// these files. This is not a problem though (although it might
+	// appear that it is). Let's suppose a client is running docker pull.
+	// The daemon it points to is Windows. Would it make sense for the
+	// client to be doing a docker pull Ubuntu for example (which has files
+	// with colons in the name under /usr/share/man/man3)? No, absolutely
+	// not as it would really only make sense that they were pulling a
+	// Windows image. However, for development, it is necessary to be able
+	// to pull Linux images which are in the repository.
+	//
+	// TODO Windows. Once the registry is aware of what images are Windows-
+	// specific or Linux-specific, this warning should be changed to an error
+	// to cater for the situation where someone does manage to upload a Linux
+	// image but have it tagged as Windows inadvertently.
+	if strings.Contains(hdr.Name, ":") {
+		logrus.Warnf("Windows: Ignoring %s (is this a Linux image?)", hdr.Name)
+		return true
+	}
+
+	return false
+}
+
+// handleTarTypeBlockCharFifo is an OS-specific helper function used by
+// createTarFile to handle the following types of header: Block; Char; Fifo
+func handleTarTypeBlockCharFifo(hdr *tar.Header, path string) error {
+	return nil
+}
+
+func handleLChmod(hdr *tar.Header, path string, hdrInfo os.FileInfo) error {
+	return nil
+}
+
+func getxattr(path, attr string) ([]byte, error) {
+	return nil, nil
+}
+
+func setxattr(path, key, value string) error {
+	// Return not support error, do not wrap underlying not supported
+	// since xattrs should not exist in windows diff archives
+	return errors.New("xattrs not supported on Windows")
+}

archive/time.go

@@ -0,0 +1,38 @@
+package archive
+
+import (
+	"syscall"
+	"time"
+	"unsafe"
+)
+
+var (
+	minTime = time.Unix(0, 0)
+	maxTime time.Time
+)
+
+func init() {
+	if unsafe.Sizeof(syscall.Timespec{}.Nsec) == 8 {
+		// This is a 64 bit timespec
+		// os.Chtimes limits time to the following
+		maxTime = time.Unix(0, 1<<63-1)
+	} else {
+		// This is a 32 bit timespec
+		maxTime = time.Unix(1<<31-1, 0)
+	}
+}
+
+func boundTime(t time.Time) time.Time {
+	if t.Before(minTime) || t.After(maxTime) {
+		return minTime
+	}
+
+	return t
+}
+
+func latestTime(t1, t2 time.Time) time.Time {
+	if t1.Before(t2) {
+		return t2
+	}
+	return t1
+}

archive/time_linux.go

@@ -0,0 +1,21 @@
+package archive
+
+import (
+	"time"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/pkg/errors"
+)
+
+func chtimes(path string, atime, mtime time.Time) error {
+	var utimes [2]unix.Timespec
+	utimes[0] = unix.NsecToTimespec(atime.UnixNano())
+	utimes[1] = unix.NsecToTimespec(mtime.UnixNano())
+
+	if err := unix.UtimesNanoAt(unix.AT_FDCWD, path, utimes[0:], unix.AT_SYMLINK_NOFOLLOW); err != nil {
+		return errors.Wrap(err, "failed call to UtimesNanoAt")
+	}
+
+	return nil
+}

archive/time_windows.go

@@ -0,0 +1,25 @@
+package archive
+
+import (
+	"syscall"
+	"time"
+)
+
+// chtimes will set the create time on a file using the given modtime.
+// This requires calling SetFileTime and explicitly including the create time.
+func chtimes(path string, atime, mtime time.Time) error {
+	ctimespec := syscall.NsecToTimespec(mtime.UnixNano())
+	pathp, e := syscall.UTF16PtrFromString(path)
+	if e != nil {
+		return e
+	}
+	h, e := syscall.CreateFile(pathp,
+		syscall.FILE_WRITE_ATTRIBUTES, syscall.FILE_SHARE_WRITE, nil,
+		syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0)
+	if e != nil {
+		return e
+	}
+	defer syscall.Close(h)
+	c := syscall.NsecToFiletime(syscall.TimespecToNsec(ctimespec))
+	return syscall.SetFileTime(h, &c, nil, nil)
+}