Add fs package

Add diff comparison with support for double walking
two trees for comparison or single walking a diff
tree. Single walking requires further implementation
for specific mount types.

Add directory copy function which is intended to provide
fastest possible local copy of file system directories
without hardlinking.

Add test package to make creating filesystems for
test easy and comparisons deep and informative.

Signed-off-by: Derek McGowan <derek@mcgstyle.net> (github: dmcgowan)

fs/copy.go

@@ -0,0 +1,106 @@
+package fs
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
+// CopyDirectory copies the directory from src to dst.
+// Most efficient copy of files is attempted.
+func CopyDirectory(dst, src string) error {
+	inodes := map[uint64]string{}
+	return copyDirectory(dst, src, inodes)
+}
+
+func copyDirectory(dst, src string, inodes map[uint64]string) error {
+	stat, err := os.Stat(src)
+	if err != nil {
+		return errors.Wrapf(err, "failed to stat %s", src)
+	}
+	if !stat.IsDir() {
+		return errors.Errorf("source is not directory")
+	}
+
+	if _, err := os.Stat(dst); err != nil {
+		if err := os.Mkdir(dst, stat.Mode()); err != nil {
+			return errors.Wrapf(err, "failed to mkdir %s", dst)
+		}
+	} else {
+		if err := os.Chmod(dst, stat.Mode()); err != nil {
+			return errors.Wrapf(err, "failed to chmod on %s", dst)
+		}
+	}
+
+	fis, err := ioutil.ReadDir(src)
+	if err != nil {
+		return errors.Wrapf(err, "failed to read %s", src)
+	}
+
+	if err := copyFileInfo(stat, dst); err != nil {
+		return errors.Wrapf(err, "failed to copy file info for %s", dst)
+	}
+
+	for _, fi := range fis {
+		source := filepath.Join(src, fi.Name())
+		target := filepath.Join(dst, fi.Name())
+		if fi.IsDir() {
+			if err := copyDirectory(target, source, inodes); err != nil {
+				return err
+			}
+			continue
+		} else if (fi.Mode() & os.ModeType) == 0 {
+			link, err := GetLinkSource(target, fi, inodes)
+			if err != nil {
+				return errors.Wrap(err, "failed to get hardlink")
+			}
+			if link != "" {
+				if err := os.Link(link, target); err != nil {
+					return errors.Wrap(err, "failed to create hard link")
+				}
+			} else if err := copyFile(source, target); err != nil {
+				return errors.Wrap(err, "failed to copy files")
+			}
+		} else if (fi.Mode() & os.ModeSymlink) == os.ModeSymlink {
+			link, err := os.Readlink(source)
+			if err != nil {
+				return errors.Wrapf(err, "failed to read link: %s", source)
+			}
+			if err := os.Symlink(link, target); err != nil {
+				return errors.Wrapf(err, "failed to create symlink: %s", target)
+			}
+		} else if (fi.Mode() & os.ModeDevice) == os.ModeDevice {
+			// TODO: support devices
+			return errors.New("devices not supported")
+		} else {
+			// TODO: Support pipes and sockets
+			return errors.Wrapf(err, "unsupported mode %s", fi.Mode())
+		}
+		if err := copyFileInfo(fi, target); err != nil {
+			return errors.Wrap(err, "failed to copy file info")
+		}
+
+		if err := copyXAttrs(target, source); err != nil {
+			return errors.Wrap(err, "failed to copy xattrs")
+		}
+	}
+
+	return nil
+}
+
+func copyFile(source, target string) error {
+	src, err := os.Open(source)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open source %s", err)
+	}
+	defer src.Close()
+	tgt, err := os.Create(target)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open target %s", err)
+	}
+	defer tgt.Close()
+
+	return copyFileContent(tgt, src)
+}

fs/copy_linux.go

@@ -0,0 +1,110 @@
+package fs
+
+import (
+	"io"
+	"os"
+	"strconv"
+	"syscall"
+
+	"github.com/pkg/errors"
+	"github.com/stevvooe/continuity/sysx"
+)
+
+var (
+	kernel, major int
+)
+
+func init() {
+	uts := &syscall.Utsname{}
+
+	err := syscall.Uname(uts)
+	if err != nil {
+		panic(err)
+	}
+
+	p := [2][]byte{{}, {}}
+
+	release := uts.Release
+	i := 0
+	for pi := 0; pi < len(p); pi++ {
+		for release[i] != 0 {
+			c := byte(release[i])
+			i++
+			if c == '.' || c == '-' {
+				break
+			}
+			p[pi] = append(p[pi], c)
+		}
+	}
+	kernel, err = strconv.Atoi(string(p[0]))
+	if err != nil {
+		panic(err)
+	}
+	major, err = strconv.Atoi(string(p[1]))
+	if err != nil {
+		panic(err)
+	}
+}
+
+func copyFileInfo(fi os.FileInfo, name string) error {
+	st := fi.Sys().(*syscall.Stat_t)
+	if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil {
+		return errors.Wrapf(err, "failed to chown %s", name)
+	}
+
+	if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
+		if err := os.Chmod(name, fi.Mode()); err != nil {
+			return errors.Wrapf(err, "failed to chmod %s", name)
+		}
+	}
+
+	if err := syscall.UtimesNano(name, []syscall.Timespec{st.Atim, st.Mtim}); err != nil {
+		return errors.Wrapf(err, "failed to utime %s", name)
+	}
+
+	return nil
+}
+
+func copyFileContent(dst, src *os.File) error {
+	if checkKernel(4, 5) {
+		// Use copy_file_range to do in kernel copying
+		// See https://lwn.net/Articles/659523/
+		// 326 on x86_64
+		st, err := src.Stat()
+		if err != nil {
+			return errors.Wrap(err, "unable to stat source")
+		}
+		n, _, e1 := syscall.Syscall6(326, src.Fd(), 0, dst.Fd(), 0, uintptr(st.Size()), 0)
+		if e1 != 0 {
+			return errors.Wrap(err, "copy_file_range failed")
+		}
+		if int64(n) != st.Size() {
+			return errors.Wrapf(err, "short copy: %d of %d", int64(n), st.Size())
+		}
+		return nil
+	}
+	_, err := io.Copy(dst, src)
+	return err
+}
+
+func checkKernel(k, m int) bool {
+	return (kernel == k && major >= m) || kernel > k
+}
+
+func copyXAttrs(dst, src string) error {
+	xattrKeys, err := sysx.LListxattr(src)
+	if err != nil {
+		return errors.Wrapf(err, "failed to list xattrs on %s", src)
+	}
+	for _, xattr := range xattrKeys {
+		data, err := sysx.LGetxattr(src, xattr)
+		if err != nil {
+			return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src)
+		}
+		if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil {
+			return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst)
+		}
+	}
+
+	return nil
+}

fs/copy_test.go

@@ -0,0 +1,53 @@
+package fs
+
+import (
+	"io/ioutil"
+	"testing"
+
+	_ "crypto/sha256"
+
+	"github.com/docker/containerd/fs/fstest"
+	"github.com/pkg/errors"
+)
+
+// TODO: Create copy directory which requires privilege
+//  chown
+//  mknod
+//  setxattr fstest.SetXAttr("/home", "trusted.overlay.opaque", "y"),
+
+func TestCopyDirectory(t *testing.T) {
+	apply := fstest.MultiApply(
+		fstest.CreateDirectory("/etc/", 0755),
+		fstest.NewTestFile("/etc/hosts", []byte("localhost 127.0.0.1"), 0644),
+		fstest.Link("/etc/hosts", "/etc/hosts.allow"),
+		fstest.CreateDirectory("/usr/local/lib", 0755),
+		fstest.NewTestFile("/usr/local/lib/libnothing.so", []byte{0x00, 0x00}, 0755),
+		fstest.Symlink("libnothing.so", "/usr/local/lib/libnothing.so.2"),
+		fstest.CreateDirectory("/home", 0755),
+	)
+
+	if err := testCopy(apply); err != nil {
+		t.Fatalf("Copy test failed: %+v", err)
+	}
+}
+
+func testCopy(apply fstest.Applier) error {
+	t1, err := ioutil.TempDir("", "test-copy-src-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create temporary directory")
+	}
+	t2, err := ioutil.TempDir("", "test-copy-dst-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create temporary directory")
+	}
+
+	if err := apply(t1); err != nil {
+		return errors.Wrap(err, "failed to apply changes")
+	}
+
+	if err := CopyDirectory(t2, t1); err != nil {
+		return errors.Wrap(err, "failed to copy")
+	}
+
+	return fstest.CheckDirectoryEqual(t1, t2)
+}

fs/copy_windows.go

@@ -0,0 +1,27 @@
+package fs
+
+import (
+	"io"
+	"os"
+
+	"github.com/pkg/errors"
+)
+
+func copyFileInfo(fi os.FileInfo, name string) error {
+	if err := os.Chmod(name, fi.Mode()); err != nil {
+		return errors.Wrapf(err, "failed to chmod %s", name)
+	}
+
+	// TODO: copy windows specific metadata
+
+	return nil
+}
+
+func copyFileContent(dst, src *os.File) error {
+	_, err := io.Copy(dst, src)
+	return err
+}
+
+func copyXAttrs(dst, src string) error {
+	return nil
+}

fs/diff.go

@@ -0,0 +1,360 @@
+package fs
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/Sirupsen/logrus"
+)
+
+// ChangeKind is the type of modification that
+// a change is making.
+type ChangeKind int
+
+const (
+	// ChangeKindAdd represents an addition of
+	// a file
+	ChangeKindAdd = iota
+
+	// ChangeKindModify represents a change to
+	// an existing file
+	ChangeKindModify
+
+	// ChangeKindDelete represents a delete of
+	// a file
+	ChangeKindDelete
+)
+
+func (k ChangeKind) String() string {
+	switch k {
+	case ChangeKindAdd:
+		return "add"
+	case ChangeKindModify:
+		return "modify"
+	case ChangeKindDelete:
+		return "delete"
+	default:
+		return ""
+	}
+}
+
+// Change represents single change between a diff and its parent.
+type Change struct {
+	Kind     ChangeKind
+	Path     string
+	FileInfo os.FileInfo
+	Source   string
+}
+
+// Changes returns a stream of changes between the provided upper
+// directory and lower directory.
+//
+// Changes are ordered by name and should be appliable in the
+// order in which they received.
+//  Due to this apply ordering, the following is true
+//  - Removed directory trees only create a single change for the root
+//    directory removed. Remaining changes are implied.
+//  - A directory which is modified to become a file will not have
+//    delete entries for sub-path items, their removal is implied
+//    by the removal of the parent directory.
+//
+// Opaque directories will not be treated specially and each file
+// removed from the lower will show up as a removal
+//
+// File content comparisons will be done on files which have timestamps
+// which may have been truncated. If either of the files being compared
+// has a zero value nanosecond value, each byte will be compared for
+// differences. If 2 files have the same seconds value but different
+// nanosecond values where one of those values is zero, the files will
+// be considered unchanged if the content is the same. This behavior
+// is to account for timestamp truncation during archiving.
+func Changes(ctx context.Context, upper, lower string) (context.Context, <-chan Change) {
+	var (
+		changes        = make(chan Change)
+		retCtx, cancel = context.WithCancel(ctx)
+	)
+
+	cc := &changeContext{
+		Context: retCtx,
+	}
+
+	go func() {
+		var err error
+		if lower == "" {
+			logrus.Debugf("Using single walk diff for %s", upper)
+			err = addDirChanges(ctx, changes, upper)
+		} else if diffOptions := detectDirDiff(upper, lower); diffOptions != nil {
+			logrus.Debugf("Using single walk diff for %s from %s", diffOptions.diffDir, lower)
+			err = diffDirChanges(ctx, changes, lower, diffOptions)
+		} else {
+			logrus.Debugf("Using double walk diff for %s from %s", upper, lower)
+			err = doubleWalkDiff(ctx, changes, upper, lower)
+		}
+
+		if err != nil {
+			cc.errL.Lock()
+			cc.err = err
+			cc.errL.Unlock()
+			cancel()
+		}
+		defer close(changes)
+	}()
+
+	return cc, changes
+}
+
+// changeContext wraps a context to allow setting an error
+// directly from a change streamer to allow streams canceled
+// due to errors to propagate the error to the caller.
+type changeContext struct {
+	context.Context
+
+	err  error
+	errL sync.Mutex
+}
+
+func (cc *changeContext) Err() error {
+	cc.errL.Lock()
+	if cc.err != nil {
+		return cc.err
+	}
+	cc.errL.Unlock()
+	return cc.Context.Err()
+}
+
+func sendChange(ctx context.Context, changes chan<- Change, change Change) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case changes <- change:
+		return nil
+	}
+}
+
+func addDirChanges(ctx context.Context, changes chan<- Change, root string) error {
+	return filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Rebase path
+		path, err = filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+
+		path = filepath.Join(string(os.PathSeparator), path)
+
+		// Skip root
+		if path == string(os.PathSeparator) {
+			return nil
+		}
+
+		change := Change{
+			Path:     path,
+			Kind:     ChangeKindAdd,
+			FileInfo: f,
+			Source:   filepath.Join(root, path),
+		}
+
+		return sendChange(ctx, changes, change)
+	})
+}
+
+// diffDirOptions is used when the diff can be directly calculated from
+// a diff directory to its lower, without walking both trees.
+type diffDirOptions struct {
+	diffDir      string
+	skipChange   func(string) (bool, error)
+	deleteChange func(string, string, os.FileInfo) (string, error)
+}
+
+// diffDirChanges walks the diff directory and compares changes against the lower.
+func diffDirChanges(ctx context.Context, changes chan<- Change, lower string, o *diffDirOptions) error {
+	changedDirs := make(map[string]struct{})
+	return filepath.Walk(o.diffDir, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Rebase path
+		path, err = filepath.Rel(o.diffDir, path)
+		if err != nil {
+			return err
+		}
+
+		path = filepath.Join(string(os.PathSeparator), path)
+
+		// Skip root
+		if path == string(os.PathSeparator) {
+			return nil
+		}
+
+		// TODO: handle opaqueness, start new double walker at this
+		// location to get deletes, and skip tree in single walker
+
+		if o.skipChange != nil {
+			if skip, err := o.skipChange(path); skip {
+				return err
+			}
+		}
+
+		change := Change{
+			Path: path,
+		}
+
+		deletedFile, err := o.deleteChange(o.diffDir, path, f)
+		if err != nil {
+			return err
+		}
+
+		// Find out what kind of modification happened
+		if deletedFile != "" {
+			change.Path = deletedFile
+			change.Kind = ChangeKindDelete
+		} else {
+			// Otherwise, the file was added
+			change.Kind = ChangeKindAdd
+			change.FileInfo = f
+			change.Source = filepath.Join(o.diffDir, path)
+
+			// ...Unless it already existed in a lower, in which case, it's a modification
+			stat, err := os.Stat(filepath.Join(lower, path))
+			if err != nil && !os.IsNotExist(err) {
+				return err
+			}
+			if err == nil {
+				// The file existed in the lower, so that's a modification
+
+				// However, if it's a directory, maybe it wasn't actually modified.
+				// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
+				if stat.IsDir() && f.IsDir() {
+					if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) {
+						// Both directories are the same, don't record the change
+						return nil
+					}
+				}
+				change.Kind = ChangeKindModify
+			}
+		}
+
+		// If /foo/bar/file.txt is modified, then /foo/bar must be part of the changed files.
+		// This block is here to ensure the change is recorded even if the
+		// modify time, mode and size of the parent directory in the rw and ro layers are all equal.
+		// Check https://github.com/docker/docker/pull/13590 for details.
+		if f.IsDir() {
+			changedDirs[path] = struct{}{}
+		}
+		if change.Kind == ChangeKindAdd || change.Kind == ChangeKindDelete {
+			parent := filepath.Dir(path)
+			if _, ok := changedDirs[parent]; !ok && parent != "/" {
+				pi, err := os.Stat(filepath.Join(o.diffDir, parent))
+				if err != nil {
+					return err
+				}
+				dirChange := Change{
+					Path:     parent,
+					Kind:     ChangeKindModify,
+					FileInfo: pi,
+					Source:   filepath.Join(o.diffDir, parent),
+				}
+				if err := sendChange(ctx, changes, dirChange); err != nil {
+					return err
+				}
+				changedDirs[parent] = struct{}{}
+			}
+		}
+
+		return sendChange(ctx, changes, change)
+	})
+}
+
+// doubleWalkDiff walks both directories to create a diff
+func doubleWalkDiff(ctx context.Context, changes chan<- Change, upper, lower string) (err error) {
+	pathCtx, cancel := context.WithCancel(ctx)
+	defer func() {
+		if err != nil {
+			cancel()
+		}
+	}()
+
+	var (
+		w1     = pathWalker(pathCtx, lower)
+		w2     = pathWalker(pathCtx, upper)
+		f1, f2 *currentPath
+		rmdir  string
+	)
+
+	for w1 != nil || w2 != nil {
+		if f1 == nil && w1 != nil {
+			f1, err = nextPath(w1)
+			if err != nil {
+				return err
+			}
+			if f1 == nil {
+				w1 = nil
+			}
+		}
+
+		if f2 == nil && w2 != nil {
+			f2, err = nextPath(w2)
+			if err != nil {
+				return err
+			}
+			if f2 == nil {
+				w2 = nil
+			}
+		}
+		if f1 == nil && f2 == nil {
+			continue
+		}
+
+		c := pathChange(f1, f2)
+		switch c.Kind {
+		case ChangeKindAdd:
+			if rmdir != "" {
+				rmdir = ""
+			}
+			c.FileInfo = f2.f
+			c.Source = filepath.Join(upper, c.Path)
+			f2 = nil
+		case ChangeKindDelete:
+			// Check if this file is already removed by being
+			// under of a removed directory
+			if rmdir != "" && strings.HasPrefix(f1.path, rmdir) {
+				f1 = nil
+				continue
+			} else if rmdir == "" && f1.f.IsDir() {
+				rmdir = f1.path + string(os.PathSeparator)
+			} else if rmdir != "" {
+				rmdir = ""
+			}
+			f1 = nil
+		case ChangeKindModify:
+			same, err := sameFile(f1, f2)
+			if err != nil {
+				return err
+			}
+			if f1.f.IsDir() && !f2.f.IsDir() {
+				rmdir = f1.path + string(os.PathSeparator)
+			} else if rmdir != "" {
+				rmdir = ""
+			}
+			c.FileInfo = f2.f
+			c.Source = filepath.Join(upper, c.Path)
+			f1 = nil
+			f2 = nil
+			if same {
+				continue
+			}
+		}
+		if err := sendChange(ctx, changes, c); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

fs/diff_linux.go

@@ -0,0 +1,92 @@
+package fs
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/pkg/errors"
+	"github.com/stevvooe/continuity/sysx"
+)
+
+// whiteouts are files with a special meaning for the layered filesystem.
+// Docker uses AUFS whiteout files inside exported archives. In other
+// filesystems these files are generated/handled on tar creation/extraction.
+
+// whiteoutPrefix prefix means file is a whiteout. If this is followed by a
+// filename this means that file has been removed from the base layer.
+const whiteoutPrefix = ".wh."
+
+// whiteoutMetaPrefix prefix means whiteout has a special meaning and is not
+// for removing an actual file. Normally these files are excluded from exported
+// archives.
+const whiteoutMetaPrefix = whiteoutPrefix + whiteoutPrefix
+
+// whiteoutLinkDir is a directory AUFS uses for storing hardlink links to other
+// layers. Normally these should not go into exported archives and all changed
+// hardlinks should be copied to the top layer.
+const whiteoutLinkDir = whiteoutMetaPrefix + "plnk"
+
+// whiteoutOpaqueDir file means directory has been made opaque - meaning
+// readdir calls to this directory do not follow to lower layers.
+const whiteoutOpaqueDir = whiteoutMetaPrefix + ".opq"
+
+// detectDirDiff returns diff dir options if a directory could
+// be found in the mount info for upper which is the direct
+// diff with the provided lower directory
+func detectDirDiff(upper, lower string) *diffDirOptions {
+	// TODO: get mount options for upper
+	// TODO: detect AUFS
+	// TODO: detect overlay
+	return nil
+}
+
+func aufsMetadataSkip(path string) (skip bool, err error) {
+	skip, err = filepath.Match(string(os.PathSeparator)+whiteoutMetaPrefix+"*", path)
+	if err != nil {
+		skip = true
+	}
+	return
+}
+
+func aufsDeletedFile(root, path string, fi os.FileInfo) (string, error) {
+	f := filepath.Base(path)
+
+	// If there is a whiteout, then the file was removed
+	if strings.HasPrefix(f, whiteoutPrefix) {
+		originalFile := f[len(whiteoutPrefix):]
+		return filepath.Join(filepath.Dir(path), originalFile), nil
+	}
+
+	return "", nil
+}
+
+// compareSysStat returns whether the stats are equivalent,
+// whether the files are considered the same file, and
+// an error
+func compareSysStat(s1, s2 interface{}) (bool, error) {
+	ls1, ok := s1.(*syscall.Stat_t)
+	if !ok {
+		return false, nil
+	}
+	ls2, ok := s2.(*syscall.Stat_t)
+	if !ok {
+		return false, nil
+	}
+
+	return ls1.Mode == ls2.Mode && ls1.Uid == ls2.Uid && ls1.Gid == ls2.Gid && ls1.Rdev == ls2.Rdev, nil
+}
+
+func compareCapabilities(p1, p2 string) (bool, error) {
+	c1, err := sysx.LGetxattr(p1, "security.capability")
+	if err != nil && err != syscall.ENODATA {
+		return false, errors.Wrapf(err, "failed to get xattr for %s", p1)
+	}
+	c2, err := sysx.LGetxattr(p2, "security.capability")
+	if err != nil && err != syscall.ENODATA {
+		return false, errors.Wrapf(err, "failed to get xattr for %s", p2)
+	}
+	return bytes.Equal(c1, c2), nil
+}

fs/diff_test.go

@@ -0,0 +1,304 @@
+package fs
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/docker/containerd/fs/fstest"
+	"github.com/pkg/errors"
+)
+
+// TODO: Additional tests
+// - capability test (requires privilege)
+// - chown test (requires privilege)
+// - symlink test
+// - hardlink test
+
+func TestSimpleDiff(t *testing.T) {
+	l1 := fstest.MultiApply(
+		fstest.CreateDirectory("/etc", 0755),
+		fstest.NewTestFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644),
+		fstest.NewTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0644),
+		fstest.NewTestFile("/etc/unchanged", []byte("PATH=/usr/bin"), 0644),
+		fstest.NewTestFile("/etc/unexpected", []byte("#!/bin/sh"), 0644),
+	)
+	l2 := fstest.MultiApply(
+		fstest.NewTestFile("/etc/hosts", []byte("mydomain 10.0.0.120"), 0644),
+		fstest.NewTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0666),
+		fstest.CreateDirectory("/root", 0700),
+		fstest.NewTestFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0644),
+		fstest.RemoveFile("/etc/unexpected"),
+	)
+	diff := []Change{
+		Modify("/etc/hosts"),
+		Modify("/etc/profile"),
+		Delete("/etc/unexpected"),
+		Add("/root"),
+		Add("/root/.bashrc"),
+	}
+
+	if err := testDiffWithBase(l1, l2, diff); err != nil {
+		t.Fatalf("Failed diff with base: %+v", err)
+	}
+}
+
+func TestDirectoryReplace(t *testing.T) {
+	l1 := fstest.MultiApply(
+		fstest.CreateDirectory("/dir1", 0755),
+		fstest.NewTestFile("/dir1/f1", []byte("#####"), 0644),
+		fstest.CreateDirectory("/dir1/f2", 0755),
+		fstest.NewTestFile("/dir1/f2/f3", []byte("#!/bin/sh"), 0644),
+	)
+	l2 := fstest.MultiApply(
+		fstest.NewTestFile("/dir1/f11", []byte("#New file here"), 0644),
+		fstest.RemoveFile("/dir1/f2"),
+		fstest.NewTestFile("/dir1/f2", []byte("Now file"), 0666),
+	)
+	diff := []Change{
+		Add("/dir1/f11"),
+		Modify("/dir1/f2"),
+	}
+
+	if err := testDiffWithBase(l1, l2, diff); err != nil {
+		t.Fatalf("Failed diff with base: %+v", err)
+	}
+}
+
+func TestRemoveDirectoryTree(t *testing.T) {
+	l1 := fstest.MultiApply(
+		fstest.CreateDirectory("/dir1/dir2/dir3", 0755),
+		fstest.NewTestFile("/dir1/f1", []byte("f1"), 0644),
+		fstest.NewTestFile("/dir1/dir2/f2", []byte("f2"), 0644),
+	)
+	l2 := fstest.MultiApply(
+		fstest.RemoveFile("/dir1"),
+	)
+	diff := []Change{
+		Delete("/dir1"),
+	}
+
+	if err := testDiffWithBase(l1, l2, diff); err != nil {
+		t.Fatalf("Failed diff with base: %+v", err)
+	}
+}
+
+func TestFileReplace(t *testing.T) {
+	l1 := fstest.MultiApply(
+		fstest.NewTestFile("/dir1", []byte("a file, not a directory"), 0644),
+	)
+	l2 := fstest.MultiApply(
+		fstest.RemoveFile("/dir1"),
+		fstest.CreateDirectory("/dir1/dir2", 0755),
+		fstest.NewTestFile("/dir1/dir2/f1", []byte("also a file"), 0644),
+	)
+	diff := []Change{
+		Modify("/dir1"),
+		Add("/dir1/dir2"),
+		Add("/dir1/dir2/f1"),
+	}
+
+	if err := testDiffWithBase(l1, l2, diff); err != nil {
+		t.Fatalf("Failed diff with base: %+v", err)
+	}
+}
+
+func TestUpdateWithSameTime(t *testing.T) {
+	tt := time.Now().Truncate(time.Second)
+	t1 := tt.Add(5 * time.Nanosecond)
+	t2 := tt.Add(6 * time.Nanosecond)
+	l1 := fstest.MultiApply(
+		fstest.NewTestFile("/file-modified-time", []byte("1"), 0644),
+		fstest.Chtime("/file-modified-time", t1),
+		fstest.NewTestFile("/file-no-change", []byte("1"), 0644),
+		fstest.Chtime("/file-no-change", t1),
+		fstest.NewTestFile("/file-same-time", []byte("1"), 0644),
+		fstest.Chtime("/file-same-time", t1),
+		fstest.NewTestFile("/file-truncated-time-1", []byte("1"), 0644),
+		fstest.Chtime("/file-truncated-time-1", t1),
+		fstest.NewTestFile("/file-truncated-time-2", []byte("1"), 0644),
+		fstest.Chtime("/file-truncated-time-2", tt),
+	)
+	l2 := fstest.MultiApply(
+		fstest.NewTestFile("/file-modified-time", []byte("2"), 0644),
+		fstest.Chtime("/file-modified-time", t2),
+		fstest.NewTestFile("/file-no-change", []byte("1"), 0644),
+		fstest.Chtime("/file-no-change", tt), // use truncated time, should be regarded as no change
+		fstest.NewTestFile("/file-same-time", []byte("2"), 0644),
+		fstest.Chtime("/file-same-time", t1),
+		fstest.NewTestFile("/file-truncated-time-1", []byte("2"), 0644),
+		fstest.Chtime("/file-truncated-time-1", tt),
+		fstest.NewTestFile("/file-truncated-time-2", []byte("2"), 0644),
+		fstest.Chtime("/file-truncated-time-2", tt),
+	)
+	diff := []Change{
+		// "/file-same-time" excluded because matching non-zero nanosecond values
+		Modify("/file-modified-time"),
+		Modify("/file-truncated-time-1"),
+		Modify("/file-truncated-time-2"),
+	}
+
+	if err := testDiffWithBase(l1, l2, diff); err != nil {
+		t.Fatalf("Failed diff with base: %+v", err)
+	}
+}
+
+func testDiffWithBase(base, diff fstest.Applier, expected []Change) error {
+	t1, err := ioutil.TempDir("", "diff-with-base-lower-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create temp dir")
+	}
+	defer os.RemoveAll(t1)
+	t2, err := ioutil.TempDir("", "diff-with-base-upper-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create temp dir")
+	}
+	defer os.RemoveAll(t2)
+
+	if err := base(t1); err != nil {
+		return errors.Wrap(err, "failed to apply base filesytem")
+	}
+
+	if err := CopyDirectory(t2, t1); err != nil {
+		return errors.Wrap(err, "failed to copy base directory")
+	}
+
+	if err := diff(t2); err != nil {
+		return errors.Wrap(err, "failed to apply diff filesystem")
+	}
+
+	changes, err := collectChanges(t2, t1)
+	if err != nil {
+		return errors.Wrap(err, "failed to collect changes")
+	}
+
+	return checkChanges(t2, changes, expected)
+}
+
+func TestBaseDirectoryChanges(t *testing.T) {
+	apply := fstest.MultiApply(
+		fstest.CreateDirectory("/etc", 0755),
+		fstest.NewTestFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644),
+		fstest.NewTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0644),
+		fstest.CreateDirectory("/root", 0700),
+		fstest.NewTestFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0644),
+	)
+	changes := []Change{
+		Add("/etc"),
+		Add("/etc/hosts"),
+		Add("/etc/profile"),
+		Add("/root"),
+		Add("/root/.bashrc"),
+	}
+
+	if err := testDiffWithoutBase(apply, changes); err != nil {
+		t.Fatalf("Failed diff without base: %+v", err)
+	}
+}
+
+func testDiffWithoutBase(apply fstest.Applier, expected []Change) error {
+	tmp, err := ioutil.TempDir("", "diff-without-base-")
+	if err != nil {
+		return errors.Wrap(err, "failed to create temp dir")
+	}
+	defer os.RemoveAll(tmp)
+
+	if err := apply(tmp); err != nil {
+		return errors.Wrap(err, "failed to apply filesytem changes")
+	}
+
+	changes, err := collectChanges(tmp, "")
+	if err != nil {
+		return errors.Wrap(err, "failed to collect changes")
+	}
+
+	return checkChanges(tmp, changes, expected)
+}
+
+func checkChanges(root string, changes, expected []Change) error {
+	if len(changes) != len(expected) {
+		return errors.Errorf("Unexpected number of changes:\n%s", diffString(changes, expected))
+	}
+	for i := range changes {
+		if changes[i].Path != expected[i].Path || changes[i].Kind != expected[i].Kind {
+			return errors.Errorf("Unexpected change at %d:\n%s", i, diffString(changes, expected))
+		}
+		if changes[i].Kind != ChangeKindDelete {
+			filename := filepath.Join(root, changes[i].Path)
+			efi, err := os.Stat(filename)
+			if err != nil {
+				return errors.Wrapf(err, "failed to stat %q", filename)
+			}
+			afi := changes[i].FileInfo
+			if afi.Size() != efi.Size() {
+				return errors.Errorf("Unexpected change size %d, %q has size %d", afi.Size(), filename, efi.Size())
+			}
+			if afi.Mode() != efi.Mode() {
+				return errors.Errorf("Unexpected change mode %s, %q has mode %s", afi.Mode(), filename, efi.Mode())
+			}
+			if afi.ModTime() != efi.ModTime() {
+				return errors.Errorf("Unexpected change modtime %s, %q has modtime %s", afi.ModTime(), filename, efi.ModTime())
+			}
+			if expected := filepath.Join(root, changes[i].Path); changes[i].Source != expected {
+				return errors.Errorf("Unexpected source path %s, expected %s", changes[i].Source, expected)
+			}
+		}
+	}
+
+	return nil
+}
+
+func collectChanges(upper, lower string) ([]Change, error) {
+	ctx, changeC := Changes(context.Background(), upper, lower)
+	changes := []Change{}
+	for {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case c, ok := <-changeC:
+			if !ok {
+				return changes, nil
+			}
+			changes = append(changes, c)
+		}
+	}
+}
+
+func diffString(c1, c2 []Change) string {
+	return fmt.Sprintf("got(%d):\n%s\nexpected(%d):\n%s", len(c1), changesString(c1), len(c2), changesString(c2))
+
+}
+
+func changesString(c []Change) string {
+	strs := make([]string, len(c))
+	for i := range c {
+		strs[i] = fmt.Sprintf("\t%s\t%s", c[i].Kind, c[i].Path)
+	}
+	return strings.Join(strs, "\n")
+}
+
+func Add(p string) Change {
+	return Change{
+		Kind: ChangeKindAdd,
+		Path: p,
+	}
+}
+
+func Delete(p string) Change {
+	return Change{
+		Kind: ChangeKindDelete,
+		Path: p,
+	}
+}
+
+func Modify(p string) Change {
+	return Change{
+		Kind: ChangeKindModify,
+		Path: p,
+	}
+}

fs/diff_windows.go

@@ -0,0 +1,15 @@
+package fs
+
+func detectDirDiff(upper, lower string) *diffDirOptions {
+	return nil
+}
+
+func compareSysStat(s1, s2 interface{}) (bool, error) {
+	// TODO: Use windows specific sys type
+	return false, nil
+}
+
+func compareCapabilities(p1, p2 string) (bool, error) {
+	// TODO: Use windows equivalent
+	return true, nil
+}

fs/fstest/compare.go

@@ -0,0 +1,37 @@
+package fstest
+
+import (
+	"github.com/pkg/errors"
+	"github.com/stevvooe/continuity"
+)
+
+// CheckDirectoryEqual compares two directory paths to make sure that
+// the content of the directories is the same.
+func CheckDirectoryEqual(d1, d2 string) error {
+	c1, err := continuity.NewContext(d1)
+	if err != nil {
+		return errors.Wrap(err, "failed to build context")
+	}
+
+	c2, err := continuity.NewContext(d2)
+	if err != nil {
+		return errors.Wrap(err, "failed to build context")
+	}
+
+	m1, err := continuity.BuildManifest(c1)
+	if err != nil {
+		return errors.Wrap(err, "failed to build manifest")
+	}
+
+	m2, err := continuity.BuildManifest(c2)
+	if err != nil {
+		return errors.Wrap(err, "failed to build manifest")
+	}
+
+	diff := diffResourceList(m1.Resources, m2.Resources)
+	if diff.HasDiff() {
+		return errors.Errorf("directory diff between %s and %s\n%s", d1, d2, diff.String())
+	}
+
+	return nil
+}

fs/fstest/continuity_util.go

@@ -0,0 +1,189 @@
+package fstest
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/stevvooe/continuity"
+)
+
+type resourceUpdate struct {
+	Original continuity.Resource
+	Updated  continuity.Resource
+}
+
+func (u resourceUpdate) String() string {
+	return fmt.Sprintf("%s(mode: %o, uid: %s, gid: %s) -> %s(mode: %o, uid: %s, gid: %s)",
+		u.Original.Path(), u.Original.Mode(), u.Original.UID(), u.Original.GID(),
+		u.Updated.Path(), u.Updated.Mode(), u.Updated.UID(), u.Updated.GID(),
+	)
+}
+
+type resourceListDifference struct {
+	Additions []continuity.Resource
+	Deletions []continuity.Resource
+	Updates   []resourceUpdate
+}
+
+func (l resourceListDifference) HasDiff() bool {
+	return len(l.Additions) > 0 || len(l.Deletions) > 0 || len(l.Updates) > 0
+}
+
+func (l resourceListDifference) String() string {
+	buf := bytes.NewBuffer(nil)
+	for _, add := range l.Additions {
+		fmt.Fprintf(buf, "+ %s\n", add.Path())
+	}
+	for _, del := range l.Deletions {
+		fmt.Fprintf(buf, "- %s\n", del.Path())
+	}
+	for _, upt := range l.Updates {
+		fmt.Fprintf(buf, "~ %s\n", upt.String())
+	}
+	return string(buf.Bytes())
+}
+
+// diffManifest compares two resource lists and returns the list
+// of adds updates and deletes, resource lists are not reordered
+// before doing difference.
+func diffResourceList(r1, r2 []continuity.Resource) resourceListDifference {
+	i1 := 0
+	i2 := 0
+	var d resourceListDifference
+
+	for i1 < len(r1) && i2 < len(r2) {
+		p1 := r1[i1].Path()
+		p2 := r2[i2].Path()
+		switch {
+		case p1 < p2:
+			d.Deletions = append(d.Deletions, r1[i1])
+			i1++
+		case p1 == p2:
+			if !compareResource(r1[i1], r2[i2]) {
+				d.Updates = append(d.Updates, resourceUpdate{
+					Original: r1[i1],
+					Updated:  r2[i2],
+				})
+			}
+			i1++
+			i2++
+		case p1 > p2:
+			d.Additions = append(d.Additions, r2[i2])
+			i2++
+		}
+	}
+
+	for i1 < len(r1) {
+		d.Deletions = append(d.Deletions, r1[i1])
+		i1++
+
+	}
+	for i2 < len(r2) {
+		d.Additions = append(d.Additions, r2[i2])
+		i2++
+	}
+
+	return d
+}
+
+func compareResource(r1, r2 continuity.Resource) bool {
+	if r1.Path() != r2.Path() {
+		return false
+	}
+	if r1.Mode() != r2.Mode() {
+		return false
+	}
+	if r1.UID() != r2.UID() {
+		return false
+	}
+	if r1.GID() != r2.GID() {
+		return false
+	}
+
+	// TODO(dmcgowan): Check if is XAttrer
+
+	return compareResourceTypes(r1, r2)
+
+}
+
+func compareResourceTypes(r1, r2 continuity.Resource) bool {
+	switch t1 := r1.(type) {
+	case continuity.RegularFile:
+		t2, ok := r2.(continuity.RegularFile)
+		if !ok {
+			return false
+		}
+		return compareRegularFile(t1, t2)
+	case continuity.Directory:
+		t2, ok := r2.(continuity.Directory)
+		if !ok {
+			return false
+		}
+		return compareDirectory(t1, t2)
+	case continuity.SymLink:
+		t2, ok := r2.(continuity.SymLink)
+		if !ok {
+			return false
+		}
+		return compareSymLink(t1, t2)
+	case continuity.NamedPipe:
+		t2, ok := r2.(continuity.NamedPipe)
+		if !ok {
+			return false
+		}
+		return compareNamedPipe(t1, t2)
+	case continuity.Device:
+		t2, ok := r2.(continuity.Device)
+		if !ok {
+			return false
+		}
+		return compareDevice(t1, t2)
+	default:
+		// TODO(dmcgowan): Should this panic?
+		return r1 == r2
+	}
+}
+
+func compareRegularFile(r1, r2 continuity.RegularFile) bool {
+	if r1.Size() != r2.Size() {
+		return false
+	}
+	p1 := r1.Paths()
+	p2 := r2.Paths()
+	if len(p1) != len(p2) {
+		return false
+	}
+	for i := range p1 {
+		if p1[i] != p2[i] {
+			return false
+		}
+	}
+	d1 := r1.Digests()
+	d2 := r2.Digests()
+	if len(d1) != len(d2) {
+		return false
+	}
+	for i := range d1 {
+		if d1[i] != d2[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareSymLink(r1, r2 continuity.SymLink) bool {
+	return r1.Target() == r2.Target()
+}
+
+func compareDirectory(r1, r2 continuity.Directory) bool {
+	return true
+}
+
+func compareNamedPipe(r1, r2 continuity.NamedPipe) bool {
+	return true
+}
+
+func compareDevice(r1, r2 continuity.Device) bool {
+	return r1.Major() == r2.Major() && r1.Minor() == r2.Minor()
+}

fs/fstest/file.go

@@ -0,0 +1,109 @@
+package fstest
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/stevvooe/continuity/sysx"
+)
+
+// Applier applies single file changes
+type Applier func(root string) error
+
+// NewTestFile returns a file applier which creates a file as the
+// provided name with the given content and permission.
+func NewTestFile(name string, content []byte, perm os.FileMode) Applier {
+	return func(root string) error {
+		fullPath := filepath.Join(root, name)
+		if err := ioutil.WriteFile(fullPath, content, perm); err != nil {
+			return err
+		}
+
+		if err := os.Chmod(fullPath, perm); err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+// RemoveFile returns a file applier which removes the provided file name
+func RemoveFile(name string) Applier {
+	return func(root string) error {
+		return os.RemoveAll(filepath.Join(root, name))
+	}
+}
+
+// CreateDirectory returns a file applier to create the directory with
+// the provided name and permission
+func CreateDirectory(name string, perm os.FileMode) Applier {
+	return func(root string) error {
+		fullPath := filepath.Join(root, name)
+		if err := os.MkdirAll(fullPath, perm); err != nil {
+			return err
+		}
+		return nil
+	}
+}
+
+// Rename returns a file applier which renames a file
+func Rename(old, new string) Applier {
+	return func(root string) error {
+		return os.Rename(filepath.Join(root, old), filepath.Join(root, new))
+	}
+}
+
+// Chown returns a file applier which changes the ownership of a file
+func Chown(name string, uid, gid int) Applier {
+	return func(root string) error {
+		return os.Chown(filepath.Join(root, name), uid, gid)
+	}
+}
+
+// Chtime changes access and mod time of file
+func Chtime(name string, t time.Time) Applier {
+	return func(root string) error {
+		return os.Chtimes(filepath.Join(root, name), t, t)
+	}
+}
+
+// Symlink returns a file applier which creates a symbolic link
+func Symlink(oldname, newname string) Applier {
+	return func(root string) error {
+		return os.Symlink(oldname, filepath.Join(root, newname))
+	}
+}
+
+// Link returns a file applier which creates a hard link
+func Link(oldname, newname string) Applier {
+	return func(root string) error {
+		return os.Link(filepath.Join(root, oldname), filepath.Join(root, newname))
+	}
+}
+
+func SetXAttr(name, key, value string) Applier {
+	return func(root string) error {
+		return sysx.LSetxattr(name, key, []byte(value), 0)
+	}
+}
+
+// TODO: Make platform specific, windows applier is always no-op
+//func Mknod(name string, mode int32, dev int) Applier {
+//	return func(root string) error {
+//		return return syscall.Mknod(path, mode, dev)
+//	}
+//}
+
+// MultiApply returns a new applier from the given appliers
+func MultiApply(appliers ...Applier) Applier {
+	return func(root string) error {
+		for _, a := range appliers {
+			if err := a(root); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}

fs/hardlink.go

@@ -0,0 +1,12 @@
+package fs
+
+import "os"
+
+// GetLinkSource returns a path for the given name and
+// file info to its link source in the provided inode
+// map. If the given file name is not in the map and
+// has other links, it is added to the inode map
+// to be a source for other link locations.
+func GetLinkSource(name string, fi os.FileInfo, inodes map[uint64]string) (string, error) {
+	return getHardLink(name, fi, inodes)
+}

fs/hardlink_unix.go

@@ -0,0 +1,33 @@
+// +build !windows
+
+package fs
+
+import (
+	"errors"
+	"os"
+	"syscall"
+)
+
+func getHardLink(name string, fi os.FileInfo, inodes map[uint64]string) (string, error) {
+	if fi.IsDir() {
+		return "", nil
+	}
+
+	s, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return "", errors.New("unsupported stat type")
+	}
+
+	// If inode is not hardlinked, no reason to lookup or save inode
+	if s.Nlink == 1 {
+		return "", nil
+	}
+
+	inode := uint64(s.Ino)
+
+	path, ok := inodes[inode]
+	if !ok {
+		inodes[inode] = name
+	}
+	return path, nil
+}

fs/hardlink_windows.go

@@ -0,0 +1,7 @@
+package fs
+
+import "os"
+
+func getHardLink(string, os.FileInfo, map[uint64]string) (string, error) {
+	return "", nil
+}

fs/path.go

@@ -0,0 +1,197 @@
+package fs
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type currentPath struct {
+	path     string
+	f        os.FileInfo
+	fullPath string
+}
+
+func pathChange(lower, upper *currentPath) Change {
+	if lower == nil {
+		if upper == nil {
+			panic("cannot compare nil paths")
+		}
+		return Change{
+			Kind: ChangeKindAdd,
+			Path: upper.path,
+		}
+	}
+	if upper == nil {
+		return Change{
+			Kind: ChangeKindDelete,
+			Path: lower.path,
+		}
+	}
+	// TODO: compare by directory
+
+	switch i := strings.Compare(lower.path, upper.path); {
+	case i < 0:
+		// File in lower that is not in upper
+		return Change{
+			Kind: ChangeKindDelete,
+			Path: lower.path,
+		}
+	case i > 0:
+		// File in upper that is not in lower
+		return Change{
+			Kind: ChangeKindAdd,
+			Path: upper.path,
+		}
+	default:
+		return Change{
+			Kind: ChangeKindModify,
+			Path: upper.path,
+		}
+	}
+}
+
+func sameFile(f1, f2 *currentPath) (bool, error) {
+	if os.SameFile(f1.f, f2.f) {
+		return true, nil
+	}
+
+	equalStat, err := compareSysStat(f1.f.Sys(), f2.f.Sys())
+	if err != nil || !equalStat {
+		return equalStat, err
+	}
+
+	if eq, err := compareCapabilities(f1.fullPath, f2.fullPath); err != nil || !eq {
+		return eq, err
+	}
+
+	// If not a directory also check size, modtime, and content
+	if !f1.f.IsDir() {
+		if f1.f.Size() != f2.f.Size() {
+			return false, nil
+		}
+		t1 := f1.f.ModTime()
+		t2 := f2.f.ModTime()
+
+		if t1.Unix() != t2.Unix() {
+			return false, nil
+		}
+
+		// If the timestamp may have been truncated in one of the
+		// files, check content of file to determine difference
+		if t1.Nanosecond() == 0 || t2.Nanosecond() == 0 {
+			if f1.f.Size() > 0 {
+				eq, err := compareFileContent(f1.fullPath, f2.fullPath)
+				if err != nil || !eq {
+					return eq, err
+				}
+			}
+		} else if t1.Nanosecond() != t2.Nanosecond() {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+const compareChuckSize = 32 * 1024
+
+func compareFileContent(p1, p2 string) (bool, error) {
+	f1, err := os.Open(p1)
+	if err != nil {
+		return false, err
+	}
+	defer f1.Close()
+	f2, err := os.Open(p2)
+	if err != nil {
+		return false, err
+	}
+	defer f2.Close()
+
+	b1 := make([]byte, compareChuckSize)
+	b2 := make([]byte, compareChuckSize)
+	for {
+		n1, err1 := f1.Read(b1)
+		if err1 != nil && err1 != io.EOF {
+			return false, err1
+		}
+		n2, err2 := f2.Read(b2)
+		if err2 != nil && err2 != io.EOF {
+			return false, err2
+		}
+		if n1 != n2 || !bytes.Equal(b1[:n1], b2[:n2]) {
+			return false, nil
+		}
+		if err1 == io.EOF && err2 == io.EOF {
+			return true, nil
+		}
+	}
+}
+
+type walker struct {
+	pathC <-chan *currentPath
+	errC  <-chan error
+}
+
+func pathWalker(ctx context.Context, root string) *walker {
+	var (
+		pathC = make(chan *currentPath)
+		errC  = make(chan error, 1)
+	)
+	go func() {
+		defer close(pathC)
+		err := filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			// Rebase path
+			path, err = filepath.Rel(root, path)
+			if err != nil {
+				return err
+			}
+
+			path = filepath.Join(string(os.PathSeparator), path)
+
+			// Skip root
+			if path == string(os.PathSeparator) {
+				return nil
+			}
+
+			return sendPath(ctx, pathC, &currentPath{
+				path:     path,
+				f:        f,
+				fullPath: filepath.Join(root, path),
+			})
+		})
+		if err != nil {
+			errC <- err
+		}
+	}()
+
+	return &walker{
+		pathC: pathC,
+		errC:  errC,
+	}
+}
+
+func sendPath(ctx context.Context, pc chan<- *currentPath, p *currentPath) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case pc <- p:
+		return nil
+	}
+}
+
+func nextPath(w *walker) (*currentPath, error) {
+	select {
+	case err := <-w.errC:
+		return nil, err
+	case p := <-w.pathC:
+		return p, nil
+	}
+}

fs/time.go

@@ -0,0 +1,21 @@
+package fs
+
+import (
+	"syscall"
+	"time"
+)
+
+// Gnu tar and the go tar writer don't have sub-second mtime
+// precision, which is problematic when we apply changes via tar
+// files, we handle this by comparing for exact times, *or* same
+// second count and either a or b having exactly 0 nanoseconds
+func sameFsTime(a, b time.Time) bool {
+	return a == b ||
+		(a.Unix() == b.Unix() &&
+			(a.Nanosecond() == 0 || b.Nanosecond() == 0))
+}
+
+func sameFsTimeSpec(a, b syscall.Timespec) bool {
+	return a.Sec == b.Sec &&
+		(a.Nsec == b.Nsec || a.Nsec == 0 || b.Nsec == 0)
+}