diff --git a/fs/copy.go b/fs/copy.go new file mode 100644 index 0000000..0ffd7f2 --- /dev/null +++ b/fs/copy.go @@ -0,0 +1,106 @@ +package fs + +import ( + "io/ioutil" + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +// CopyDirectory copies the directory from src to dst. +// Most efficient copy of files is attempted. +func CopyDirectory(dst, src string) error { + inodes := map[uint64]string{} + return copyDirectory(dst, src, inodes) +} + +func copyDirectory(dst, src string, inodes map[uint64]string) error { + stat, err := os.Stat(src) + if err != nil { + return errors.Wrapf(err, "failed to stat %s", src) + } + if !stat.IsDir() { + return errors.Errorf("source is not directory") + } + + if _, err := os.Stat(dst); err != nil { + if err := os.Mkdir(dst, stat.Mode()); err != nil { + return errors.Wrapf(err, "failed to mkdir %s", dst) + } + } else { + if err := os.Chmod(dst, stat.Mode()); err != nil { + return errors.Wrapf(err, "failed to chmod on %s", dst) + } + } + + fis, err := ioutil.ReadDir(src) + if err != nil { + return errors.Wrapf(err, "failed to read %s", src) + } + + if err := copyFileInfo(stat, dst); err != nil { + return errors.Wrapf(err, "failed to copy file info for %s", dst) + } + + for _, fi := range fis { + source := filepath.Join(src, fi.Name()) + target := filepath.Join(dst, fi.Name()) + if fi.IsDir() { + if err := copyDirectory(target, source, inodes); err != nil { + return err + } + continue + } else if (fi.Mode() & os.ModeType) == 0 { + link, err := GetLinkSource(target, fi, inodes) + if err != nil { + return errors.Wrap(err, "failed to get hardlink") + } + if link != "" { + if err := os.Link(link, target); err != nil { + return errors.Wrap(err, "failed to create hard link") + } + } else if err := copyFile(source, target); err != nil { + return errors.Wrap(err, "failed to copy files") + } + } else if (fi.Mode() & os.ModeSymlink) == os.ModeSymlink { + link, err := os.Readlink(source) + if err != nil { + return errors.Wrapf(err, "failed to read link: %s", source) + } + if err := os.Symlink(link, target); err != nil { + return errors.Wrapf(err, "failed to create symlink: %s", target) + } + } else if (fi.Mode() & os.ModeDevice) == os.ModeDevice { + // TODO: support devices + return errors.New("devices not supported") + } else { + // TODO: Support pipes and sockets + return errors.Wrapf(err, "unsupported mode %s", fi.Mode()) + } + if err := copyFileInfo(fi, target); err != nil { + return errors.Wrap(err, "failed to copy file info") + } + + if err := copyXAttrs(target, source); err != nil { + return errors.Wrap(err, "failed to copy xattrs") + } + } + + return nil +} + +func copyFile(source, target string) error { + src, err := os.Open(source) + if err != nil { + return errors.Wrapf(err, "failed to open source %s", err) + } + defer src.Close() + tgt, err := os.Create(target) + if err != nil { + return errors.Wrapf(err, "failed to open target %s", err) + } + defer tgt.Close() + + return copyFileContent(tgt, src) +} diff --git a/fs/copy_linux.go b/fs/copy_linux.go new file mode 100644 index 0000000..7ecd574 --- /dev/null +++ b/fs/copy_linux.go @@ -0,0 +1,110 @@ +package fs + +import ( + "io" + "os" + "strconv" + "syscall" + + "github.com/pkg/errors" + "github.com/stevvooe/continuity/sysx" +) + +var ( + kernel, major int +) + +func init() { + uts := &syscall.Utsname{} + + err := syscall.Uname(uts) + if err != nil { + panic(err) + } + + p := [2][]byte{{}, {}} + + release := uts.Release + i := 0 + for pi := 0; pi < len(p); pi++ { + for release[i] != 0 { + c := byte(release[i]) + i++ + if c == '.' || c == '-' { + break + } + p[pi] = append(p[pi], c) + } + } + kernel, err = strconv.Atoi(string(p[0])) + if err != nil { + panic(err) + } + major, err = strconv.Atoi(string(p[1])) + if err != nil { + panic(err) + } +} + +func copyFileInfo(fi os.FileInfo, name string) error { + st := fi.Sys().(*syscall.Stat_t) + if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil { + return errors.Wrapf(err, "failed to chown %s", name) + } + + if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink { + if err := os.Chmod(name, fi.Mode()); err != nil { + return errors.Wrapf(err, "failed to chmod %s", name) + } + } + + if err := syscall.UtimesNano(name, []syscall.Timespec{st.Atim, st.Mtim}); err != nil { + return errors.Wrapf(err, "failed to utime %s", name) + } + + return nil +} + +func copyFileContent(dst, src *os.File) error { + if checkKernel(4, 5) { + // Use copy_file_range to do in kernel copying + // See https://lwn.net/Articles/659523/ + // 326 on x86_64 + st, err := src.Stat() + if err != nil { + return errors.Wrap(err, "unable to stat source") + } + n, _, e1 := syscall.Syscall6(326, src.Fd(), 0, dst.Fd(), 0, uintptr(st.Size()), 0) + if e1 != 0 { + return errors.Wrap(err, "copy_file_range failed") + } + if int64(n) != st.Size() { + return errors.Wrapf(err, "short copy: %d of %d", int64(n), st.Size()) + } + return nil + } + _, err := io.Copy(dst, src) + return err +} + +func checkKernel(k, m int) bool { + return (kernel == k && major >= m) || kernel > k +} + +func copyXAttrs(dst, src string) error { + xattrKeys, err := sysx.LListxattr(src) + if err != nil { + return errors.Wrapf(err, "failed to list xattrs on %s", src) + } + for _, xattr := range xattrKeys { + data, err := sysx.LGetxattr(src, xattr) + if err != nil { + return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src) + } + if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil { + return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst) + } + } + + return nil +} diff --git a/fs/copy_test.go b/fs/copy_test.go new file mode 100644 index 0000000..1c1f7c5 --- /dev/null +++ b/fs/copy_test.go @@ -0,0 +1,53 @@ +package fs + +import ( + "io/ioutil" + "testing" + + _ "crypto/sha256" + + "github.com/docker/containerd/fs/fstest" + "github.com/pkg/errors" +) + +// TODO: Create copy directory which requires privilege +// chown +// mknod +// setxattr fstest.SetXAttr("/home", "trusted.overlay.opaque", "y"), + +func TestCopyDirectory(t *testing.T) { + apply := fstest.MultiApply( + fstest.CreateDirectory("/etc/", 0755), + fstest.NewTestFile("/etc/hosts", []byte("localhost 127.0.0.1"), 0644), + fstest.Link("/etc/hosts", "/etc/hosts.allow"), + fstest.CreateDirectory("/usr/local/lib", 0755), + fstest.NewTestFile("/usr/local/lib/libnothing.so", []byte{0x00, 0x00}, 0755), + fstest.Symlink("libnothing.so", "/usr/local/lib/libnothing.so.2"), + fstest.CreateDirectory("/home", 0755), + ) + + if err := testCopy(apply); err != nil { + t.Fatalf("Copy test failed: %+v", err) + } +} + +func testCopy(apply fstest.Applier) error { + t1, err := ioutil.TempDir("", "test-copy-src-") + if err != nil { + return errors.Wrap(err, "failed to create temporary directory") + } + t2, err := ioutil.TempDir("", "test-copy-dst-") + if err != nil { + return errors.Wrap(err, "failed to create temporary directory") + } + + if err := apply(t1); err != nil { + return errors.Wrap(err, "failed to apply changes") + } + + if err := CopyDirectory(t2, t1); err != nil { + return errors.Wrap(err, "failed to copy") + } + + return fstest.CheckDirectoryEqual(t1, t2) +} diff --git a/fs/copy_windows.go b/fs/copy_windows.go new file mode 100644 index 0000000..47ff2c5 --- /dev/null +++ b/fs/copy_windows.go @@ -0,0 +1,27 @@ +package fs + +import ( + "io" + "os" + + "github.com/pkg/errors" +) + +func copyFileInfo(fi os.FileInfo, name string) error { + if err := os.Chmod(name, fi.Mode()); err != nil { + return errors.Wrapf(err, "failed to chmod %s", name) + } + + // TODO: copy windows specific metadata + + return nil +} + +func copyFileContent(dst, src *os.File) error { + _, err := io.Copy(dst, src) + return err +} + +func copyXAttrs(dst, src string) error { + return nil +} diff --git a/fs/diff.go b/fs/diff.go new file mode 100644 index 0000000..77d5732 --- /dev/null +++ b/fs/diff.go @@ -0,0 +1,360 @@ +package fs + +import ( + "context" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/Sirupsen/logrus" +) + +// ChangeKind is the type of modification that +// a change is making. +type ChangeKind int + +const ( + // ChangeKindAdd represents an addition of + // a file + ChangeKindAdd = iota + + // ChangeKindModify represents a change to + // an existing file + ChangeKindModify + + // ChangeKindDelete represents a delete of + // a file + ChangeKindDelete +) + +func (k ChangeKind) String() string { + switch k { + case ChangeKindAdd: + return "add" + case ChangeKindModify: + return "modify" + case ChangeKindDelete: + return "delete" + default: + return "" + } +} + +// Change represents single change between a diff and its parent. +type Change struct { + Kind ChangeKind + Path string + FileInfo os.FileInfo + Source string +} + +// Changes returns a stream of changes between the provided upper +// directory and lower directory. +// +// Changes are ordered by name and should be appliable in the +// order in which they received. +// Due to this apply ordering, the following is true +// - Removed directory trees only create a single change for the root +// directory removed. Remaining changes are implied. +// - A directory which is modified to become a file will not have +// delete entries for sub-path items, their removal is implied +// by the removal of the parent directory. +// +// Opaque directories will not be treated specially and each file +// removed from the lower will show up as a removal +// +// File content comparisons will be done on files which have timestamps +// which may have been truncated. If either of the files being compared +// has a zero value nanosecond value, each byte will be compared for +// differences. If 2 files have the same seconds value but different +// nanosecond values where one of those values is zero, the files will +// be considered unchanged if the content is the same. This behavior +// is to account for timestamp truncation during archiving. +func Changes(ctx context.Context, upper, lower string) (context.Context, <-chan Change) { + var ( + changes = make(chan Change) + retCtx, cancel = context.WithCancel(ctx) + ) + + cc := &changeContext{ + Context: retCtx, + } + + go func() { + var err error + if lower == "" { + logrus.Debugf("Using single walk diff for %s", upper) + err = addDirChanges(ctx, changes, upper) + } else if diffOptions := detectDirDiff(upper, lower); diffOptions != nil { + logrus.Debugf("Using single walk diff for %s from %s", diffOptions.diffDir, lower) + err = diffDirChanges(ctx, changes, lower, diffOptions) + } else { + logrus.Debugf("Using double walk diff for %s from %s", upper, lower) + err = doubleWalkDiff(ctx, changes, upper, lower) + } + + if err != nil { + cc.errL.Lock() + cc.err = err + cc.errL.Unlock() + cancel() + } + defer close(changes) + }() + + return cc, changes +} + +// changeContext wraps a context to allow setting an error +// directly from a change streamer to allow streams canceled +// due to errors to propagate the error to the caller. +type changeContext struct { + context.Context + + err error + errL sync.Mutex +} + +func (cc *changeContext) Err() error { + cc.errL.Lock() + if cc.err != nil { + return cc.err + } + cc.errL.Unlock() + return cc.Context.Err() +} + +func sendChange(ctx context.Context, changes chan<- Change, change Change) error { + select { + case <-ctx.Done(): + return ctx.Err() + case changes <- change: + return nil + } +} + +func addDirChanges(ctx context.Context, changes chan<- Change, root string) error { + return filepath.Walk(root, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(root, path) + if err != nil { + return err + } + + path = filepath.Join(string(os.PathSeparator), path) + + // Skip root + if path == string(os.PathSeparator) { + return nil + } + + change := Change{ + Path: path, + Kind: ChangeKindAdd, + FileInfo: f, + Source: filepath.Join(root, path), + } + + return sendChange(ctx, changes, change) + }) +} + +// diffDirOptions is used when the diff can be directly calculated from +// a diff directory to its lower, without walking both trees. +type diffDirOptions struct { + diffDir string + skipChange func(string) (bool, error) + deleteChange func(string, string, os.FileInfo) (string, error) +} + +// diffDirChanges walks the diff directory and compares changes against the lower. +func diffDirChanges(ctx context.Context, changes chan<- Change, lower string, o *diffDirOptions) error { + changedDirs := make(map[string]struct{}) + return filepath.Walk(o.diffDir, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(o.diffDir, path) + if err != nil { + return err + } + + path = filepath.Join(string(os.PathSeparator), path) + + // Skip root + if path == string(os.PathSeparator) { + return nil + } + + // TODO: handle opaqueness, start new double walker at this + // location to get deletes, and skip tree in single walker + + if o.skipChange != nil { + if skip, err := o.skipChange(path); skip { + return err + } + } + + change := Change{ + Path: path, + } + + deletedFile, err := o.deleteChange(o.diffDir, path, f) + if err != nil { + return err + } + + // Find out what kind of modification happened + if deletedFile != "" { + change.Path = deletedFile + change.Kind = ChangeKindDelete + } else { + // Otherwise, the file was added + change.Kind = ChangeKindAdd + change.FileInfo = f + change.Source = filepath.Join(o.diffDir, path) + + // ...Unless it already existed in a lower, in which case, it's a modification + stat, err := os.Stat(filepath.Join(lower, path)) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + // The file existed in the lower, so that's a modification + + // However, if it's a directory, maybe it wasn't actually modified. + // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar + if stat.IsDir() && f.IsDir() { + if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) { + // Both directories are the same, don't record the change + return nil + } + } + change.Kind = ChangeKindModify + } + } + + // If /foo/bar/file.txt is modified, then /foo/bar must be part of the changed files. + // This block is here to ensure the change is recorded even if the + // modify time, mode and size of the parent directory in the rw and ro layers are all equal. + // Check https://github.com/docker/docker/pull/13590 for details. + if f.IsDir() { + changedDirs[path] = struct{}{} + } + if change.Kind == ChangeKindAdd || change.Kind == ChangeKindDelete { + parent := filepath.Dir(path) + if _, ok := changedDirs[parent]; !ok && parent != "/" { + pi, err := os.Stat(filepath.Join(o.diffDir, parent)) + if err != nil { + return err + } + dirChange := Change{ + Path: parent, + Kind: ChangeKindModify, + FileInfo: pi, + Source: filepath.Join(o.diffDir, parent), + } + if err := sendChange(ctx, changes, dirChange); err != nil { + return err + } + changedDirs[parent] = struct{}{} + } + } + + return sendChange(ctx, changes, change) + }) +} + +// doubleWalkDiff walks both directories to create a diff +func doubleWalkDiff(ctx context.Context, changes chan<- Change, upper, lower string) (err error) { + pathCtx, cancel := context.WithCancel(ctx) + defer func() { + if err != nil { + cancel() + } + }() + + var ( + w1 = pathWalker(pathCtx, lower) + w2 = pathWalker(pathCtx, upper) + f1, f2 *currentPath + rmdir string + ) + + for w1 != nil || w2 != nil { + if f1 == nil && w1 != nil { + f1, err = nextPath(w1) + if err != nil { + return err + } + if f1 == nil { + w1 = nil + } + } + + if f2 == nil && w2 != nil { + f2, err = nextPath(w2) + if err != nil { + return err + } + if f2 == nil { + w2 = nil + } + } + if f1 == nil && f2 == nil { + continue + } + + c := pathChange(f1, f2) + switch c.Kind { + case ChangeKindAdd: + if rmdir != "" { + rmdir = "" + } + c.FileInfo = f2.f + c.Source = filepath.Join(upper, c.Path) + f2 = nil + case ChangeKindDelete: + // Check if this file is already removed by being + // under of a removed directory + if rmdir != "" && strings.HasPrefix(f1.path, rmdir) { + f1 = nil + continue + } else if rmdir == "" && f1.f.IsDir() { + rmdir = f1.path + string(os.PathSeparator) + } else if rmdir != "" { + rmdir = "" + } + f1 = nil + case ChangeKindModify: + same, err := sameFile(f1, f2) + if err != nil { + return err + } + if f1.f.IsDir() && !f2.f.IsDir() { + rmdir = f1.path + string(os.PathSeparator) + } else if rmdir != "" { + rmdir = "" + } + c.FileInfo = f2.f + c.Source = filepath.Join(upper, c.Path) + f1 = nil + f2 = nil + if same { + continue + } + } + if err := sendChange(ctx, changes, c); err != nil { + return err + } + } + + return nil +} diff --git a/fs/diff_linux.go b/fs/diff_linux.go new file mode 100644 index 0000000..b6bad84 --- /dev/null +++ b/fs/diff_linux.go @@ -0,0 +1,92 @@ +package fs + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/pkg/errors" + "github.com/stevvooe/continuity/sysx" +) + +// whiteouts are files with a special meaning for the layered filesystem. +// Docker uses AUFS whiteout files inside exported archives. In other +// filesystems these files are generated/handled on tar creation/extraction. + +// whiteoutPrefix prefix means file is a whiteout. If this is followed by a +// filename this means that file has been removed from the base layer. +const whiteoutPrefix = ".wh." + +// whiteoutMetaPrefix prefix means whiteout has a special meaning and is not +// for removing an actual file. Normally these files are excluded from exported +// archives. +const whiteoutMetaPrefix = whiteoutPrefix + whiteoutPrefix + +// whiteoutLinkDir is a directory AUFS uses for storing hardlink links to other +// layers. Normally these should not go into exported archives and all changed +// hardlinks should be copied to the top layer. +const whiteoutLinkDir = whiteoutMetaPrefix + "plnk" + +// whiteoutOpaqueDir file means directory has been made opaque - meaning +// readdir calls to this directory do not follow to lower layers. +const whiteoutOpaqueDir = whiteoutMetaPrefix + ".opq" + +// detectDirDiff returns diff dir options if a directory could +// be found in the mount info for upper which is the direct +// diff with the provided lower directory +func detectDirDiff(upper, lower string) *diffDirOptions { + // TODO: get mount options for upper + // TODO: detect AUFS + // TODO: detect overlay + return nil +} + +func aufsMetadataSkip(path string) (skip bool, err error) { + skip, err = filepath.Match(string(os.PathSeparator)+whiteoutMetaPrefix+"*", path) + if err != nil { + skip = true + } + return +} + +func aufsDeletedFile(root, path string, fi os.FileInfo) (string, error) { + f := filepath.Base(path) + + // If there is a whiteout, then the file was removed + if strings.HasPrefix(f, whiteoutPrefix) { + originalFile := f[len(whiteoutPrefix):] + return filepath.Join(filepath.Dir(path), originalFile), nil + } + + return "", nil +} + +// compareSysStat returns whether the stats are equivalent, +// whether the files are considered the same file, and +// an error +func compareSysStat(s1, s2 interface{}) (bool, error) { + ls1, ok := s1.(*syscall.Stat_t) + if !ok { + return false, nil + } + ls2, ok := s2.(*syscall.Stat_t) + if !ok { + return false, nil + } + + return ls1.Mode == ls2.Mode && ls1.Uid == ls2.Uid && ls1.Gid == ls2.Gid && ls1.Rdev == ls2.Rdev, nil +} + +func compareCapabilities(p1, p2 string) (bool, error) { + c1, err := sysx.LGetxattr(p1, "security.capability") + if err != nil && err != syscall.ENODATA { + return false, errors.Wrapf(err, "failed to get xattr for %s", p1) + } + c2, err := sysx.LGetxattr(p2, "security.capability") + if err != nil && err != syscall.ENODATA { + return false, errors.Wrapf(err, "failed to get xattr for %s", p2) + } + return bytes.Equal(c1, c2), nil +} diff --git a/fs/diff_test.go b/fs/diff_test.go new file mode 100644 index 0000000..255d347 --- /dev/null +++ b/fs/diff_test.go @@ -0,0 +1,304 @@ +package fs + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/docker/containerd/fs/fstest" + "github.com/pkg/errors" +) + +// TODO: Additional tests +// - capability test (requires privilege) +// - chown test (requires privilege) +// - symlink test +// - hardlink test + +func TestSimpleDiff(t *testing.T) { + l1 := fstest.MultiApply( + fstest.CreateDirectory("/etc", 0755), + fstest.NewTestFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644), + fstest.NewTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0644), + fstest.NewTestFile("/etc/unchanged", []byte("PATH=/usr/bin"), 0644), + fstest.NewTestFile("/etc/unexpected", []byte("#!/bin/sh"), 0644), + ) + l2 := fstest.MultiApply( + fstest.NewTestFile("/etc/hosts", []byte("mydomain 10.0.0.120"), 0644), + fstest.NewTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0666), + fstest.CreateDirectory("/root", 0700), + fstest.NewTestFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0644), + fstest.RemoveFile("/etc/unexpected"), + ) + diff := []Change{ + Modify("/etc/hosts"), + Modify("/etc/profile"), + Delete("/etc/unexpected"), + Add("/root"), + Add("/root/.bashrc"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +func TestDirectoryReplace(t *testing.T) { + l1 := fstest.MultiApply( + fstest.CreateDirectory("/dir1", 0755), + fstest.NewTestFile("/dir1/f1", []byte("#####"), 0644), + fstest.CreateDirectory("/dir1/f2", 0755), + fstest.NewTestFile("/dir1/f2/f3", []byte("#!/bin/sh"), 0644), + ) + l2 := fstest.MultiApply( + fstest.NewTestFile("/dir1/f11", []byte("#New file here"), 0644), + fstest.RemoveFile("/dir1/f2"), + fstest.NewTestFile("/dir1/f2", []byte("Now file"), 0666), + ) + diff := []Change{ + Add("/dir1/f11"), + Modify("/dir1/f2"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +func TestRemoveDirectoryTree(t *testing.T) { + l1 := fstest.MultiApply( + fstest.CreateDirectory("/dir1/dir2/dir3", 0755), + fstest.NewTestFile("/dir1/f1", []byte("f1"), 0644), + fstest.NewTestFile("/dir1/dir2/f2", []byte("f2"), 0644), + ) + l2 := fstest.MultiApply( + fstest.RemoveFile("/dir1"), + ) + diff := []Change{ + Delete("/dir1"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +func TestFileReplace(t *testing.T) { + l1 := fstest.MultiApply( + fstest.NewTestFile("/dir1", []byte("a file, not a directory"), 0644), + ) + l2 := fstest.MultiApply( + fstest.RemoveFile("/dir1"), + fstest.CreateDirectory("/dir1/dir2", 0755), + fstest.NewTestFile("/dir1/dir2/f1", []byte("also a file"), 0644), + ) + diff := []Change{ + Modify("/dir1"), + Add("/dir1/dir2"), + Add("/dir1/dir2/f1"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +func TestUpdateWithSameTime(t *testing.T) { + tt := time.Now().Truncate(time.Second) + t1 := tt.Add(5 * time.Nanosecond) + t2 := tt.Add(6 * time.Nanosecond) + l1 := fstest.MultiApply( + fstest.NewTestFile("/file-modified-time", []byte("1"), 0644), + fstest.Chtime("/file-modified-time", t1), + fstest.NewTestFile("/file-no-change", []byte("1"), 0644), + fstest.Chtime("/file-no-change", t1), + fstest.NewTestFile("/file-same-time", []byte("1"), 0644), + fstest.Chtime("/file-same-time", t1), + fstest.NewTestFile("/file-truncated-time-1", []byte("1"), 0644), + fstest.Chtime("/file-truncated-time-1", t1), + fstest.NewTestFile("/file-truncated-time-2", []byte("1"), 0644), + fstest.Chtime("/file-truncated-time-2", tt), + ) + l2 := fstest.MultiApply( + fstest.NewTestFile("/file-modified-time", []byte("2"), 0644), + fstest.Chtime("/file-modified-time", t2), + fstest.NewTestFile("/file-no-change", []byte("1"), 0644), + fstest.Chtime("/file-no-change", tt), // use truncated time, should be regarded as no change + fstest.NewTestFile("/file-same-time", []byte("2"), 0644), + fstest.Chtime("/file-same-time", t1), + fstest.NewTestFile("/file-truncated-time-1", []byte("2"), 0644), + fstest.Chtime("/file-truncated-time-1", tt), + fstest.NewTestFile("/file-truncated-time-2", []byte("2"), 0644), + fstest.Chtime("/file-truncated-time-2", tt), + ) + diff := []Change{ + // "/file-same-time" excluded because matching non-zero nanosecond values + Modify("/file-modified-time"), + Modify("/file-truncated-time-1"), + Modify("/file-truncated-time-2"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +func testDiffWithBase(base, diff fstest.Applier, expected []Change) error { + t1, err := ioutil.TempDir("", "diff-with-base-lower-") + if err != nil { + return errors.Wrap(err, "failed to create temp dir") + } + defer os.RemoveAll(t1) + t2, err := ioutil.TempDir("", "diff-with-base-upper-") + if err != nil { + return errors.Wrap(err, "failed to create temp dir") + } + defer os.RemoveAll(t2) + + if err := base(t1); err != nil { + return errors.Wrap(err, "failed to apply base filesytem") + } + + if err := CopyDirectory(t2, t1); err != nil { + return errors.Wrap(err, "failed to copy base directory") + } + + if err := diff(t2); err != nil { + return errors.Wrap(err, "failed to apply diff filesystem") + } + + changes, err := collectChanges(t2, t1) + if err != nil { + return errors.Wrap(err, "failed to collect changes") + } + + return checkChanges(t2, changes, expected) +} + +func TestBaseDirectoryChanges(t *testing.T) { + apply := fstest.MultiApply( + fstest.CreateDirectory("/etc", 0755), + fstest.NewTestFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644), + fstest.NewTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0644), + fstest.CreateDirectory("/root", 0700), + fstest.NewTestFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0644), + ) + changes := []Change{ + Add("/etc"), + Add("/etc/hosts"), + Add("/etc/profile"), + Add("/root"), + Add("/root/.bashrc"), + } + + if err := testDiffWithoutBase(apply, changes); err != nil { + t.Fatalf("Failed diff without base: %+v", err) + } +} + +func testDiffWithoutBase(apply fstest.Applier, expected []Change) error { + tmp, err := ioutil.TempDir("", "diff-without-base-") + if err != nil { + return errors.Wrap(err, "failed to create temp dir") + } + defer os.RemoveAll(tmp) + + if err := apply(tmp); err != nil { + return errors.Wrap(err, "failed to apply filesytem changes") + } + + changes, err := collectChanges(tmp, "") + if err != nil { + return errors.Wrap(err, "failed to collect changes") + } + + return checkChanges(tmp, changes, expected) +} + +func checkChanges(root string, changes, expected []Change) error { + if len(changes) != len(expected) { + return errors.Errorf("Unexpected number of changes:\n%s", diffString(changes, expected)) + } + for i := range changes { + if changes[i].Path != expected[i].Path || changes[i].Kind != expected[i].Kind { + return errors.Errorf("Unexpected change at %d:\n%s", i, diffString(changes, expected)) + } + if changes[i].Kind != ChangeKindDelete { + filename := filepath.Join(root, changes[i].Path) + efi, err := os.Stat(filename) + if err != nil { + return errors.Wrapf(err, "failed to stat %q", filename) + } + afi := changes[i].FileInfo + if afi.Size() != efi.Size() { + return errors.Errorf("Unexpected change size %d, %q has size %d", afi.Size(), filename, efi.Size()) + } + if afi.Mode() != efi.Mode() { + return errors.Errorf("Unexpected change mode %s, %q has mode %s", afi.Mode(), filename, efi.Mode()) + } + if afi.ModTime() != efi.ModTime() { + return errors.Errorf("Unexpected change modtime %s, %q has modtime %s", afi.ModTime(), filename, efi.ModTime()) + } + if expected := filepath.Join(root, changes[i].Path); changes[i].Source != expected { + return errors.Errorf("Unexpected source path %s, expected %s", changes[i].Source, expected) + } + } + } + + return nil +} + +func collectChanges(upper, lower string) ([]Change, error) { + ctx, changeC := Changes(context.Background(), upper, lower) + changes := []Change{} + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case c, ok := <-changeC: + if !ok { + return changes, nil + } + changes = append(changes, c) + } + } +} + +func diffString(c1, c2 []Change) string { + return fmt.Sprintf("got(%d):\n%s\nexpected(%d):\n%s", len(c1), changesString(c1), len(c2), changesString(c2)) + +} + +func changesString(c []Change) string { + strs := make([]string, len(c)) + for i := range c { + strs[i] = fmt.Sprintf("\t%s\t%s", c[i].Kind, c[i].Path) + } + return strings.Join(strs, "\n") +} + +func Add(p string) Change { + return Change{ + Kind: ChangeKindAdd, + Path: p, + } +} + +func Delete(p string) Change { + return Change{ + Kind: ChangeKindDelete, + Path: p, + } +} + +func Modify(p string) Change { + return Change{ + Kind: ChangeKindModify, + Path: p, + } +} diff --git a/fs/diff_windows.go b/fs/diff_windows.go new file mode 100644 index 0000000..90fb30f --- /dev/null +++ b/fs/diff_windows.go @@ -0,0 +1,15 @@ +package fs + +func detectDirDiff(upper, lower string) *diffDirOptions { + return nil +} + +func compareSysStat(s1, s2 interface{}) (bool, error) { + // TODO: Use windows specific sys type + return false, nil +} + +func compareCapabilities(p1, p2 string) (bool, error) { + // TODO: Use windows equivalent + return true, nil +} diff --git a/fs/fstest/compare.go b/fs/fstest/compare.go new file mode 100644 index 0000000..84781b3 --- /dev/null +++ b/fs/fstest/compare.go @@ -0,0 +1,37 @@ +package fstest + +import ( + "github.com/pkg/errors" + "github.com/stevvooe/continuity" +) + +// CheckDirectoryEqual compares two directory paths to make sure that +// the content of the directories is the same. +func CheckDirectoryEqual(d1, d2 string) error { + c1, err := continuity.NewContext(d1) + if err != nil { + return errors.Wrap(err, "failed to build context") + } + + c2, err := continuity.NewContext(d2) + if err != nil { + return errors.Wrap(err, "failed to build context") + } + + m1, err := continuity.BuildManifest(c1) + if err != nil { + return errors.Wrap(err, "failed to build manifest") + } + + m2, err := continuity.BuildManifest(c2) + if err != nil { + return errors.Wrap(err, "failed to build manifest") + } + + diff := diffResourceList(m1.Resources, m2.Resources) + if diff.HasDiff() { + return errors.Errorf("directory diff between %s and %s\n%s", d1, d2, diff.String()) + } + + return nil +} diff --git a/fs/fstest/continuity_util.go b/fs/fstest/continuity_util.go new file mode 100644 index 0000000..a300388 --- /dev/null +++ b/fs/fstest/continuity_util.go @@ -0,0 +1,189 @@ +package fstest + +import ( + "bytes" + "fmt" + + "github.com/stevvooe/continuity" +) + +type resourceUpdate struct { + Original continuity.Resource + Updated continuity.Resource +} + +func (u resourceUpdate) String() string { + return fmt.Sprintf("%s(mode: %o, uid: %s, gid: %s) -> %s(mode: %o, uid: %s, gid: %s)", + u.Original.Path(), u.Original.Mode(), u.Original.UID(), u.Original.GID(), + u.Updated.Path(), u.Updated.Mode(), u.Updated.UID(), u.Updated.GID(), + ) +} + +type resourceListDifference struct { + Additions []continuity.Resource + Deletions []continuity.Resource + Updates []resourceUpdate +} + +func (l resourceListDifference) HasDiff() bool { + return len(l.Additions) > 0 || len(l.Deletions) > 0 || len(l.Updates) > 0 +} + +func (l resourceListDifference) String() string { + buf := bytes.NewBuffer(nil) + for _, add := range l.Additions { + fmt.Fprintf(buf, "+ %s\n", add.Path()) + } + for _, del := range l.Deletions { + fmt.Fprintf(buf, "- %s\n", del.Path()) + } + for _, upt := range l.Updates { + fmt.Fprintf(buf, "~ %s\n", upt.String()) + } + return string(buf.Bytes()) +} + +// diffManifest compares two resource lists and returns the list +// of adds updates and deletes, resource lists are not reordered +// before doing difference. +func diffResourceList(r1, r2 []continuity.Resource) resourceListDifference { + i1 := 0 + i2 := 0 + var d resourceListDifference + + for i1 < len(r1) && i2 < len(r2) { + p1 := r1[i1].Path() + p2 := r2[i2].Path() + switch { + case p1 < p2: + d.Deletions = append(d.Deletions, r1[i1]) + i1++ + case p1 == p2: + if !compareResource(r1[i1], r2[i2]) { + d.Updates = append(d.Updates, resourceUpdate{ + Original: r1[i1], + Updated: r2[i2], + }) + } + i1++ + i2++ + case p1 > p2: + d.Additions = append(d.Additions, r2[i2]) + i2++ + } + } + + for i1 < len(r1) { + d.Deletions = append(d.Deletions, r1[i1]) + i1++ + + } + for i2 < len(r2) { + d.Additions = append(d.Additions, r2[i2]) + i2++ + } + + return d +} + +func compareResource(r1, r2 continuity.Resource) bool { + if r1.Path() != r2.Path() { + return false + } + if r1.Mode() != r2.Mode() { + return false + } + if r1.UID() != r2.UID() { + return false + } + if r1.GID() != r2.GID() { + return false + } + + // TODO(dmcgowan): Check if is XAttrer + + return compareResourceTypes(r1, r2) + +} + +func compareResourceTypes(r1, r2 continuity.Resource) bool { + switch t1 := r1.(type) { + case continuity.RegularFile: + t2, ok := r2.(continuity.RegularFile) + if !ok { + return false + } + return compareRegularFile(t1, t2) + case continuity.Directory: + t2, ok := r2.(continuity.Directory) + if !ok { + return false + } + return compareDirectory(t1, t2) + case continuity.SymLink: + t2, ok := r2.(continuity.SymLink) + if !ok { + return false + } + return compareSymLink(t1, t2) + case continuity.NamedPipe: + t2, ok := r2.(continuity.NamedPipe) + if !ok { + return false + } + return compareNamedPipe(t1, t2) + case continuity.Device: + t2, ok := r2.(continuity.Device) + if !ok { + return false + } + return compareDevice(t1, t2) + default: + // TODO(dmcgowan): Should this panic? + return r1 == r2 + } +} + +func compareRegularFile(r1, r2 continuity.RegularFile) bool { + if r1.Size() != r2.Size() { + return false + } + p1 := r1.Paths() + p2 := r2.Paths() + if len(p1) != len(p2) { + return false + } + for i := range p1 { + if p1[i] != p2[i] { + return false + } + } + d1 := r1.Digests() + d2 := r2.Digests() + if len(d1) != len(d2) { + return false + } + for i := range d1 { + if d1[i] != d2[i] { + return false + } + } + + return true +} + +func compareSymLink(r1, r2 continuity.SymLink) bool { + return r1.Target() == r2.Target() +} + +func compareDirectory(r1, r2 continuity.Directory) bool { + return true +} + +func compareNamedPipe(r1, r2 continuity.NamedPipe) bool { + return true +} + +func compareDevice(r1, r2 continuity.Device) bool { + return r1.Major() == r2.Major() && r1.Minor() == r2.Minor() +} diff --git a/fs/fstest/file.go b/fs/fstest/file.go new file mode 100644 index 0000000..640fad6 --- /dev/null +++ b/fs/fstest/file.go @@ -0,0 +1,109 @@ +package fstest + +import ( + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/stevvooe/continuity/sysx" +) + +// Applier applies single file changes +type Applier func(root string) error + +// NewTestFile returns a file applier which creates a file as the +// provided name with the given content and permission. +func NewTestFile(name string, content []byte, perm os.FileMode) Applier { + return func(root string) error { + fullPath := filepath.Join(root, name) + if err := ioutil.WriteFile(fullPath, content, perm); err != nil { + return err + } + + if err := os.Chmod(fullPath, perm); err != nil { + return err + } + + return nil + } +} + +// RemoveFile returns a file applier which removes the provided file name +func RemoveFile(name string) Applier { + return func(root string) error { + return os.RemoveAll(filepath.Join(root, name)) + } +} + +// CreateDirectory returns a file applier to create the directory with +// the provided name and permission +func CreateDirectory(name string, perm os.FileMode) Applier { + return func(root string) error { + fullPath := filepath.Join(root, name) + if err := os.MkdirAll(fullPath, perm); err != nil { + return err + } + return nil + } +} + +// Rename returns a file applier which renames a file +func Rename(old, new string) Applier { + return func(root string) error { + return os.Rename(filepath.Join(root, old), filepath.Join(root, new)) + } +} + +// Chown returns a file applier which changes the ownership of a file +func Chown(name string, uid, gid int) Applier { + return func(root string) error { + return os.Chown(filepath.Join(root, name), uid, gid) + } +} + +// Chtime changes access and mod time of file +func Chtime(name string, t time.Time) Applier { + return func(root string) error { + return os.Chtimes(filepath.Join(root, name), t, t) + } +} + +// Symlink returns a file applier which creates a symbolic link +func Symlink(oldname, newname string) Applier { + return func(root string) error { + return os.Symlink(oldname, filepath.Join(root, newname)) + } +} + +// Link returns a file applier which creates a hard link +func Link(oldname, newname string) Applier { + return func(root string) error { + return os.Link(filepath.Join(root, oldname), filepath.Join(root, newname)) + } +} + +func SetXAttr(name, key, value string) Applier { + return func(root string) error { + return sysx.LSetxattr(name, key, []byte(value), 0) + } +} + +// TODO: Make platform specific, windows applier is always no-op +//func Mknod(name string, mode int32, dev int) Applier { +// return func(root string) error { +// return return syscall.Mknod(path, mode, dev) +// } +//} + +// MultiApply returns a new applier from the given appliers +func MultiApply(appliers ...Applier) Applier { + return func(root string) error { + for _, a := range appliers { + if err := a(root); err != nil { + return err + } + } + return nil + } +} diff --git a/fs/hardlink.go b/fs/hardlink.go new file mode 100644 index 0000000..a6c256b --- /dev/null +++ b/fs/hardlink.go @@ -0,0 +1,12 @@ +package fs + +import "os" + +// GetLinkSource returns a path for the given name and +// file info to its link source in the provided inode +// map. If the given file name is not in the map and +// has other links, it is added to the inode map +// to be a source for other link locations. +func GetLinkSource(name string, fi os.FileInfo, inodes map[uint64]string) (string, error) { + return getHardLink(name, fi, inodes) +} diff --git a/fs/hardlink_unix.go b/fs/hardlink_unix.go new file mode 100644 index 0000000..76d1897 --- /dev/null +++ b/fs/hardlink_unix.go @@ -0,0 +1,33 @@ +// +build !windows + +package fs + +import ( + "errors" + "os" + "syscall" +) + +func getHardLink(name string, fi os.FileInfo, inodes map[uint64]string) (string, error) { + if fi.IsDir() { + return "", nil + } + + s, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return "", errors.New("unsupported stat type") + } + + // If inode is not hardlinked, no reason to lookup or save inode + if s.Nlink == 1 { + return "", nil + } + + inode := uint64(s.Ino) + + path, ok := inodes[inode] + if !ok { + inodes[inode] = name + } + return path, nil +} diff --git a/fs/hardlink_windows.go b/fs/hardlink_windows.go new file mode 100644 index 0000000..81e50ee --- /dev/null +++ b/fs/hardlink_windows.go @@ -0,0 +1,7 @@ +package fs + +import "os" + +func getHardLink(string, os.FileInfo, map[uint64]string) (string, error) { + return "", nil +} diff --git a/fs/path.go b/fs/path.go new file mode 100644 index 0000000..8abf4f1 --- /dev/null +++ b/fs/path.go @@ -0,0 +1,197 @@ +package fs + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "strings" +) + +type currentPath struct { + path string + f os.FileInfo + fullPath string +} + +func pathChange(lower, upper *currentPath) Change { + if lower == nil { + if upper == nil { + panic("cannot compare nil paths") + } + return Change{ + Kind: ChangeKindAdd, + Path: upper.path, + } + } + if upper == nil { + return Change{ + Kind: ChangeKindDelete, + Path: lower.path, + } + } + // TODO: compare by directory + + switch i := strings.Compare(lower.path, upper.path); { + case i < 0: + // File in lower that is not in upper + return Change{ + Kind: ChangeKindDelete, + Path: lower.path, + } + case i > 0: + // File in upper that is not in lower + return Change{ + Kind: ChangeKindAdd, + Path: upper.path, + } + default: + return Change{ + Kind: ChangeKindModify, + Path: upper.path, + } + } +} + +func sameFile(f1, f2 *currentPath) (bool, error) { + if os.SameFile(f1.f, f2.f) { + return true, nil + } + + equalStat, err := compareSysStat(f1.f.Sys(), f2.f.Sys()) + if err != nil || !equalStat { + return equalStat, err + } + + if eq, err := compareCapabilities(f1.fullPath, f2.fullPath); err != nil || !eq { + return eq, err + } + + // If not a directory also check size, modtime, and content + if !f1.f.IsDir() { + if f1.f.Size() != f2.f.Size() { + return false, nil + } + t1 := f1.f.ModTime() + t2 := f2.f.ModTime() + + if t1.Unix() != t2.Unix() { + return false, nil + } + + // If the timestamp may have been truncated in one of the + // files, check content of file to determine difference + if t1.Nanosecond() == 0 || t2.Nanosecond() == 0 { + if f1.f.Size() > 0 { + eq, err := compareFileContent(f1.fullPath, f2.fullPath) + if err != nil || !eq { + return eq, err + } + } + } else if t1.Nanosecond() != t2.Nanosecond() { + return false, nil + } + } + + return true, nil +} + +const compareChuckSize = 32 * 1024 + +func compareFileContent(p1, p2 string) (bool, error) { + f1, err := os.Open(p1) + if err != nil { + return false, err + } + defer f1.Close() + f2, err := os.Open(p2) + if err != nil { + return false, err + } + defer f2.Close() + + b1 := make([]byte, compareChuckSize) + b2 := make([]byte, compareChuckSize) + for { + n1, err1 := f1.Read(b1) + if err1 != nil && err1 != io.EOF { + return false, err1 + } + n2, err2 := f2.Read(b2) + if err2 != nil && err2 != io.EOF { + return false, err2 + } + if n1 != n2 || !bytes.Equal(b1[:n1], b2[:n2]) { + return false, nil + } + if err1 == io.EOF && err2 == io.EOF { + return true, nil + } + } +} + +type walker struct { + pathC <-chan *currentPath + errC <-chan error +} + +func pathWalker(ctx context.Context, root string) *walker { + var ( + pathC = make(chan *currentPath) + errC = make(chan error, 1) + ) + go func() { + defer close(pathC) + err := filepath.Walk(root, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(root, path) + if err != nil { + return err + } + + path = filepath.Join(string(os.PathSeparator), path) + + // Skip root + if path == string(os.PathSeparator) { + return nil + } + + return sendPath(ctx, pathC, ¤tPath{ + path: path, + f: f, + fullPath: filepath.Join(root, path), + }) + }) + if err != nil { + errC <- err + } + }() + + return &walker{ + pathC: pathC, + errC: errC, + } +} + +func sendPath(ctx context.Context, pc chan<- *currentPath, p *currentPath) error { + select { + case <-ctx.Done(): + return ctx.Err() + case pc <- p: + return nil + } +} + +func nextPath(w *walker) (*currentPath, error) { + select { + case err := <-w.errC: + return nil, err + case p := <-w.pathC: + return p, nil + } +} diff --git a/fs/time.go b/fs/time.go new file mode 100644 index 0000000..92966a3 --- /dev/null +++ b/fs/time.go @@ -0,0 +1,21 @@ +package fs + +import ( + "syscall" + "time" +) + +// Gnu tar and the go tar writer don't have sub-second mtime +// precision, which is problematic when we apply changes via tar +// files, we handle this by comparing for exact times, *or* same +// second count and either a or b having exactly 0 nanoseconds +func sameFsTime(a, b time.Time) bool { + return a == b || + (a.Unix() == b.Unix() && + (a.Nanosecond() == 0 || b.Nanosecond() == 0)) +} + +func sameFsTimeSpec(a, b syscall.Timespec) bool { + return a.Sec == b.Sec && + (a.Nsec == b.Nsec || a.Nsec == 0 || b.Nsec == 0) +}