2017-01-31 23:17:33 +00:00
|
|
|
package fs
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"strings"
|
2017-02-03 00:37:04 +00:00
|
|
|
|
|
|
|
"golang.org/x/sync/errgroup"
|
2017-01-31 23:17:33 +00:00
|
|
|
|
|
|
|
"github.com/Sirupsen/logrus"
|
|
|
|
)
|
|
|
|
|
|
|
|
// ChangeKind is the type of modification that
|
|
|
|
// a change is making.
|
|
|
|
type ChangeKind int
|
|
|
|
|
|
|
|
const (
|
|
|
|
// ChangeKindAdd represents an addition of
|
|
|
|
// a file
|
|
|
|
ChangeKindAdd = iota
|
|
|
|
|
|
|
|
// ChangeKindModify represents a change to
|
|
|
|
// an existing file
|
|
|
|
ChangeKindModify
|
|
|
|
|
|
|
|
// ChangeKindDelete represents a delete of
|
|
|
|
// a file
|
|
|
|
ChangeKindDelete
|
|
|
|
)
|
|
|
|
|
|
|
|
func (k ChangeKind) String() string {
|
|
|
|
switch k {
|
|
|
|
case ChangeKindAdd:
|
|
|
|
return "add"
|
|
|
|
case ChangeKindModify:
|
|
|
|
return "modify"
|
|
|
|
case ChangeKindDelete:
|
|
|
|
return "delete"
|
|
|
|
default:
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Change represents single change between a diff and its parent.
|
|
|
|
type Change struct {
|
2017-02-03 00:37:04 +00:00
|
|
|
Kind ChangeKind
|
|
|
|
Path string
|
2017-01-31 23:17:33 +00:00
|
|
|
}
|
|
|
|
|
2017-02-03 00:37:04 +00:00
|
|
|
// Changes computes changes between lower and upper calling the
|
|
|
|
// given change function for each computed change. Callbacks
|
|
|
|
// will be done serialially and order by path name.
|
2017-01-31 23:17:33 +00:00
|
|
|
//
|
|
|
|
// Changes are ordered by name and should be appliable in the
|
|
|
|
// order in which they received.
|
|
|
|
// Due to this apply ordering, the following is true
|
|
|
|
// - Removed directory trees only create a single change for the root
|
|
|
|
// directory removed. Remaining changes are implied.
|
|
|
|
// - A directory which is modified to become a file will not have
|
|
|
|
// delete entries for sub-path items, their removal is implied
|
|
|
|
// by the removal of the parent directory.
|
|
|
|
//
|
|
|
|
// Opaque directories will not be treated specially and each file
|
|
|
|
// removed from the lower will show up as a removal
|
|
|
|
//
|
|
|
|
// File content comparisons will be done on files which have timestamps
|
|
|
|
// which may have been truncated. If either of the files being compared
|
|
|
|
// has a zero value nanosecond value, each byte will be compared for
|
|
|
|
// differences. If 2 files have the same seconds value but different
|
|
|
|
// nanosecond values where one of those values is zero, the files will
|
|
|
|
// be considered unchanged if the content is the same. This behavior
|
|
|
|
// is to account for timestamp truncation during archiving.
|
2017-02-03 00:37:04 +00:00
|
|
|
func Changes(ctx context.Context, upper, lower string, ch func(Change, os.FileInfo) error) error {
|
|
|
|
if lower == "" {
|
|
|
|
logrus.Debugf("Using single walk diff for %s", upper)
|
|
|
|
return addDirChanges(ctx, ch, upper)
|
|
|
|
} else if diffOptions := detectDirDiff(upper, lower); diffOptions != nil {
|
|
|
|
logrus.Debugf("Using single walk diff for %s from %s", diffOptions.diffDir, lower)
|
|
|
|
return diffDirChanges(ctx, ch, lower, diffOptions)
|
2017-01-31 23:17:33 +00:00
|
|
|
}
|
|
|
|
|
2017-02-03 00:37:04 +00:00
|
|
|
logrus.Debugf("Using double walk diff for %s from %s", upper, lower)
|
|
|
|
return doubleWalkDiff(ctx, ch, upper, lower)
|
2017-01-31 23:17:33 +00:00
|
|
|
}
|
|
|
|
|
2017-02-03 00:37:04 +00:00
|
|
|
type changeFn func(Change, os.FileInfo) error
|
2017-01-31 23:17:33 +00:00
|
|
|
|
2017-02-03 00:37:04 +00:00
|
|
|
func addDirChanges(ctx context.Context, changes changeFn, root string) error {
|
2017-01-31 23:17:33 +00:00
|
|
|
return filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Rebase path
|
|
|
|
path, err = filepath.Rel(root, path)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
path = filepath.Join(string(os.PathSeparator), path)
|
|
|
|
|
|
|
|
// Skip root
|
|
|
|
if path == string(os.PathSeparator) {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
change := Change{
|
2017-02-03 00:37:04 +00:00
|
|
|
Path: path,
|
|
|
|
Kind: ChangeKindAdd,
|
2017-01-31 23:17:33 +00:00
|
|
|
}
|
|
|
|
|
2017-02-03 00:37:04 +00:00
|
|
|
return changes(change, f)
|
2017-01-31 23:17:33 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
// diffDirOptions is used when the diff can be directly calculated from
|
|
|
|
// a diff directory to its lower, without walking both trees.
|
|
|
|
type diffDirOptions struct {
|
|
|
|
diffDir string
|
|
|
|
skipChange func(string) (bool, error)
|
|
|
|
deleteChange func(string, string, os.FileInfo) (string, error)
|
|
|
|
}
|
|
|
|
|
|
|
|
// diffDirChanges walks the diff directory and compares changes against the lower.
|
2017-02-03 00:37:04 +00:00
|
|
|
func diffDirChanges(ctx context.Context, changes changeFn, lower string, o *diffDirOptions) error {
|
2017-01-31 23:17:33 +00:00
|
|
|
changedDirs := make(map[string]struct{})
|
|
|
|
return filepath.Walk(o.diffDir, func(path string, f os.FileInfo, err error) error {
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Rebase path
|
|
|
|
path, err = filepath.Rel(o.diffDir, path)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
path = filepath.Join(string(os.PathSeparator), path)
|
|
|
|
|
|
|
|
// Skip root
|
|
|
|
if path == string(os.PathSeparator) {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// TODO: handle opaqueness, start new double walker at this
|
|
|
|
// location to get deletes, and skip tree in single walker
|
|
|
|
|
|
|
|
if o.skipChange != nil {
|
|
|
|
if skip, err := o.skipChange(path); skip {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
change := Change{
|
|
|
|
Path: path,
|
|
|
|
}
|
|
|
|
|
|
|
|
deletedFile, err := o.deleteChange(o.diffDir, path, f)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Find out what kind of modification happened
|
|
|
|
if deletedFile != "" {
|
|
|
|
change.Path = deletedFile
|
|
|
|
change.Kind = ChangeKindDelete
|
2017-02-03 00:37:04 +00:00
|
|
|
f = nil
|
2017-01-31 23:17:33 +00:00
|
|
|
} else {
|
|
|
|
// Otherwise, the file was added
|
|
|
|
change.Kind = ChangeKindAdd
|
|
|
|
|
|
|
|
// ...Unless it already existed in a lower, in which case, it's a modification
|
|
|
|
stat, err := os.Stat(filepath.Join(lower, path))
|
|
|
|
if err != nil && !os.IsNotExist(err) {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err == nil {
|
|
|
|
// The file existed in the lower, so that's a modification
|
|
|
|
|
|
|
|
// However, if it's a directory, maybe it wasn't actually modified.
|
|
|
|
// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
|
|
|
|
if stat.IsDir() && f.IsDir() {
|
|
|
|
if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) {
|
|
|
|
// Both directories are the same, don't record the change
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
change.Kind = ChangeKindModify
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// If /foo/bar/file.txt is modified, then /foo/bar must be part of the changed files.
|
|
|
|
// This block is here to ensure the change is recorded even if the
|
|
|
|
// modify time, mode and size of the parent directory in the rw and ro layers are all equal.
|
|
|
|
// Check https://github.com/docker/docker/pull/13590 for details.
|
|
|
|
if f.IsDir() {
|
|
|
|
changedDirs[path] = struct{}{}
|
|
|
|
}
|
|
|
|
if change.Kind == ChangeKindAdd || change.Kind == ChangeKindDelete {
|
|
|
|
parent := filepath.Dir(path)
|
|
|
|
if _, ok := changedDirs[parent]; !ok && parent != "/" {
|
|
|
|
pi, err := os.Stat(filepath.Join(o.diffDir, parent))
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
dirChange := Change{
|
2017-02-03 00:37:04 +00:00
|
|
|
Path: parent,
|
|
|
|
Kind: ChangeKindModify,
|
2017-01-31 23:17:33 +00:00
|
|
|
}
|
2017-02-03 00:37:04 +00:00
|
|
|
if err := changes(dirChange, pi); err != nil {
|
2017-01-31 23:17:33 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
changedDirs[parent] = struct{}{}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-03 00:37:04 +00:00
|
|
|
return changes(change, f)
|
2017-01-31 23:17:33 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
// doubleWalkDiff walks both directories to create a diff
|
2017-02-03 00:37:04 +00:00
|
|
|
func doubleWalkDiff(ctx context.Context, changes changeFn, upper, lower string) (err error) {
|
|
|
|
g, ctx := errgroup.WithContext(ctx)
|
2017-01-31 23:17:33 +00:00
|
|
|
|
|
|
|
var (
|
2017-02-03 00:37:04 +00:00
|
|
|
c1 = make(chan *currentPath)
|
|
|
|
c2 = make(chan *currentPath)
|
|
|
|
|
2017-01-31 23:17:33 +00:00
|
|
|
f1, f2 *currentPath
|
|
|
|
rmdir string
|
|
|
|
)
|
2017-02-03 00:37:04 +00:00
|
|
|
g.Go(func() error {
|
|
|
|
defer close(c1)
|
|
|
|
return pathWalk(ctx, lower, c1)
|
|
|
|
})
|
|
|
|
g.Go(func() error {
|
|
|
|
defer close(c2)
|
|
|
|
return pathWalk(ctx, upper, c2)
|
|
|
|
})
|
|
|
|
g.Go(func() error {
|
|
|
|
for c1 != nil || c2 != nil {
|
|
|
|
if f1 == nil && c1 != nil {
|
|
|
|
f1, err = nextPath(ctx, c1)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if f1 == nil {
|
|
|
|
c1 = nil
|
|
|
|
}
|
2017-01-31 23:17:33 +00:00
|
|
|
}
|
|
|
|
|
2017-02-03 00:37:04 +00:00
|
|
|
if f2 == nil && c2 != nil {
|
|
|
|
f2, err = nextPath(ctx, c2)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if f2 == nil {
|
|
|
|
c2 = nil
|
|
|
|
}
|
2017-01-31 23:17:33 +00:00
|
|
|
}
|
2017-02-03 00:37:04 +00:00
|
|
|
if f1 == nil && f2 == nil {
|
|
|
|
continue
|
2017-01-31 23:17:33 +00:00
|
|
|
}
|
|
|
|
|
2017-02-03 00:37:04 +00:00
|
|
|
var f os.FileInfo
|
|
|
|
c := pathChange(f1, f2)
|
|
|
|
switch c.Kind {
|
|
|
|
case ChangeKindAdd:
|
|
|
|
if rmdir != "" {
|
|
|
|
rmdir = ""
|
|
|
|
}
|
|
|
|
f = f2.f
|
|
|
|
f2 = nil
|
|
|
|
case ChangeKindDelete:
|
|
|
|
// Check if this file is already removed by being
|
|
|
|
// under of a removed directory
|
|
|
|
if rmdir != "" && strings.HasPrefix(f1.path, rmdir) {
|
|
|
|
f1 = nil
|
|
|
|
continue
|
|
|
|
} else if rmdir == "" && f1.f.IsDir() {
|
|
|
|
rmdir = f1.path + string(os.PathSeparator)
|
|
|
|
} else if rmdir != "" {
|
|
|
|
rmdir = ""
|
|
|
|
}
|
2017-01-31 23:17:33 +00:00
|
|
|
f1 = nil
|
2017-02-03 00:37:04 +00:00
|
|
|
case ChangeKindModify:
|
|
|
|
same, err := sameFile(f1, f2)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if f1.f.IsDir() && !f2.f.IsDir() {
|
|
|
|
rmdir = f1.path + string(os.PathSeparator)
|
|
|
|
} else if rmdir != "" {
|
|
|
|
rmdir = ""
|
|
|
|
}
|
|
|
|
f = f2.f
|
|
|
|
f1 = nil
|
|
|
|
f2 = nil
|
|
|
|
if same {
|
|
|
|
continue
|
|
|
|
}
|
2017-01-31 23:17:33 +00:00
|
|
|
}
|
2017-02-03 00:37:04 +00:00
|
|
|
if err := changes(c, f); err != nil {
|
2017-01-31 23:17:33 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
2017-02-03 00:37:04 +00:00
|
|
|
return nil
|
|
|
|
})
|
2017-01-31 23:17:33 +00:00
|
|
|
|
2017-02-03 00:37:04 +00:00
|
|
|
return g.Wait()
|
2017-01-31 23:17:33 +00:00
|
|
|
}
|