Add fs package
Add diff comparison with support for double walking two trees for comparison or single walking a diff tree. Single walking requires further implementation for specific mount types. Add directory copy function which is intended to provide fastest possible local copy of file system directories without hardlinking. Add test package to make creating filesystems for test easy and comparisons deep and informative. Signed-off-by: Derek McGowan <derek@mcgstyle.net> (github: dmcgowan)
This commit is contained in:
Derek McGowan
parent
5f08e609c0
commit
574862fd89
16 changed files with 1672 additions and 0 deletions
360
fs/diff.go
Normal file
360
fs/diff.go
Normal file
|
|
@ -0,0 +1,360 @@
|
|||
package fs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
)
|
||||
|
||||
// ChangeKind is the type of modification that
|
||||
// a change is making.
|
||||
type ChangeKind int
|
||||
|
||||
const (
|
||||
// ChangeKindAdd represents an addition of
|
||||
// a file
|
||||
ChangeKindAdd = iota
|
||||
|
||||
// ChangeKindModify represents a change to
|
||||
// an existing file
|
||||
ChangeKindModify
|
||||
|
||||
// ChangeKindDelete represents a delete of
|
||||
// a file
|
||||
ChangeKindDelete
|
||||
)
|
||||
|
||||
func (k ChangeKind) String() string {
|
||||
switch k {
|
||||
case ChangeKindAdd:
|
||||
return "add"
|
||||
case ChangeKindModify:
|
||||
return "modify"
|
||||
case ChangeKindDelete:
|
||||
return "delete"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// Change represents single change between a diff and its parent.
|
||||
type Change struct {
|
||||
Kind ChangeKind
|
||||
Path string
|
||||
FileInfo os.FileInfo
|
||||
Source string
|
||||
}
|
||||
|
||||
// Changes returns a stream of changes between the provided upper
|
||||
// directory and lower directory.
|
||||
//
|
||||
// Changes are ordered by name and should be appliable in the
|
||||
// order in which they received.
|
||||
// Due to this apply ordering, the following is true
|
||||
// - Removed directory trees only create a single change for the root
|
||||
// directory removed. Remaining changes are implied.
|
||||
// - A directory which is modified to become a file will not have
|
||||
// delete entries for sub-path items, their removal is implied
|
||||
// by the removal of the parent directory.
|
||||
//
|
||||
// Opaque directories will not be treated specially and each file
|
||||
// removed from the lower will show up as a removal
|
||||
//
|
||||
// File content comparisons will be done on files which have timestamps
|
||||
// which may have been truncated. If either of the files being compared
|
||||
// has a zero value nanosecond value, each byte will be compared for
|
||||
// differences. If 2 files have the same seconds value but different
|
||||
// nanosecond values where one of those values is zero, the files will
|
||||
// be considered unchanged if the content is the same. This behavior
|
||||
// is to account for timestamp truncation during archiving.
|
||||
func Changes(ctx context.Context, upper, lower string) (context.Context, <-chan Change) {
|
||||
var (
|
||||
changes = make(chan Change)
|
||||
retCtx, cancel = context.WithCancel(ctx)
|
||||
)
|
||||
|
||||
cc := &changeContext{
|
||||
Context: retCtx,
|
||||
}
|
||||
|
||||
go func() {
|
||||
var err error
|
||||
if lower == "" {
|
||||
logrus.Debugf("Using single walk diff for %s", upper)
|
||||
err = addDirChanges(ctx, changes, upper)
|
||||
} else if diffOptions := detectDirDiff(upper, lower); diffOptions != nil {
|
||||
logrus.Debugf("Using single walk diff for %s from %s", diffOptions.diffDir, lower)
|
||||
err = diffDirChanges(ctx, changes, lower, diffOptions)
|
||||
} else {
|
||||
logrus.Debugf("Using double walk diff for %s from %s", upper, lower)
|
||||
err = doubleWalkDiff(ctx, changes, upper, lower)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
cc.errL.Lock()
|
||||
cc.err = err
|
||||
cc.errL.Unlock()
|
||||
cancel()
|
||||
}
|
||||
defer close(changes)
|
||||
}()
|
||||
|
||||
return cc, changes
|
||||
}
|
||||
|
||||
// changeContext wraps a context to allow setting an error
|
||||
// directly from a change streamer to allow streams canceled
|
||||
// due to errors to propagate the error to the caller.
|
||||
type changeContext struct {
|
||||
context.Context
|
||||
|
||||
err error
|
||||
errL sync.Mutex
|
||||
}
|
||||
|
||||
func (cc *changeContext) Err() error {
|
||||
cc.errL.Lock()
|
||||
if cc.err != nil {
|
||||
return cc.err
|
||||
}
|
||||
cc.errL.Unlock()
|
||||
return cc.Context.Err()
|
||||
}
|
||||
|
||||
func sendChange(ctx context.Context, changes chan<- Change, change Change) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case changes <- change:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func addDirChanges(ctx context.Context, changes chan<- Change, root string) error {
|
||||
return filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Rebase path
|
||||
path, err = filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
path = filepath.Join(string(os.PathSeparator), path)
|
||||
|
||||
// Skip root
|
||||
if path == string(os.PathSeparator) {
|
||||
return nil
|
||||
}
|
||||
|
||||
change := Change{
|
||||
Path: path,
|
||||
Kind: ChangeKindAdd,
|
||||
FileInfo: f,
|
||||
Source: filepath.Join(root, path),
|
||||
}
|
||||
|
||||
return sendChange(ctx, changes, change)
|
||||
})
|
||||
}
|
||||
|
||||
// diffDirOptions is used when the diff can be directly calculated from
|
||||
// a diff directory to its lower, without walking both trees.
|
||||
type diffDirOptions struct {
|
||||
diffDir string
|
||||
skipChange func(string) (bool, error)
|
||||
deleteChange func(string, string, os.FileInfo) (string, error)
|
||||
}
|
||||
|
||||
// diffDirChanges walks the diff directory and compares changes against the lower.
|
||||
func diffDirChanges(ctx context.Context, changes chan<- Change, lower string, o *diffDirOptions) error {
|
||||
changedDirs := make(map[string]struct{})
|
||||
return filepath.Walk(o.diffDir, func(path string, f os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Rebase path
|
||||
path, err = filepath.Rel(o.diffDir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
path = filepath.Join(string(os.PathSeparator), path)
|
||||
|
||||
// Skip root
|
||||
if path == string(os.PathSeparator) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO: handle opaqueness, start new double walker at this
|
||||
// location to get deletes, and skip tree in single walker
|
||||
|
||||
if o.skipChange != nil {
|
||||
if skip, err := o.skipChange(path); skip {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
change := Change{
|
||||
Path: path,
|
||||
}
|
||||
|
||||
deletedFile, err := o.deleteChange(o.diffDir, path, f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Find out what kind of modification happened
|
||||
if deletedFile != "" {
|
||||
change.Path = deletedFile
|
||||
change.Kind = ChangeKindDelete
|
||||
} else {
|
||||
// Otherwise, the file was added
|
||||
change.Kind = ChangeKindAdd
|
||||
change.FileInfo = f
|
||||
change.Source = filepath.Join(o.diffDir, path)
|
||||
|
||||
// ...Unless it already existed in a lower, in which case, it's a modification
|
||||
stat, err := os.Stat(filepath.Join(lower, path))
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
if err == nil {
|
||||
// The file existed in the lower, so that's a modification
|
||||
|
||||
// However, if it's a directory, maybe it wasn't actually modified.
|
||||
// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
|
||||
if stat.IsDir() && f.IsDir() {
|
||||
if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) {
|
||||
// Both directories are the same, don't record the change
|
||||
return nil
|
||||
}
|
||||
}
|
||||
change.Kind = ChangeKindModify
|
||||
}
|
||||
}
|
||||
|
||||
// If /foo/bar/file.txt is modified, then /foo/bar must be part of the changed files.
|
||||
// This block is here to ensure the change is recorded even if the
|
||||
// modify time, mode and size of the parent directory in the rw and ro layers are all equal.
|
||||
// Check https://github.com/docker/docker/pull/13590 for details.
|
||||
if f.IsDir() {
|
||||
changedDirs[path] = struct{}{}
|
||||
}
|
||||
if change.Kind == ChangeKindAdd || change.Kind == ChangeKindDelete {
|
||||
parent := filepath.Dir(path)
|
||||
if _, ok := changedDirs[parent]; !ok && parent != "/" {
|
||||
pi, err := os.Stat(filepath.Join(o.diffDir, parent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dirChange := Change{
|
||||
Path: parent,
|
||||
Kind: ChangeKindModify,
|
||||
FileInfo: pi,
|
||||
Source: filepath.Join(o.diffDir, parent),
|
||||
}
|
||||
if err := sendChange(ctx, changes, dirChange); err != nil {
|
||||
return err
|
||||
}
|
||||
changedDirs[parent] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
return sendChange(ctx, changes, change)
|
||||
})
|
||||
}
|
||||
|
||||
// doubleWalkDiff walks both directories to create a diff
|
||||
func doubleWalkDiff(ctx context.Context, changes chan<- Change, upper, lower string) (err error) {
|
||||
pathCtx, cancel := context.WithCancel(ctx)
|
||||
defer func() {
|
||||
if err != nil {
|
||||
cancel()
|
||||
}
|
||||
}()
|
||||
|
||||
var (
|
||||
w1 = pathWalker(pathCtx, lower)
|
||||
w2 = pathWalker(pathCtx, upper)
|
||||
f1, f2 *currentPath
|
||||
rmdir string
|
||||
)
|
||||
|
||||
for w1 != nil || w2 != nil {
|
||||
if f1 == nil && w1 != nil {
|
||||
f1, err = nextPath(w1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if f1 == nil {
|
||||
w1 = nil
|
||||
}
|
||||
}
|
||||
|
||||
if f2 == nil && w2 != nil {
|
||||
f2, err = nextPath(w2)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if f2 == nil {
|
||||
w2 = nil
|
||||
}
|
||||
}
|
||||
if f1 == nil && f2 == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
c := pathChange(f1, f2)
|
||||
switch c.Kind {
|
||||
case ChangeKindAdd:
|
||||
if rmdir != "" {
|
||||
rmdir = ""
|
||||
}
|
||||
c.FileInfo = f2.f
|
||||
c.Source = filepath.Join(upper, c.Path)
|
||||
f2 = nil
|
||||
case ChangeKindDelete:
|
||||
// Check if this file is already removed by being
|
||||
// under of a removed directory
|
||||
if rmdir != "" && strings.HasPrefix(f1.path, rmdir) {
|
||||
f1 = nil
|
||||
continue
|
||||
} else if rmdir == "" && f1.f.IsDir() {
|
||||
rmdir = f1.path + string(os.PathSeparator)
|
||||
} else if rmdir != "" {
|
||||
rmdir = ""
|
||||
}
|
||||
f1 = nil
|
||||
case ChangeKindModify:
|
||||
same, err := sameFile(f1, f2)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if f1.f.IsDir() && !f2.f.IsDir() {
|
||||
rmdir = f1.path + string(os.PathSeparator)
|
||||
} else if rmdir != "" {
|
||||
rmdir = ""
|
||||
}
|
||||
c.FileInfo = f2.f
|
||||
c.Source = filepath.Join(upper, c.Path)
|
||||
f1 = nil
|
||||
f2 = nil
|
||||
if same {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := sendChange(ctx, changes, c); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue