vbatts/go-mtree
1
0
Fork 0
Code Issues Pull requests Releases Activity

walk: implement FsEval hooks

Browse source 
In certain circumstances (such as the manifest generation of a
filesystem as an unprivileged user) it is important to provide hooks
that override the default os.* implementation of filesystem-related
functions.

In order to avoid merging too much code from outside projects (such as
umoci) this is implemented by providing FsEval hooks to Walk() and
Check(). This allows for users of go-mtree to modify how filesystem
checks are done, without compromising the simplicity of go-mtree's code.

Signed-off-by: Aleksa Sarai <asarai@suse.de>
This commit is contained in:
Aleksa Sarai 2016-11-18 18:53:26 +11:00
parent 98824a87da
commit e22043cb86
No known key found for this signature in database
GPG key ID: 9E18AA267DDB8DB4
9 changed files with 110 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
check.go
View file

@ -5,13 +5,13 @@ package mtree
// If keywords is nil, the check all present in the DirectoryHierarchy
//
// This is equivalent to creating a new DirectoryHierarchy with Walk(root, nil,
// keywords) and then doing a Compare(dh, newDh, keywords).
func Check(root string, dh *DirectoryHierarchy, keywords []Keyword) ([]InodeDelta, error) {
// keywords, fs) and then doing a Compare(dh, newDh, keywords).
func Check(root string, dh *DirectoryHierarchy, keywords []Keyword, fs FsEval) ([]InodeDelta, error) {
if keywords == nil {
keywords = dh.UsedKeywords()
}
newDh, err := Walk(root, nil, keywords)
newDh, err := Walk(root, nil, keywords, fs)
if err != nil {
return nil, err
}

34
check_test.go
View file

@ -12,12 +12,12 @@ import (
// simple walk of current directory, and imediately check it.
// may not be parallelizable.
func TestCheck(t *testing.T) {
dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"))
dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
res, err := Check(".", dh, nil)
res, err := Check(".", dh, nil, nil)
if err != nil {
t.Fatal(err)
}
@ -43,13 +43,13 @@ func TestCheckKeywords(t *testing.T) {
}
// Walk this tempdir
dh, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
dh, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
// Check for sanity. This ought to pass.
res, err := Check(dir, dh, nil)
res, err := Check(dir, dh, nil, nil)
if err != nil {
t.Fatal(err)
}
@ -64,7 +64,7 @@ func TestCheckKeywords(t *testing.T) {
}
// Check again. This ought to fail.
res, err = Check(dir, dh, nil)
res, err = Check(dir, dh, nil, nil)
if err != nil {
t.Fatal(err)
}
@ -76,7 +76,7 @@ func TestCheckKeywords(t *testing.T) {
}
// Check again, but only sha1 and mode. This ought to pass.
res, err = Check(dir, dh, []Keyword{"sha1", "mode"})
res, err = Check(dir, dh, []Keyword{"sha1", "mode"}, nil)
if err != nil {
t.Fatal(err)
}
@ -86,12 +86,12 @@ func TestCheckKeywords(t *testing.T) {
}
func ExampleCheck() {
dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"))
dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
// handle error ...
}
res, err := Check(".", dh, nil)
res, err := Check(".", dh, nil, nil)
if err != nil {
// handle error ...
}
@ -103,11 +103,11 @@ func ExampleCheck() {
// Tests default action for evaluating a symlink, which is just to compare the
// link itself, not to follow it
func TestDefaultBrokenLink(t *testing.T) {
dh, err := Walk("./testdata/dirwithbrokenlink", nil, append(DefaultKeywords, "sha1"))
dh, err := Walk("./testdata/dirwithbrokenlink", nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
res, err := Check("./testdata/dirwithbrokenlink", dh, nil)
res, err := Check("./testdata/dirwithbrokenlink", dh, nil, nil)
if err != nil {
t.Fatal(err)
}
@ -155,7 +155,7 @@ func TestTimeComparison(t *testing.T) {
t.Fatal(err)
}
res, err := Check(dir, dh, nil)
res, err := Check(dir, dh, nil, nil)
if err != nil {
t.Error(err)
}
@ -203,13 +203,13 @@ func TestTarTime(t *testing.T) {
keywords := dh.UsedKeywords()
// make sure "time" keyword works
_, err = Check(dir, dh, keywords)
_, err = Check(dir, dh, keywords, nil)
if err != nil {
t.Error(err)
}
// make sure tar_time wins
res, err := Check(dir, dh, append(keywords, "tar_time"))
res, err := Check(dir, dh, append(keywords, "tar_time"), nil)
if err != nil {
t.Error(err)
}
@ -254,7 +254,7 @@ func TestIgnoreComments(t *testing.T) {
t.Fatal(err)
}
res, err := Check(dir, dh, nil)
res, err := Check(dir, dh, nil, nil)
if err != nil {
t.Error(err)
}
@ -274,7 +274,7 @@ func TestIgnoreComments(t *testing.T) {
`
dh, err = ParseSpec(bytes.NewBufferString(spec))
res, err = Check(dir, dh, nil)
res, err = Check(dir, dh, nil, nil)
if err != nil {
t.Error(err)
}
@ -306,11 +306,11 @@ func TestCheckNeedsEncoding(t *testing.T) {
t.Error(err)
}
dh, err := Walk(dir, nil, DefaultKeywords)
dh, err := Walk(dir, nil, DefaultKeywords, nil)
if err != nil {
t.Fatal(err)
}
res, err := Check(dir, dh, nil)
res, err := Check(dir, dh, nil, nil)
if err != nil {
t.Fatal(err)
}

2
cmd/gomtree/main.go
View file

@ -236,7 +236,7 @@ func app() error {
}
} else {
// with a root directory
stateDh, err = mtree.Walk(rootPath, excludes, currentKeywords)
stateDh, err = mtree.Walk(rootPath, excludes, currentKeywords, nil)
if err != nil {
return err
}

22
compare_test.go
View file

@ -15,12 +15,12 @@ import (
// simple walk of current directory, and imediately check it.
// may not be parallelizable.
func TestCompare(t *testing.T) {
old, err := Walk(".", nil, append(DefaultKeywords, "sha1"))
old, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
new, err := Walk(".", nil, append(DefaultKeywords, "sha1"))
new, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -59,7 +59,7 @@ func TestCompareModified(t *testing.T) {
}
// Walk the current state.
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -70,7 +70,7 @@ func TestCompareModified(t *testing.T) {
}
// Walk the new state.
new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -140,7 +140,7 @@ func TestCompareMissing(t *testing.T) {
}
// Walk the current state.
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -159,7 +159,7 @@ func TestCompareMissing(t *testing.T) {
}
// Walk the new state.
new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -215,7 +215,7 @@ func TestCompareExtra(t *testing.T) {
defer os.RemoveAll(dir)
// Walk the current state.
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -237,7 +237,7 @@ func TestCompareExtra(t *testing.T) {
}
// Walk the new state.
new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -309,7 +309,7 @@ func TestCompareKeys(t *testing.T) {
}
// Walk the current state.
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -320,7 +320,7 @@ func TestCompareKeys(t *testing.T) {
}
// Walk the new state.
new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -382,7 +382,7 @@ func TestTarCompare(t *testing.T) {
}
// Walk the current state.
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}

1
creator.go
View file

@ -3,6 +3,7 @@ package mtree
// dhCreator is used in when building a DirectoryHierarchy
type dhCreator struct {
DH *DirectoryHierarchy
fs FsEval
curSet *Entry
curDir *Entry
curEnt *Entry

54
fseval.go Normal file
View file

@ -0,0 +1,54 @@
package mtree
import "os"
// FsEval is a mock-friendly method of specifying to go-mtree how to carry out
// filesystem operations such as opening files and the like. The semantics of
// all of these wrappers MUST be identical to the semantics described here.
type FsEval interface {
// Open must have the same semantics as os.Open.
Open(path string) (*os.File, error)
// Lstat must have the same semantics as os.Lstat.
Lstat(path string) (os.FileInfo, error)
// Readdir must have the same semantics as calling os.Open on the given
// path and then returning the result of (*os.File).Readdir(-1).
Readdir(path string) ([]os.FileInfo, error)
// KeywordFunc must return a wrapper around the provided function (in other
// words, the returned function must refer to the same keyword).
KeywordFunc(fn KeywordFunc) KeywordFunc
}
// DefaultFsEval is the default implementation of FsEval (and is the default
// used if a nil interface is passed to any mtree function). It does not modify
// or wrap any of the methods (they all just call out to os.*).
type DefaultFsEval struct{}
// Open must have the same semantics as os.Open.
func (fs DefaultFsEval) Open(path string) (*os.File, error) {
return os.Open(path)
}
// Lstat must have the same semantics as os.Lstat.
func (fs DefaultFsEval) Lstat(path string) (os.FileInfo, error) {
return os.Lstat(path)
}
// Readdir must have the same semantics as calling os.Open on the given
// path and then returning the result of (*os.File).Readdir(-1).
func (fs DefaultFsEval) Readdir(path string) ([]os.FileInfo, error) {
fh, err := os.Open(path)
if err != nil {
return nil, err
}
defer fh.Close()
return fh.Readdir(-1)
}
// KeywordFunc must return a wrapper around the provided function (in other
// words, the returned function must refer to the same keyword).
func (fs DefaultFsEval) KeywordFunc(fn KeywordFunc) KeywordFunc {
return fn
}

10
tar_test.go
View file

@ -26,7 +26,7 @@ func ExampleStreamer() {
// handle error ...
}
res, err := Check("/tmp/dir/", dh, nil)
res, err := Check("/tmp/dir/", dh, nil, nil)
if err != nil {
// handle error ...
}
@ -145,7 +145,7 @@ func TestArchiveCreation(t *testing.T) {
}
// Test the tar manifest against the actual directory
res, err := Check("./testdata/collection", tdh, []Keyword{"sha1"})
res, err := Check("./testdata/collection", tdh, []Keyword{"sha1"}, nil)
if err != nil {
t.Fatal(err)
}
@ -170,7 +170,7 @@ func TestArchiveCreation(t *testing.T) {
}
// Validate the directory manifest against the archive
dh, err := Walk("./testdata/collection", nil, []Keyword{"sha1"})
dh, err := Walk("./testdata/collection", nil, []Keyword{"sha1"}, nil)
if err != nil {
t.Fatal(err)
}
@ -224,7 +224,7 @@ func TestTreeTraversal(t *testing.T) {
}
// top-level "." directory will contain contents of traversal.tar
res, err = Check("./testdata/.", tdh, []Keyword{"sha1"})
res, err = Check("./testdata/.", tdh, []Keyword{"sha1"}, nil)
if err != nil {
t.Fatal(err)
}
@ -262,7 +262,7 @@ func TestTreeTraversal(t *testing.T) {
}
// Implied top-level "." directory will contain the contents of singlefile.tar
res, err = Check("./testdata/.", tdh, []Keyword{"sha1"})
res, err = Check("./testdata/.", tdh, []Keyword{"sha1"}, nil)
if err != nil {
t.Fatal(err)
}

34
walk.go
View file

@ -26,8 +26,11 @@ var defaultSetKeywords = []KeyVal{"type=file", "nlink=1", "flags=none", "mode=06
// Walk from root directory and assemble the DirectoryHierarchy. excludes
// provided are used to skip paths. keywords are the set to collect from the
// walked paths. The recommended default list is DefaultKeywords.
func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHierarchy, error) {
creator := dhCreator{DH: &DirectoryHierarchy{}}
func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval) (*DirectoryHierarchy, error) {
if fsEval == nil {
fsEval = DefaultFsEval{}
}
creator := dhCreator{DH: &DirectoryHierarchy{}, fs: fsEval}
// insert signature and metadata comments first (user, machine, tree, date)
for _, e := range signatureEntries(root) {
e.Pos = len(creator.DH.Entries)
@ -88,7 +91,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
err := func() error {
var r io.Reader
if info.Mode().IsRegular() {
fh, err := os.Open(path)
fh, err := creator.fs.Open(path)
if err != nil {
return err
}
@ -99,7 +102,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
if !ok {
return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
}
if str, err := keywordFunc(path, info, r); err == nil && str != "" {
if str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r); err == nil && str != "" {
e.Keywords = append(e.Keywords, str)
} else if err != nil {
return err
@ -119,7 +122,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
err := func() error {
var r io.Reader
if info.Mode().IsRegular() {
fh, err := os.Open(path)
fh, err := creator.fs.Open(path)
if err != nil {
return err
}
@ -130,7 +133,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
if !ok {
return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
}
str, err := keywordFunc(path, info, r)
str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r)
if err != nil {
return err
}
@ -177,7 +180,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
err := func() error {
var r io.Reader
if info.Mode().IsRegular() {
fh, err := os.Open(path)
fh, err := creator.fs.Open(path)
if err != nil {
return err
}
@ -188,7 +191,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
if !ok {
return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
}
str, err := keywordFunc(path, info, r)
str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r)
if err != nil {
return err
}
@ -227,7 +230,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
// large directories Walk can be inefficient.
// Walk does not follow symbolic links.
func startWalk(c *dhCreator, root string, walkFn filepath.WalkFunc) error {
info, err := os.Lstat(root)
info, err := c.fs.Lstat(root)
if err != nil {
return walkFn(root, nil, err)
}
@ -248,14 +251,14 @@ func walk(c *dhCreator, path string, info os.FileInfo, walkFn filepath.WalkFunc)
return nil
}
names, err := readOrderedDirNames(path)
names, err := readOrderedDirNames(c, path)
if err != nil {
return walkFn(path, info, err)
}
for _, name := range names {
filename := filepath.Join(path, name)
fileInfo, err := os.Lstat(filename)
fileInfo, err := c.fs.Lstat(filename)
if err != nil {
if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir {
return err
@ -282,13 +285,8 @@ func walk(c *dhCreator, path string, info os.FileInfo, walkFn filepath.WalkFunc)
// readOrderedDirNames reads the directory and returns a sorted list of all
// entries with non-directories first, followed by directories.
func readOrderedDirNames(dirname string) ([]string, error) {
f, err := os.Open(dirname)
if err != nil {
return nil, err
}
infos, err := f.Readdir(-1)
f.Close()
func readOrderedDirNames(c *dhCreator, dirname string) ([]string, error) {
infos, err := c.fs.Readdir(dirname)
if err != nil {
return nil, err
}

4
walk_test.go
View file

@ -7,7 +7,7 @@ import (
)
func TestWalk(t *testing.T) {
dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"))
dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -37,7 +37,7 @@ func TestWalk(t *testing.T) {
}
func TestWalkDirectory(t *testing.T) {
dh, err := Walk(".", []ExcludeFunc{ExcludeNonDirectories}, []Keyword{"type"})
dh, err := Walk(".", []ExcludeFunc{ExcludeNonDirectories}, []Keyword{"type"}, nil)
if err != nil {
t.Fatal(err)
}