From e22043cb86e7e93e2a4da8ef5df0408223de2385 Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Fri, 18 Nov 2016 18:53:26 +1100 Subject: [PATCH] walk: implement FsEval hooks In certain circumstances (such as the manifest generation of a filesystem as an unprivileged user) it is important to provide hooks that override the default os.* implementation of filesystem-related functions. In order to avoid merging too much code from outside projects (such as umoci) this is implemented by providing FsEval hooks to Walk() and Check(). This allows for users of go-mtree to modify how filesystem checks are done, without compromising the simplicity of go-mtree's code. Signed-off-by: Aleksa Sarai --- check.go | 6 ++--- check_test.go | 34 ++++++++++++++-------------- cmd/gomtree/main.go | 2 +- compare_test.go | 22 +++++++++--------- creator.go | 1 + fseval.go | 54 +++++++++++++++++++++++++++++++++++++++++++++ tar_test.go | 10 ++++----- walk.go | 34 ++++++++++++++-------------- walk_test.go | 4 ++-- 9 files changed, 110 insertions(+), 57 deletions(-) create mode 100644 fseval.go diff --git a/check.go b/check.go index cca12e3..29e05e3 100644 --- a/check.go +++ b/check.go @@ -5,13 +5,13 @@ package mtree // If keywords is nil, the check all present in the DirectoryHierarchy // // This is equivalent to creating a new DirectoryHierarchy with Walk(root, nil, -// keywords) and then doing a Compare(dh, newDh, keywords). -func Check(root string, dh *DirectoryHierarchy, keywords []Keyword) ([]InodeDelta, error) { +// keywords, fs) and then doing a Compare(dh, newDh, keywords). +func Check(root string, dh *DirectoryHierarchy, keywords []Keyword, fs FsEval) ([]InodeDelta, error) { if keywords == nil { keywords = dh.UsedKeywords() } - newDh, err := Walk(root, nil, keywords) + newDh, err := Walk(root, nil, keywords, fs) if err != nil { return nil, err } diff --git a/check_test.go b/check_test.go index 6627b53..c6e73d5 100644 --- a/check_test.go +++ b/check_test.go @@ -12,12 +12,12 @@ import ( // simple walk of current directory, and imediately check it. // may not be parallelizable. func TestCheck(t *testing.T) { - dh, err := Walk(".", nil, append(DefaultKeywords, "sha1")) + dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil) if err != nil { t.Fatal(err) } - res, err := Check(".", dh, nil) + res, err := Check(".", dh, nil, nil) if err != nil { t.Fatal(err) } @@ -43,13 +43,13 @@ func TestCheckKeywords(t *testing.T) { } // Walk this tempdir - dh, err := Walk(dir, nil, append(DefaultKeywords, "sha1")) + dh, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil) if err != nil { t.Fatal(err) } // Check for sanity. This ought to pass. - res, err := Check(dir, dh, nil) + res, err := Check(dir, dh, nil, nil) if err != nil { t.Fatal(err) } @@ -64,7 +64,7 @@ func TestCheckKeywords(t *testing.T) { } // Check again. This ought to fail. - res, err = Check(dir, dh, nil) + res, err = Check(dir, dh, nil, nil) if err != nil { t.Fatal(err) } @@ -76,7 +76,7 @@ func TestCheckKeywords(t *testing.T) { } // Check again, but only sha1 and mode. This ought to pass. - res, err = Check(dir, dh, []Keyword{"sha1", "mode"}) + res, err = Check(dir, dh, []Keyword{"sha1", "mode"}, nil) if err != nil { t.Fatal(err) } @@ -86,12 +86,12 @@ func TestCheckKeywords(t *testing.T) { } func ExampleCheck() { - dh, err := Walk(".", nil, append(DefaultKeywords, "sha1")) + dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil) if err != nil { // handle error ... } - res, err := Check(".", dh, nil) + res, err := Check(".", dh, nil, nil) if err != nil { // handle error ... } @@ -103,11 +103,11 @@ func ExampleCheck() { // Tests default action for evaluating a symlink, which is just to compare the // link itself, not to follow it func TestDefaultBrokenLink(t *testing.T) { - dh, err := Walk("./testdata/dirwithbrokenlink", nil, append(DefaultKeywords, "sha1")) + dh, err := Walk("./testdata/dirwithbrokenlink", nil, append(DefaultKeywords, "sha1"), nil) if err != nil { t.Fatal(err) } - res, err := Check("./testdata/dirwithbrokenlink", dh, nil) + res, err := Check("./testdata/dirwithbrokenlink", dh, nil, nil) if err != nil { t.Fatal(err) } @@ -155,7 +155,7 @@ func TestTimeComparison(t *testing.T) { t.Fatal(err) } - res, err := Check(dir, dh, nil) + res, err := Check(dir, dh, nil, nil) if err != nil { t.Error(err) } @@ -203,13 +203,13 @@ func TestTarTime(t *testing.T) { keywords := dh.UsedKeywords() // make sure "time" keyword works - _, err = Check(dir, dh, keywords) + _, err = Check(dir, dh, keywords, nil) if err != nil { t.Error(err) } // make sure tar_time wins - res, err := Check(dir, dh, append(keywords, "tar_time")) + res, err := Check(dir, dh, append(keywords, "tar_time"), nil) if err != nil { t.Error(err) } @@ -254,7 +254,7 @@ func TestIgnoreComments(t *testing.T) { t.Fatal(err) } - res, err := Check(dir, dh, nil) + res, err := Check(dir, dh, nil, nil) if err != nil { t.Error(err) } @@ -274,7 +274,7 @@ func TestIgnoreComments(t *testing.T) { ` dh, err = ParseSpec(bytes.NewBufferString(spec)) - res, err = Check(dir, dh, nil) + res, err = Check(dir, dh, nil, nil) if err != nil { t.Error(err) } @@ -306,11 +306,11 @@ func TestCheckNeedsEncoding(t *testing.T) { t.Error(err) } - dh, err := Walk(dir, nil, DefaultKeywords) + dh, err := Walk(dir, nil, DefaultKeywords, nil) if err != nil { t.Fatal(err) } - res, err := Check(dir, dh, nil) + res, err := Check(dir, dh, nil, nil) if err != nil { t.Fatal(err) } diff --git a/cmd/gomtree/main.go b/cmd/gomtree/main.go index af1380f..2f00eae 100644 --- a/cmd/gomtree/main.go +++ b/cmd/gomtree/main.go @@ -236,7 +236,7 @@ func app() error { } } else { // with a root directory - stateDh, err = mtree.Walk(rootPath, excludes, currentKeywords) + stateDh, err = mtree.Walk(rootPath, excludes, currentKeywords, nil) if err != nil { return err } diff --git a/compare_test.go b/compare_test.go index c496915..2a8e791 100644 --- a/compare_test.go +++ b/compare_test.go @@ -15,12 +15,12 @@ import ( // simple walk of current directory, and imediately check it. // may not be parallelizable. func TestCompare(t *testing.T) { - old, err := Walk(".", nil, append(DefaultKeywords, "sha1")) + old, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil) if err != nil { t.Fatal(err) } - new, err := Walk(".", nil, append(DefaultKeywords, "sha1")) + new, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil) if err != nil { t.Fatal(err) } @@ -59,7 +59,7 @@ func TestCompareModified(t *testing.T) { } // Walk the current state. - old, err := Walk(dir, nil, append(DefaultKeywords, "sha1")) + old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil) if err != nil { t.Fatal(err) } @@ -70,7 +70,7 @@ func TestCompareModified(t *testing.T) { } // Walk the new state. - new, err := Walk(dir, nil, append(DefaultKeywords, "sha1")) + new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil) if err != nil { t.Fatal(err) } @@ -140,7 +140,7 @@ func TestCompareMissing(t *testing.T) { } // Walk the current state. - old, err := Walk(dir, nil, append(DefaultKeywords, "sha1")) + old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil) if err != nil { t.Fatal(err) } @@ -159,7 +159,7 @@ func TestCompareMissing(t *testing.T) { } // Walk the new state. - new, err := Walk(dir, nil, append(DefaultKeywords, "sha1")) + new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil) if err != nil { t.Fatal(err) } @@ -215,7 +215,7 @@ func TestCompareExtra(t *testing.T) { defer os.RemoveAll(dir) // Walk the current state. - old, err := Walk(dir, nil, append(DefaultKeywords, "sha1")) + old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil) if err != nil { t.Fatal(err) } @@ -237,7 +237,7 @@ func TestCompareExtra(t *testing.T) { } // Walk the new state. - new, err := Walk(dir, nil, append(DefaultKeywords, "sha1")) + new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil) if err != nil { t.Fatal(err) } @@ -309,7 +309,7 @@ func TestCompareKeys(t *testing.T) { } // Walk the current state. - old, err := Walk(dir, nil, append(DefaultKeywords, "sha1")) + old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil) if err != nil { t.Fatal(err) } @@ -320,7 +320,7 @@ func TestCompareKeys(t *testing.T) { } // Walk the new state. - new, err := Walk(dir, nil, append(DefaultKeywords, "sha1")) + new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil) if err != nil { t.Fatal(err) } @@ -382,7 +382,7 @@ func TestTarCompare(t *testing.T) { } // Walk the current state. - old, err := Walk(dir, nil, append(DefaultKeywords, "sha1")) + old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil) if err != nil { t.Fatal(err) } diff --git a/creator.go b/creator.go index 2a23a22..43149c1 100644 --- a/creator.go +++ b/creator.go @@ -3,6 +3,7 @@ package mtree // dhCreator is used in when building a DirectoryHierarchy type dhCreator struct { DH *DirectoryHierarchy + fs FsEval curSet *Entry curDir *Entry curEnt *Entry diff --git a/fseval.go b/fseval.go new file mode 100644 index 0000000..2f006c5 --- /dev/null +++ b/fseval.go @@ -0,0 +1,54 @@ +package mtree + +import "os" + +// FsEval is a mock-friendly method of specifying to go-mtree how to carry out +// filesystem operations such as opening files and the like. The semantics of +// all of these wrappers MUST be identical to the semantics described here. +type FsEval interface { + // Open must have the same semantics as os.Open. + Open(path string) (*os.File, error) + + // Lstat must have the same semantics as os.Lstat. + Lstat(path string) (os.FileInfo, error) + + // Readdir must have the same semantics as calling os.Open on the given + // path and then returning the result of (*os.File).Readdir(-1). + Readdir(path string) ([]os.FileInfo, error) + + // KeywordFunc must return a wrapper around the provided function (in other + // words, the returned function must refer to the same keyword). + KeywordFunc(fn KeywordFunc) KeywordFunc +} + +// DefaultFsEval is the default implementation of FsEval (and is the default +// used if a nil interface is passed to any mtree function). It does not modify +// or wrap any of the methods (they all just call out to os.*). +type DefaultFsEval struct{} + +// Open must have the same semantics as os.Open. +func (fs DefaultFsEval) Open(path string) (*os.File, error) { + return os.Open(path) +} + +// Lstat must have the same semantics as os.Lstat. +func (fs DefaultFsEval) Lstat(path string) (os.FileInfo, error) { + return os.Lstat(path) +} + +// Readdir must have the same semantics as calling os.Open on the given +// path and then returning the result of (*os.File).Readdir(-1). +func (fs DefaultFsEval) Readdir(path string) ([]os.FileInfo, error) { + fh, err := os.Open(path) + if err != nil { + return nil, err + } + defer fh.Close() + return fh.Readdir(-1) +} + +// KeywordFunc must return a wrapper around the provided function (in other +// words, the returned function must refer to the same keyword). +func (fs DefaultFsEval) KeywordFunc(fn KeywordFunc) KeywordFunc { + return fn +} diff --git a/tar_test.go b/tar_test.go index 8ae89a0..c8621dc 100644 --- a/tar_test.go +++ b/tar_test.go @@ -26,7 +26,7 @@ func ExampleStreamer() { // handle error ... } - res, err := Check("/tmp/dir/", dh, nil) + res, err := Check("/tmp/dir/", dh, nil, nil) if err != nil { // handle error ... } @@ -145,7 +145,7 @@ func TestArchiveCreation(t *testing.T) { } // Test the tar manifest against the actual directory - res, err := Check("./testdata/collection", tdh, []Keyword{"sha1"}) + res, err := Check("./testdata/collection", tdh, []Keyword{"sha1"}, nil) if err != nil { t.Fatal(err) } @@ -170,7 +170,7 @@ func TestArchiveCreation(t *testing.T) { } // Validate the directory manifest against the archive - dh, err := Walk("./testdata/collection", nil, []Keyword{"sha1"}) + dh, err := Walk("./testdata/collection", nil, []Keyword{"sha1"}, nil) if err != nil { t.Fatal(err) } @@ -224,7 +224,7 @@ func TestTreeTraversal(t *testing.T) { } // top-level "." directory will contain contents of traversal.tar - res, err = Check("./testdata/.", tdh, []Keyword{"sha1"}) + res, err = Check("./testdata/.", tdh, []Keyword{"sha1"}, nil) if err != nil { t.Fatal(err) } @@ -262,7 +262,7 @@ func TestTreeTraversal(t *testing.T) { } // Implied top-level "." directory will contain the contents of singlefile.tar - res, err = Check("./testdata/.", tdh, []Keyword{"sha1"}) + res, err = Check("./testdata/.", tdh, []Keyword{"sha1"}, nil) if err != nil { t.Fatal(err) } diff --git a/walk.go b/walk.go index 2be6d31..b1dfe87 100644 --- a/walk.go +++ b/walk.go @@ -26,8 +26,11 @@ var defaultSetKeywords = []KeyVal{"type=file", "nlink=1", "flags=none", "mode=06 // Walk from root directory and assemble the DirectoryHierarchy. excludes // provided are used to skip paths. keywords are the set to collect from the // walked paths. The recommended default list is DefaultKeywords. -func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHierarchy, error) { - creator := dhCreator{DH: &DirectoryHierarchy{}} +func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval) (*DirectoryHierarchy, error) { + if fsEval == nil { + fsEval = DefaultFsEval{} + } + creator := dhCreator{DH: &DirectoryHierarchy{}, fs: fsEval} // insert signature and metadata comments first (user, machine, tree, date) for _, e := range signatureEntries(root) { e.Pos = len(creator.DH.Entries) @@ -88,7 +91,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi err := func() error { var r io.Reader if info.Mode().IsRegular() { - fh, err := os.Open(path) + fh, err := creator.fs.Open(path) if err != nil { return err } @@ -99,7 +102,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi if !ok { return fmt.Errorf("Unknown keyword %q for file %q", keyword, path) } - if str, err := keywordFunc(path, info, r); err == nil && str != "" { + if str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r); err == nil && str != "" { e.Keywords = append(e.Keywords, str) } else if err != nil { return err @@ -119,7 +122,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi err := func() error { var r io.Reader if info.Mode().IsRegular() { - fh, err := os.Open(path) + fh, err := creator.fs.Open(path) if err != nil { return err } @@ -130,7 +133,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi if !ok { return fmt.Errorf("Unknown keyword %q for file %q", keyword, path) } - str, err := keywordFunc(path, info, r) + str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r) if err != nil { return err } @@ -177,7 +180,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi err := func() error { var r io.Reader if info.Mode().IsRegular() { - fh, err := os.Open(path) + fh, err := creator.fs.Open(path) if err != nil { return err } @@ -188,7 +191,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi if !ok { return fmt.Errorf("Unknown keyword %q for file %q", keyword, path) } - str, err := keywordFunc(path, info, r) + str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r) if err != nil { return err } @@ -227,7 +230,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi // large directories Walk can be inefficient. // Walk does not follow symbolic links. func startWalk(c *dhCreator, root string, walkFn filepath.WalkFunc) error { - info, err := os.Lstat(root) + info, err := c.fs.Lstat(root) if err != nil { return walkFn(root, nil, err) } @@ -248,14 +251,14 @@ func walk(c *dhCreator, path string, info os.FileInfo, walkFn filepath.WalkFunc) return nil } - names, err := readOrderedDirNames(path) + names, err := readOrderedDirNames(c, path) if err != nil { return walkFn(path, info, err) } for _, name := range names { filename := filepath.Join(path, name) - fileInfo, err := os.Lstat(filename) + fileInfo, err := c.fs.Lstat(filename) if err != nil { if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { return err @@ -282,13 +285,8 @@ func walk(c *dhCreator, path string, info os.FileInfo, walkFn filepath.WalkFunc) // readOrderedDirNames reads the directory and returns a sorted list of all // entries with non-directories first, followed by directories. -func readOrderedDirNames(dirname string) ([]string, error) { - f, err := os.Open(dirname) - if err != nil { - return nil, err - } - infos, err := f.Readdir(-1) - f.Close() +func readOrderedDirNames(c *dhCreator, dirname string) ([]string, error) { + infos, err := c.fs.Readdir(dirname) if err != nil { return nil, err } diff --git a/walk_test.go b/walk_test.go index d40ea90..218262d 100644 --- a/walk_test.go +++ b/walk_test.go @@ -7,7 +7,7 @@ import ( ) func TestWalk(t *testing.T) { - dh, err := Walk(".", nil, append(DefaultKeywords, "sha1")) + dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil) if err != nil { t.Fatal(err) } @@ -37,7 +37,7 @@ func TestWalk(t *testing.T) { } func TestWalkDirectory(t *testing.T) { - dh, err := Walk(".", []ExcludeFunc{ExcludeNonDirectories}, []Keyword{"type"}) + dh, err := Walk(".", []ExcludeFunc{ExcludeNonDirectories}, []Keyword{"type"}, nil) if err != nil { t.Fatal(err) }