walk: implement FsEval hooks

In certain circumstances (such as the manifest generation of a
filesystem as an unprivileged user) it is important to provide hooks
that override the default os.* implementation of filesystem-related
functions.

In order to avoid merging too much code from outside projects (such as
umoci) this is implemented by providing FsEval hooks to Walk() and
Check(). This allows for users of go-mtree to modify how filesystem
checks are done, without compromising the simplicity of go-mtree's code.

Signed-off-by: Aleksa Sarai <asarai@suse.de>

check.go

@@ -5,13 +5,13 @@ package mtree
 // If keywords is nil, the check all present in the DirectoryHierarchy
 //
 // This is equivalent to creating a new DirectoryHierarchy with Walk(root, nil,
-// keywords) and then doing a Compare(dh, newDh, keywords).
+// keywords, fs) and then doing a Compare(dh, newDh, keywords).
-func Check(root string, dh *DirectoryHierarchy, keywords []Keyword) ([]InodeDelta, error) {
+func Check(root string, dh *DirectoryHierarchy, keywords []Keyword, fs FsEval) ([]InodeDelta, error) {
 	if keywords == nil {
 		keywords = dh.UsedKeywords()
 	}
 
-	newDh, err := Walk(root, nil, keywords)
+	newDh, err := Walk(root, nil, keywords, fs)
 	if err != nil {
 		return nil, err
 	}

check_test.go

@@ -12,12 +12,12 @@ import (
 // simple walk of current directory, and imediately check it.
 // may not be parallelizable.
 func TestCheck(t *testing.T) {
-	dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"))
+	dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	res, err := Check(".", dh, nil)
+	res, err := Check(".", dh, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -43,13 +43,13 @@ func TestCheckKeywords(t *testing.T) {
 	}
 
 	// Walk this tempdir
-	dh, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
+	dh, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Check for sanity. This ought to pass.
-	res, err := Check(dir, dh, nil)
+	res, err := Check(dir, dh, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -64,7 +64,7 @@ func TestCheckKeywords(t *testing.T) {
 	}
 
 	// Check again. This ought to fail.
-	res, err = Check(dir, dh, nil)
+	res, err = Check(dir, dh, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -76,7 +76,7 @@ func TestCheckKeywords(t *testing.T) {
 	}
 
 	// Check again, but only sha1 and mode. This ought to pass.
-	res, err = Check(dir, dh, []Keyword{"sha1", "mode"})
+	res, err = Check(dir, dh, []Keyword{"sha1", "mode"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -86,12 +86,12 @@ func TestCheckKeywords(t *testing.T) {
 }
 
 func ExampleCheck() {
-	dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"))
+	dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
 	if err != nil {
 		// handle error ...
 	}
 
-	res, err := Check(".", dh, nil)
+	res, err := Check(".", dh, nil, nil)
 	if err != nil {
 		// handle error ...
 	}
@@ -103,11 +103,11 @@ func ExampleCheck() {
 // Tests default action for evaluating a symlink, which is just to compare the
 // link itself, not to follow it
 func TestDefaultBrokenLink(t *testing.T) {
-	dh, err := Walk("./testdata/dirwithbrokenlink", nil, append(DefaultKeywords, "sha1"))
+	dh, err := Walk("./testdata/dirwithbrokenlink", nil, append(DefaultKeywords, "sha1"), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	res, err := Check("./testdata/dirwithbrokenlink", dh, nil)
+	res, err := Check("./testdata/dirwithbrokenlink", dh, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -155,7 +155,7 @@ func TestTimeComparison(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	res, err := Check(dir, dh, nil)
+	res, err := Check(dir, dh, nil, nil)
 	if err != nil {
 		t.Error(err)
 	}
@@ -203,13 +203,13 @@ func TestTarTime(t *testing.T) {
 	keywords := dh.UsedKeywords()
 
 	// make sure "time" keyword works
-	_, err = Check(dir, dh, keywords)
+	_, err = Check(dir, dh, keywords, nil)
 	if err != nil {
 		t.Error(err)
 	}
 
 	// make sure tar_time wins
-	res, err := Check(dir, dh, append(keywords, "tar_time"))
+	res, err := Check(dir, dh, append(keywords, "tar_time"), nil)
 	if err != nil {
 		t.Error(err)
 	}
@@ -254,7 +254,7 @@ func TestIgnoreComments(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	res, err := Check(dir, dh, nil)
+	res, err := Check(dir, dh, nil, nil)
 	if err != nil {
 		t.Error(err)
 	}
@@ -274,7 +274,7 @@ func TestIgnoreComments(t *testing.T) {
 `
 	dh, err = ParseSpec(bytes.NewBufferString(spec))
 
-	res, err = Check(dir, dh, nil)
+	res, err = Check(dir, dh, nil, nil)
 	if err != nil {
 		t.Error(err)
 	}
@@ -306,11 +306,11 @@ func TestCheckNeedsEncoding(t *testing.T) {
 		t.Error(err)
 	}
 
-	dh, err := Walk(dir, nil, DefaultKeywords)
+	dh, err := Walk(dir, nil, DefaultKeywords, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	res, err := Check(dir, dh, nil)
+	res, err := Check(dir, dh, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/gomtree/main.go

@@ -236,7 +236,7 @@ func app() error {
 		}
 	} else {
 		// with a root directory
-		stateDh, err = mtree.Walk(rootPath, excludes, currentKeywords)
+		stateDh, err = mtree.Walk(rootPath, excludes, currentKeywords, nil)
 		if err != nil {
 			return err
 		}

compare_test.go

@@ -15,12 +15,12 @@ import (
 // simple walk of current directory, and imediately check it.
 // may not be parallelizable.
 func TestCompare(t *testing.T) {
-	old, err := Walk(".", nil, append(DefaultKeywords, "sha1"))
+	old, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	new, err := Walk(".", nil, append(DefaultKeywords, "sha1"))
+	new, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -59,7 +59,7 @@ func TestCompareModified(t *testing.T) {
 	}
 
 	// Walk the current state.
-	old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
+	old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -70,7 +70,7 @@ func TestCompareModified(t *testing.T) {
 	}
 
 	// Walk the new state.
-	new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
+	new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -140,7 +140,7 @@ func TestCompareMissing(t *testing.T) {
 	}
 
 	// Walk the current state.
-	old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
+	old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -159,7 +159,7 @@ func TestCompareMissing(t *testing.T) {
 	}
 
 	// Walk the new state.
-	new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
+	new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -215,7 +215,7 @@ func TestCompareExtra(t *testing.T) {
 	defer os.RemoveAll(dir)
 
 	// Walk the current state.
-	old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
+	old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -237,7 +237,7 @@ func TestCompareExtra(t *testing.T) {
 	}
 
 	// Walk the new state.
-	new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
+	new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -309,7 +309,7 @@ func TestCompareKeys(t *testing.T) {
 	}
 
 	// Walk the current state.
-	old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
+	old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -320,7 +320,7 @@ func TestCompareKeys(t *testing.T) {
 	}
 
 	// Walk the new state.
-	new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
+	new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -382,7 +382,7 @@ func TestTarCompare(t *testing.T) {
 	}
 
 	// Walk the current state.
-	old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
+	old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
 	if err != nil {
 		t.Fatal(err)
 	}

creator.go

@@ -3,6 +3,7 @@ package mtree
 // dhCreator is used in when building a DirectoryHierarchy
 type dhCreator struct {
 	DH     *DirectoryHierarchy
+	fs     FsEval
 	curSet *Entry
 	curDir *Entry
 	curEnt *Entry

fseval.go

@@ -0,0 +1,54 @@
+package mtree
+
+import "os"
+
+// FsEval is a mock-friendly method of specifying to go-mtree how to carry out
+// filesystem operations such as opening files and the like. The semantics of
+// all of these wrappers MUST be identical to the semantics described here.
+type FsEval interface {
+	// Open must have the same semantics as os.Open.
+	Open(path string) (*os.File, error)
+
+	// Lstat must have the same semantics as os.Lstat.
+	Lstat(path string) (os.FileInfo, error)
+
+	// Readdir must have the same semantics as calling os.Open on the given
+	// path and then returning the result of (*os.File).Readdir(-1).
+	Readdir(path string) ([]os.FileInfo, error)
+
+	// KeywordFunc must return a wrapper around the provided function (in other
+	// words, the returned function must refer to the same keyword).
+	KeywordFunc(fn KeywordFunc) KeywordFunc
+}
+
+// DefaultFsEval is the default implementation of FsEval (and is the default
+// used if a nil interface is passed to any mtree function). It does not modify
+// or wrap any of the methods (they all just call out to os.*).
+type DefaultFsEval struct{}
+
+// Open must have the same semantics as os.Open.
+func (fs DefaultFsEval) Open(path string) (*os.File, error) {
+	return os.Open(path)
+}
+
+// Lstat must have the same semantics as os.Lstat.
+func (fs DefaultFsEval) Lstat(path string) (os.FileInfo, error) {
+	return os.Lstat(path)
+}
+
+// Readdir must have the same semantics as calling os.Open on the given
+// path and then returning the result of (*os.File).Readdir(-1).
+func (fs DefaultFsEval) Readdir(path string) ([]os.FileInfo, error) {
+	fh, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer fh.Close()
+	return fh.Readdir(-1)
+}
+
+// KeywordFunc must return a wrapper around the provided function (in other
+// words, the returned function must refer to the same keyword).
+func (fs DefaultFsEval) KeywordFunc(fn KeywordFunc) KeywordFunc {
+	return fn
+}

tar_test.go

@@ -26,7 +26,7 @@ func ExampleStreamer() {
 		// handle error ...
 	}
 
-	res, err := Check("/tmp/dir/", dh, nil)
+	res, err := Check("/tmp/dir/", dh, nil, nil)
 	if err != nil {
 		// handle error ...
 	}
@@ -145,7 +145,7 @@ func TestArchiveCreation(t *testing.T) {
 	}
 
 	// Test the tar manifest against the actual directory
-	res, err := Check("./testdata/collection", tdh, []Keyword{"sha1"})
+	res, err := Check("./testdata/collection", tdh, []Keyword{"sha1"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -170,7 +170,7 @@ func TestArchiveCreation(t *testing.T) {
 	}
 
 	// Validate the directory manifest against the archive
-	dh, err := Walk("./testdata/collection", nil, []Keyword{"sha1"})
+	dh, err := Walk("./testdata/collection", nil, []Keyword{"sha1"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -224,7 +224,7 @@ func TestTreeTraversal(t *testing.T) {
 	}
 
 	// top-level "." directory will contain contents of traversal.tar
-	res, err = Check("./testdata/.", tdh, []Keyword{"sha1"})
+	res, err = Check("./testdata/.", tdh, []Keyword{"sha1"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -262,7 +262,7 @@ func TestTreeTraversal(t *testing.T) {
 	}
 
 	// Implied top-level "." directory will contain the contents of singlefile.tar
-	res, err = Check("./testdata/.", tdh, []Keyword{"sha1"})
+	res, err = Check("./testdata/.", tdh, []Keyword{"sha1"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

walk.go

@@ -26,8 +26,11 @@ var defaultSetKeywords = []KeyVal{"type=file", "nlink=1", "flags=none", "mode=06
 // Walk from root directory and assemble the DirectoryHierarchy. excludes
 // provided are used to skip paths. keywords are the set to collect from the
 // walked paths. The recommended default list is DefaultKeywords.
-func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHierarchy, error) {
+func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval) (*DirectoryHierarchy, error) {
-	creator := dhCreator{DH: &DirectoryHierarchy{}}
+	if fsEval == nil {
+		fsEval = DefaultFsEval{}
+	}
+	creator := dhCreator{DH: &DirectoryHierarchy{}, fs: fsEval}
 	// insert signature and metadata comments first (user, machine, tree, date)
 	for _, e := range signatureEntries(root) {
 		e.Pos = len(creator.DH.Entries)
@@ -88,7 +91,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
 					err := func() error {
 						var r io.Reader
 						if info.Mode().IsRegular() {
-							fh, err := os.Open(path)
+							fh, err := creator.fs.Open(path)
 							if err != nil {
 								return err
 							}
@@ -99,7 +102,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
 						if !ok {
 							return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
 						}
-						if str, err := keywordFunc(path, info, r); err == nil && str != "" {
+						if str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r); err == nil && str != "" {
 							e.Keywords = append(e.Keywords, str)
 						} else if err != nil {
 							return err
@@ -119,7 +122,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
 					err := func() error {
 						var r io.Reader
 						if info.Mode().IsRegular() {
-							fh, err := os.Open(path)
+							fh, err := creator.fs.Open(path)
 							if err != nil {
 								return err
 							}
@@ -130,7 +133,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
 						if !ok {
 							return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
 						}
-						str, err := keywordFunc(path, info, r)
+						str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r)
 						if err != nil {
 							return err
 						}
@@ -177,7 +180,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
 			err := func() error {
 				var r io.Reader
 				if info.Mode().IsRegular() {
-					fh, err := os.Open(path)
+					fh, err := creator.fs.Open(path)
 					if err != nil {
 						return err
 					}
@@ -188,7 +191,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
 				if !ok {
 					return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
 				}
-				str, err := keywordFunc(path, info, r)
+				str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r)
 				if err != nil {
 					return err
 				}
@@ -227,7 +230,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
 // large directories Walk can be inefficient.
 // Walk does not follow symbolic links.
 func startWalk(c *dhCreator, root string, walkFn filepath.WalkFunc) error {
-	info, err := os.Lstat(root)
+	info, err := c.fs.Lstat(root)
 	if err != nil {
 		return walkFn(root, nil, err)
 	}
@@ -248,14 +251,14 @@ func walk(c *dhCreator, path string, info os.FileInfo, walkFn filepath.WalkFunc)
 		return nil
 	}
 
-	names, err := readOrderedDirNames(path)
+	names, err := readOrderedDirNames(c, path)
 	if err != nil {
 		return walkFn(path, info, err)
 	}
 
 	for _, name := range names {
 		filename := filepath.Join(path, name)
-		fileInfo, err := os.Lstat(filename)
+		fileInfo, err := c.fs.Lstat(filename)
 		if err != nil {
 			if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir {
 				return err
@@ -282,13 +285,8 @@ func walk(c *dhCreator, path string, info os.FileInfo, walkFn filepath.WalkFunc)
 
 // readOrderedDirNames reads the directory and returns a sorted list of all
 // entries with non-directories first, followed by directories.
-func readOrderedDirNames(dirname string) ([]string, error) {
+func readOrderedDirNames(c *dhCreator, dirname string) ([]string, error) {
-	f, err := os.Open(dirname)
+	infos, err := c.fs.Readdir(dirname)
-	if err != nil {
-		return nil, err
-	}
-	infos, err := f.Readdir(-1)
-	f.Close()
 	if err != nil {
 		return nil, err
 	}

walk_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func TestWalk(t *testing.T) {
-	dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"))
+	dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -37,7 +37,7 @@ func TestWalk(t *testing.T) {
 }
 
 func TestWalkDirectory(t *testing.T) {
-	dh, err := Walk(".", []ExcludeFunc{ExcludeNonDirectories}, []Keyword{"type"})
+	dh, err := Walk(".", []ExcludeFunc{ExcludeNonDirectories}, []Keyword{"type"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}