vbatts/go-mtree
1
0
Fork 0
Code Issues Pull requests Releases Activity

Merge pull request #96 from cyphar/add-unpriv-walking

Browse source 
walk: implement "unprivileged manifest generation"
This commit is contained in:
Vincent Batts 2016-12-14 13:32:14 -05:00 committed by GitHub
commit 0dc720e861
10 changed files with 272 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
check.go
View file

@ -5,22 +5,17 @@ package mtree
// If keywords is nil, the check all present in the DirectoryHierarchy
//
// This is equivalent to creating a new DirectoryHierarchy with Walk(root, nil,
// keywords) and then doing a Compare(dh, newDh, keywords).
func Check(root string, dh *DirectoryHierarchy, keywords []Keyword) ([]InodeDelta, error) {
// keywords, fs) and then doing a Compare(dh, newDh, keywords).
func Check(root string, dh *DirectoryHierarchy, keywords []Keyword, fs FsEval) ([]InodeDelta, error) {
if keywords == nil {
used := dh.UsedKeywords()
newDh, err := Walk(root, nil, used)
if err != nil {
return nil, err
}
return Compare(dh, newDh, used)
keywords = dh.UsedKeywords()
}
newDh, err := Walk(root, nil, keywords)
newDh, err := Walk(root, nil, keywords, fs)
if err != nil {
return nil, err
}
// TODO: Handle tar_time, if necessary.
return Compare(dh, newDh, keywords)
}

34
check_test.go
View file

@ -12,12 +12,12 @@ import (
// simple walk of current directory, and imediately check it.
// may not be parallelizable.
func TestCheck(t *testing.T) {
dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"))
dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
res, err := Check(".", dh, nil)
res, err := Check(".", dh, nil, nil)
if err != nil {
t.Fatal(err)
}
@ -43,13 +43,13 @@ func TestCheckKeywords(t *testing.T) {
}
// Walk this tempdir
dh, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
dh, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
// Check for sanity. This ought to pass.
res, err := Check(dir, dh, nil)
res, err := Check(dir, dh, nil, nil)
if err != nil {
t.Fatal(err)
}
@ -64,7 +64,7 @@ func TestCheckKeywords(t *testing.T) {
}
// Check again. This ought to fail.
res, err = Check(dir, dh, nil)
res, err = Check(dir, dh, nil, nil)
if err != nil {
t.Fatal(err)
}
@ -76,7 +76,7 @@ func TestCheckKeywords(t *testing.T) {
}
// Check again, but only sha1 and mode. This ought to pass.
res, err = Check(dir, dh, []Keyword{"sha1", "mode"})
res, err = Check(dir, dh, []Keyword{"sha1", "mode"}, nil)
if err != nil {
t.Fatal(err)
}
@ -86,12 +86,12 @@ func TestCheckKeywords(t *testing.T) {
}
func ExampleCheck() {
dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"))
dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
// handle error ...
}
res, err := Check(".", dh, nil)
res, err := Check(".", dh, nil, nil)
if err != nil {
// handle error ...
}
@ -103,11 +103,11 @@ func ExampleCheck() {
// Tests default action for evaluating a symlink, which is just to compare the
// link itself, not to follow it
func TestDefaultBrokenLink(t *testing.T) {
dh, err := Walk("./testdata/dirwithbrokenlink", nil, append(DefaultKeywords, "sha1"))
dh, err := Walk("./testdata/dirwithbrokenlink", nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
res, err := Check("./testdata/dirwithbrokenlink", dh, nil)
res, err := Check("./testdata/dirwithbrokenlink", dh, nil, nil)
if err != nil {
t.Fatal(err)
}
@ -155,7 +155,7 @@ func TestTimeComparison(t *testing.T) {
t.Fatal(err)
}
res, err := Check(dir, dh, nil)
res, err := Check(dir, dh, nil, nil)
if err != nil {
t.Error(err)
}
@ -203,13 +203,13 @@ func TestTarTime(t *testing.T) {
keywords := dh.UsedKeywords()
// make sure "time" keyword works
_, err = Check(dir, dh, keywords)
_, err = Check(dir, dh, keywords, nil)
if err != nil {
t.Error(err)
}
// make sure tar_time wins
res, err := Check(dir, dh, append(keywords, "tar_time"))
res, err := Check(dir, dh, append(keywords, "tar_time"), nil)
if err != nil {
t.Error(err)
}
@ -254,7 +254,7 @@ func TestIgnoreComments(t *testing.T) {
t.Fatal(err)
}
res, err := Check(dir, dh, nil)
res, err := Check(dir, dh, nil, nil)
if err != nil {
t.Error(err)
}
@ -274,7 +274,7 @@ func TestIgnoreComments(t *testing.T) {
`
dh, err = ParseSpec(bytes.NewBufferString(spec))
res, err = Check(dir, dh, nil)
res, err = Check(dir, dh, nil, nil)
if err != nil {
t.Error(err)
}
@ -306,11 +306,11 @@ func TestCheckNeedsEncoding(t *testing.T) {
t.Error(err)
}
dh, err := Walk(dir, nil, DefaultKeywords)
dh, err := Walk(dir, nil, DefaultKeywords, nil)
if err != nil {
t.Fatal(err)
}
res, err := Check(dir, dh, nil)
res, err := Check(dir, dh, nil, nil)
if err != nil {
t.Fatal(err)
}

2
cmd/gomtree/main.go
View file

@ -236,7 +236,7 @@ func app() error {
}
} else {
// with a root directory
stateDh, err = mtree.Walk(rootPath, excludes, currentKeywords)
stateDh, err = mtree.Walk(rootPath, excludes, currentKeywords, nil)
if err != nil {
return err
}

22
compare_test.go
View file

@ -15,12 +15,12 @@ import (
// simple walk of current directory, and imediately check it.
// may not be parallelizable.
func TestCompare(t *testing.T) {
old, err := Walk(".", nil, append(DefaultKeywords, "sha1"))
old, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
new, err := Walk(".", nil, append(DefaultKeywords, "sha1"))
new, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -59,7 +59,7 @@ func TestCompareModified(t *testing.T) {
}
// Walk the current state.
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -70,7 +70,7 @@ func TestCompareModified(t *testing.T) {
}
// Walk the new state.
new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -140,7 +140,7 @@ func TestCompareMissing(t *testing.T) {
}
// Walk the current state.
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -159,7 +159,7 @@ func TestCompareMissing(t *testing.T) {
}
// Walk the new state.
new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -215,7 +215,7 @@ func TestCompareExtra(t *testing.T) {
defer os.RemoveAll(dir)
// Walk the current state.
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -237,7 +237,7 @@ func TestCompareExtra(t *testing.T) {
}
// Walk the new state.
new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -309,7 +309,7 @@ func TestCompareKeys(t *testing.T) {
}
// Walk the current state.
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -320,7 +320,7 @@ func TestCompareKeys(t *testing.T) {
}
// Walk the new state.
new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -382,7 +382,7 @@ func TestTarCompare(t *testing.T) {
}
// Walk the current state.
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"))
old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}

1
creator.go
View file

@ -3,6 +3,7 @@ package mtree
// dhCreator is used in when building a DirectoryHierarchy
type dhCreator struct {
DH *DirectoryHierarchy
fs FsEval
curSet *Entry
curDir *Entry
curEnt *Entry

54
fseval.go Normal file
View file

@ -0,0 +1,54 @@
package mtree
import "os"
// FsEval is a mock-friendly method of specifying to go-mtree how to carry out
// filesystem operations such as opening files and the like. The semantics of
// all of these wrappers MUST be identical to the semantics described here.
type FsEval interface {
// Open must have the same semantics as os.Open.
Open(path string) (*os.File, error)
// Lstat must have the same semantics as os.Lstat.
Lstat(path string) (os.FileInfo, error)
// Readdir must have the same semantics as calling os.Open on the given
// path and then returning the result of (*os.File).Readdir(-1).
Readdir(path string) ([]os.FileInfo, error)
// KeywordFunc must return a wrapper around the provided function (in other
// words, the returned function must refer to the same keyword).
KeywordFunc(fn KeywordFunc) KeywordFunc
}
// DefaultFsEval is the default implementation of FsEval (and is the default
// used if a nil interface is passed to any mtree function). It does not modify
// or wrap any of the methods (they all just call out to os.*).
type DefaultFsEval struct{}
// Open must have the same semantics as os.Open.
func (fs DefaultFsEval) Open(path string) (*os.File, error) {
return os.Open(path)
}
// Lstat must have the same semantics as os.Lstat.
func (fs DefaultFsEval) Lstat(path string) (os.FileInfo, error) {
return os.Lstat(path)
}
// Readdir must have the same semantics as calling os.Open on the given
// path and then returning the result of (*os.File).Readdir(-1).
func (fs DefaultFsEval) Readdir(path string) ([]os.FileInfo, error) {
fh, err := os.Open(path)
if err != nil {
return nil, err
}
defer fh.Close()
return fh.Readdir(-1)
}
// KeywordFunc must return a wrapper around the provided function (in other
// words, the returned function must refer to the same keyword).
func (fs DefaultFsEval) KeywordFunc(fn KeywordFunc) KeywordFunc {
return fn
}

160
fseval_test.go Normal file
View file

@ -0,0 +1,160 @@
package mtree
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"time"
)
var mockTime = time.Unix(1337888823, 88288518233)
// Here be some dodgy testing. In particular, we have to mess around with some
// of the FsEval functions. In particular, we change all of the FileInfos to a
// different value.
type mockFileInfo struct {
os.FileInfo
}
func (fi mockFileInfo) Mode() os.FileMode {
return os.FileMode(fi.FileInfo.Mode() | 0777)
}
func (fi mockFileInfo) ModTime() time.Time {
return mockTime
}
type MockFsEval struct {
open, lstat, readdir, keywordFunc int
}
// Open must have the same semantics as os.Open.
func (fs *MockFsEval) Open(path string) (*os.File, error) {
fs.open++
return os.Open(path)
}
// Lstat must have the same semantics as os.Lstat.
func (fs *MockFsEval) Lstat(path string) (os.FileInfo, error) {
fs.lstat++
fi, err := os.Lstat(path)
return mockFileInfo{fi}, err
}
// Readdir must have the same semantics as calling os.Open on the given
// path and then returning the result of (*os.File).Readdir(-1).
func (fs *MockFsEval) Readdir(path string) ([]os.FileInfo, error) {
fs.readdir++
fh, err := os.Open(path)
if err != nil {
return nil, err
}
defer fh.Close()
fis, err := fh.Readdir(-1)
if err != nil {
return nil, err
}
for idx := range fis {
fis[idx] = mockFileInfo{fis[idx]}
}
return fis, nil
}
// KeywordFunc must return a wrapper around the provided function (in other
// words, the returned function must refer to the same keyword).
func (fs *MockFsEval) KeywordFunc(fn KeywordFunc) KeywordFunc {
fs.keywordFunc++
return fn
}
func TestCheckFsEval(t *testing.T) {
dir, err := ioutil.TempDir("", "test-check-fs-eval")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir) // clean up
content := []byte("If you hide your ignorance, no one will hit you and you'll never learn.")
tmpfn := filepath.Join(dir, "tmpfile")
if err := ioutil.WriteFile(tmpfn, content, 0451); err != nil {
t.Fatal(err)
}
// Walk this tempdir
mock := &MockFsEval{}
dh, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), mock)
if err != nil {
t.Fatal(err)
}
// Make sure that mock functions have been called.
if mock.open == 0 {
t.Errorf("mock.Open not called")
}
if mock.lstat == 0 {
t.Errorf("mock.Lstat not called")
}
if mock.readdir == 0 {
t.Errorf("mock.Readdir not called")
}
if mock.keywordFunc == 0 {
t.Errorf("mock.KeywordFunc not called")
}
// Check for sanity. This ought to pass.
mock = &MockFsEval{}
res, err := Check(dir, dh, nil, mock)
if err != nil {
t.Fatal(err)
}
if len(res) > 0 {
t.Errorf("%#v", res)
}
// Make sure that mock functions have been called.
if mock.open == 0 {
t.Errorf("mock.Open not called")
}
if mock.lstat == 0 {
t.Errorf("mock.Lstat not called")
}
if mock.readdir == 0 {
t.Errorf("mock.Readdir not called")
}
if mock.keywordFunc == 0 {
t.Errorf("mock.KeywordFunc not called")
}
// This should FAIL.
res, err = Check(dir, dh, nil, nil)
if err != nil {
t.Fatal(err)
}
if len(res) == 0 {
t.Errorf("expected Check to fail")
}
// Modify the metadata so you can get the right output.
if err := os.Chmod(tmpfn, 0777); err != nil {
t.Fatal(err)
}
if err := os.Chtimes(tmpfn, mockTime, mockTime); err != nil {
t.Fatal(err)
}
if err := os.Chmod(dir, 0777); err != nil {
t.Fatal(err)
}
if err := os.Chtimes(dir, mockTime, mockTime); err != nil {
t.Fatal(err)
}
// It should now succeed.
res, err = Check(dir, dh, nil, nil)
if err != nil {
t.Fatal(err)
}
if len(res) > 0 {
t.Errorf("%#v", res)
}
}

10
tar_test.go
View file

@ -26,7 +26,7 @@ func ExampleStreamer() {
// handle error ...
}
res, err := Check("/tmp/dir/", dh, nil)
res, err := Check("/tmp/dir/", dh, nil, nil)
if err != nil {
// handle error ...
}
@ -145,7 +145,7 @@ func TestArchiveCreation(t *testing.T) {
}
// Test the tar manifest against the actual directory
res, err := Check("./testdata/collection", tdh, []Keyword{"sha1"})
res, err := Check("./testdata/collection", tdh, []Keyword{"sha1"}, nil)
if err != nil {
t.Fatal(err)
}
@ -170,7 +170,7 @@ func TestArchiveCreation(t *testing.T) {
}
// Validate the directory manifest against the archive
dh, err := Walk("./testdata/collection", nil, []Keyword{"sha1"})
dh, err := Walk("./testdata/collection", nil, []Keyword{"sha1"}, nil)
if err != nil {
t.Fatal(err)
}
@ -224,7 +224,7 @@ func TestTreeTraversal(t *testing.T) {
}
// top-level "." directory will contain contents of traversal.tar
res, err = Check("./testdata/.", tdh, []Keyword{"sha1"})
res, err = Check("./testdata/.", tdh, []Keyword{"sha1"}, nil)
if err != nil {
t.Fatal(err)
}
@ -262,7 +262,7 @@ func TestTreeTraversal(t *testing.T) {
}
// Implied top-level "." directory will contain the contents of singlefile.tar
res, err = Check("./testdata/.", tdh, []Keyword{"sha1"})
res, err = Check("./testdata/.", tdh, []Keyword{"sha1"}, nil)
if err != nil {
t.Fatal(err)
}

34
walk.go
View file

@ -26,8 +26,11 @@ var defaultSetKeywords = []KeyVal{"type=file", "nlink=1", "flags=none", "mode=06
// Walk from root directory and assemble the DirectoryHierarchy. excludes
// provided are used to skip paths. keywords are the set to collect from the
// walked paths. The recommended default list is DefaultKeywords.
func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHierarchy, error) {
creator := dhCreator{DH: &DirectoryHierarchy{}}
func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval) (*DirectoryHierarchy, error) {
if fsEval == nil {
fsEval = DefaultFsEval{}
}
creator := dhCreator{DH: &DirectoryHierarchy{}, fs: fsEval}
// insert signature and metadata comments first (user, machine, tree, date)
for _, e := range signatureEntries(root) {
e.Pos = len(creator.DH.Entries)
@ -88,7 +91,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
err := func() error {
var r io.Reader
if info.Mode().IsRegular() {
fh, err := os.Open(path)
fh, err := creator.fs.Open(path)
if err != nil {
return err
}
@ -99,7 +102,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
if !ok {
return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
}
if str, err := keywordFunc(path, info, r); err == nil && str != "" {
if str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r); err == nil && str != "" {
e.Keywords = append(e.Keywords, str)
} else if err != nil {
return err
@ -119,7 +122,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
err := func() error {
var r io.Reader
if info.Mode().IsRegular() {
fh, err := os.Open(path)
fh, err := creator.fs.Open(path)
if err != nil {
return err
}
@ -130,7 +133,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
if !ok {
return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
}
str, err := keywordFunc(path, info, r)
str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r)
if err != nil {
return err
}
@ -177,7 +180,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
err := func() error {
var r io.Reader
if info.Mode().IsRegular() {
fh, err := os.Open(path)
fh, err := creator.fs.Open(path)
if err != nil {
return err
}
@ -188,7 +191,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
if !ok {
return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
}
str, err := keywordFunc(path, info, r)
str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r)
if err != nil {
return err
}
@ -227,7 +230,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHi
// large directories Walk can be inefficient.
// Walk does not follow symbolic links.
func startWalk(c *dhCreator, root string, walkFn filepath.WalkFunc) error {
info, err := os.Lstat(root)
info, err := c.fs.Lstat(root)
if err != nil {
return walkFn(root, nil, err)
}
@ -248,14 +251,14 @@ func walk(c *dhCreator, path string, info os.FileInfo, walkFn filepath.WalkFunc)
return nil
}
names, err := readOrderedDirNames(path)
names, err := readOrderedDirNames(c, path)
if err != nil {
return walkFn(path, info, err)
}
for _, name := range names {
filename := filepath.Join(path, name)
fileInfo, err := os.Lstat(filename)
fileInfo, err := c.fs.Lstat(filename)
if err != nil {
if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir {
return err
@ -282,13 +285,8 @@ func walk(c *dhCreator, path string, info os.FileInfo, walkFn filepath.WalkFunc)
// readOrderedDirNames reads the directory and returns a sorted list of all
// entries with non-directories first, followed by directories.
func readOrderedDirNames(dirname string) ([]string, error) {
f, err := os.Open(dirname)
if err != nil {
return nil, err
}
infos, err := f.Readdir(-1)
f.Close()
func readOrderedDirNames(c *dhCreator, dirname string) ([]string, error) {
infos, err := c.fs.Readdir(dirname)
if err != nil {
return nil, err
}

4
walk_test.go
View file

@ -7,7 +7,7 @@ import (
)
func TestWalk(t *testing.T) {
dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"))
dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
if err != nil {
t.Fatal(err)
}
@ -37,7 +37,7 @@ func TestWalk(t *testing.T) {
}
func TestWalkDirectory(t *testing.T) {
dh, err := Walk(".", []ExcludeFunc{ExcludeNonDirectories}, []Keyword{"type"})
dh, err := Walk(".", []ExcludeFunc{ExcludeNonDirectories}, []Keyword{"type"}, nil)
if err != nil {
t.Fatal(err)
}