From fc5450ed711e64037eadce2f1690faea80215cb3 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 2 Aug 2016 15:57:51 -0400 Subject: [PATCH] *: add an update/restore functionality This allows for restoring some attributes of files from the state in an mtree Manifest Reported-by: Matthew Garrett Signed-off-by: Vincent Batts --- .travis.yml | 4 +- cmd/gomtree/main.go | 38 +++++- compare.go | 5 + debug.go | 10 +- keywords_linux.go => keywordfuncs_linux.go | 0 ...upported.go => keywordfuncs_unsupported.go | 0 update.go | 87 ++++++++++++++ update_test.go | 104 +++++++++++++++++ updatefuncs.go | 108 ++++++++++++++++++ walk.go | 6 +- 10 files changed, 351 insertions(+), 11 deletions(-) rename keywords_linux.go => keywordfuncs_linux.go (100%) rename keywords_unsupported.go => keywordfuncs_unsupported.go (100%) create mode 100644 update.go create mode 100644 update_test.go create mode 100644 updatefuncs.go diff --git a/.travis.yml b/.travis.yml index 398a2bb..cb00680 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ language: go go: - - 1.7.4 + - 1.x + - 1.8.x + - 1.7.x - 1.6.3 sudo: false diff --git a/cmd/gomtree/main.go b/cmd/gomtree/main.go index 2f97690..4c3d488 100644 --- a/cmd/gomtree/main.go +++ b/cmd/gomtree/main.go @@ -16,12 +16,13 @@ import ( var ( // Flags common with mtree(8) - flCreate = flag.Bool("c", false, "create a directory hierarchy spec") - flFile = flag.String("f", "", "directory hierarchy spec to validate") - flPath = flag.String("p", "", "root path that the hierarchy spec is relative to") - flAddKeywords = flag.String("K", "", "Add the specified (delimited by comma or space) keywords to the current set of keywords") - flUseKeywords = flag.String("k", "", "Use the specified (delimited by comma or space) keywords as the current set of keywords") - flDirectoryOnly = flag.Bool("d", false, "Ignore everything except directory type files") + flCreate = flag.Bool("c", false, "create a directory hierarchy spec") + flFile = flag.String("f", "", "directory hierarchy spec to validate") + flPath = flag.String("p", "", "root path that the hierarchy spec is relative to") + flAddKeywords = flag.String("K", "", "Add the specified (delimited by comma or space) keywords to the current set of keywords") + flUseKeywords = flag.String("k", "", "Use the specified (delimited by comma or space) keywords as the current set of keywords") + flDirectoryOnly = flag.Bool("d", false, "Ignore everything except directory type files") + flUpdateAttributes = flag.Bool("u", false, "Modify the owner, group, permissions and xattrs of files, symbolic links and devices, to match the provided specification. This is not compatible with '-T'.") // Flags unique to gomtree flListKeywords = flag.Bool("list-keywords", false, "List the keywords available") @@ -208,6 +209,12 @@ func app() error { excludes = append(excludes, mtree.ExcludeNonDirectories) } + // -u + // Failing early here. Processing is done below. + if *flUpdateAttributes && *flTar != "" { + return fmt.Errorf("ERROR: -u can not be used with -T") + } + // -T if *flTar != "" { var input io.Reader @@ -242,6 +249,25 @@ func app() error { } } + // -u + if *flUpdateAttributes && stateDh != nil { + // -u + // this comes before the next case, intentionally. + + // TODO brainstorm where to allow setting of xattrs. Maybe a new flag that allows a comma delimited list of keywords to update? + updateKeywords := []mtree.Keyword{"uid", "gid", "mode"} + + result, err := mtree.Update(rootPath, stateDh, updateKeywords, nil) + if err != nil { + return err + } + + if result != nil { + fmt.Printf("%#v\n", result) + } + return nil + } + // -c if *flCreate { fh := os.Stdout diff --git a/compare.go b/compare.go index 57fb444..73cebc7 100644 --- a/compare.go +++ b/compare.go @@ -29,6 +29,10 @@ const ( // have different values (or have not been set in one of the // manifests). Modified DifferenceType = "modified" + + // ErrorDifference represents an attempted update to the values of + // a keyword that failed + ErrorDifference DifferenceType = "errored" ) // These functions return *type from the parameter. It's just shorthand, to @@ -126,6 +130,7 @@ type KeyDelta struct { name Keyword old string new string + err error // used for update delta results } // Type returns the type of discrepancy encountered when comparing this key diff --git a/debug.go b/debug.go index 832035e..2decc6d 100644 --- a/debug.go +++ b/debug.go @@ -9,7 +9,15 @@ import ( // DebugOutput is the where DEBUG output is written var DebugOutput = os.Stderr -// Debugf does formatted output to DebugOutput, only if DEBUG environment variable is set +// Debugln writes output to DebugOutput, only if DEBUG environment variable is set +func Debugln(a ...interface{}) (n int, err error) { + if os.Getenv("DEBUG") != "" { + return fmt.Fprintf(DebugOutput, "[%d] [DEBUG] %s\n", time.Now().UnixNano(), a) + } + return 0, nil +} + +// Debugf writes formatted output to DebugOutput, only if DEBUG environment variable is set func Debugf(format string, a ...interface{}) (n int, err error) { if os.Getenv("DEBUG") != "" { return fmt.Fprintf(DebugOutput, "[%d] [DEBUG] %s\n", time.Now().UnixNano(), fmt.Sprintf(format, a...)) diff --git a/keywords_linux.go b/keywordfuncs_linux.go similarity index 100% rename from keywords_linux.go rename to keywordfuncs_linux.go diff --git a/keywords_unsupported.go b/keywordfuncs_unsupported.go similarity index 100% rename from keywords_unsupported.go rename to keywordfuncs_unsupported.go diff --git a/update.go b/update.go new file mode 100644 index 0000000..ec93e02 --- /dev/null +++ b/update.go @@ -0,0 +1,87 @@ +package mtree + +import ( + "os" + "sort" +) + +// DefaultUpdateKeywords is the default set of keywords that can take updates to the files on disk +var DefaultUpdateKeywords = []Keyword{ + "uid", + "gid", + "mode", + "time", + // TODO xattr +} + +// Update attempts to set the attributes of root directory path, given the values of `keywords` in dh DirectoryHierarchy. +func Update(root string, dh *DirectoryHierarchy, keywords []Keyword, fs FsEval) ([]InodeDelta, error) { + creator := dhCreator{DH: dh} + curDir, err := os.Getwd() + if err == nil { + defer os.Chdir(curDir) + } + + if err := os.Chdir(root); err != nil { + return nil, err + } + sort.Sort(byPos(creator.DH.Entries)) + + results := []InodeDelta{} + for i, e := range creator.DH.Entries { + switch e.Type { + case SpecialType: + if e.Name == "/set" { + creator.curSet = &creator.DH.Entries[i] + } else if e.Name == "/unset" { + creator.curSet = nil + } + Debugf("%#v", e) + continue + case RelativeType, FullType: + e.Set = creator.curSet + pathname, err := e.Path() + if err != nil { + return nil, err + } + + // filter the keywords to update on the file, from the keywords available for this entry: + var toCheck []KeyVal + toCheck = keyvalSelector(e.AllKeys(), keywords) + Debugf("toCheck(%q): %v", pathname, toCheck) + + for _, kv := range toCheck { + if !InKeywordSlice(kv.Keyword(), keywords) { + continue + } + ukFunc, ok := UpdateKeywordFuncs[kv.Keyword()] + if !ok { + Debugf("no UpdateKeywordFunc for %s; skipping", kv.Keyword()) + continue + } + if _, err := ukFunc(pathname, kv.Value()); err != nil { + results = append(results, InodeDelta{ + diff: ErrorDifference, + path: pathname, + old: e, + keys: []KeyDelta{ + { + diff: ErrorDifference, + name: kv.Keyword(), + err: err, + }, + }}) + } + } + } + } + + return results, nil +} + +// Result of an "update" returns the produced results +type Result struct { + Path string + Keyword Keyword + Got string +} diff --git a/update_test.go b/update_test.go new file mode 100644 index 0000000..6197e8a --- /dev/null +++ b/update_test.go @@ -0,0 +1,104 @@ +// +build go1.7 + +package mtree + +import ( + "encoding/json" + "io/ioutil" + "os" + "os/user" + "path/filepath" + "strconv" + "testing" + "time" +) + +func TestUpdate(t *testing.T) { + content := []byte("I know half of you half as well as I ought to") + dir, err := ioutil.TempDir("", "test-check-keywords") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) // clean up + + tmpfn := filepath.Join(dir, "tmpfile") + if err := ioutil.WriteFile(tmpfn, content, 0666); err != nil { + t.Fatal(err) + } + + // Walk this tempdir + dh, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil) + if err != nil { + t.Fatal(err) + } + + // Touch a file, so the mtime changes. + now := time.Now() + if err := os.Chtimes(tmpfn, now, now); err != nil { + t.Fatal(err) + } + if err := os.Chmod(tmpfn, os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + // Changing user is a little tough, but the group can be changed by a limited user to any group that the user is a member of. So just choose one that is not the current main group. + u, err := user.Current() + if err != nil { + t.Fatal(err) + } + ugroups, err := u.GroupIds() + if err != nil { + t.Fatal(err) + } + for _, ugroup := range ugroups { + if ugroup == u.Gid { + continue + } + gid, err := strconv.Atoi(ugroup) + if err != nil { + t.Fatal(ugroup) + } + if err := os.Lchown(tmpfn, -1, gid); err != nil { + t.Fatal(err) + } + } + + // Check for sanity. This ought to have failures + res, err := Check(dir, dh, nil, nil) + if err != nil { + t.Fatal(err) + } + if len(res) == 0 { + t.Error("expected failures (like mtimes), but got none") + } + //dh.WriteTo(os.Stdout) + + res, err = Update(dir, dh, DefaultUpdateKeywords, nil) + if err != nil { + t.Error(err) + } + if len(res) > 0 { + // pretty this shit up + buf, err := json.MarshalIndent(res, "", " ") + if err != nil { + t.Errorf("%#v", res) + } + t.Error(string(buf)) + } + + // Now check that we're sane again + res, err = Check(dir, dh, nil, nil) + if err != nil { + t.Fatal(err) + } + // should have no failures now + if len(res) > 0 { + // pretty this shit up + buf, err := json.MarshalIndent(res, "", " ") + if err != nil { + t.Errorf("%#v", res) + } + t.Error(string(buf)) + } + +} diff --git a/updatefuncs.go b/updatefuncs.go new file mode 100644 index 0000000..1a5867e --- /dev/null +++ b/updatefuncs.go @@ -0,0 +1,108 @@ +package mtree + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" +) + +// UpdateKeywordFunc is the signature for a function that will restore a file's +// attributes. Where path is relative path to the file, and value to be +// restored to. +type UpdateKeywordFunc func(path string, value string) (os.FileInfo, error) + +// UpdateKeywordFuncs is the registered list of functions to update file attributes. +// Keyed by the keyword as it would show up in the manifest +var UpdateKeywordFuncs = map[Keyword]UpdateKeywordFunc{ + "mode": modeUpdateKeywordFunc, + "time": timeUpdateKeywordFunc, + "tar_time": tartimeUpdateKeywordFunc, + "uid": uidUpdateKeywordFunc, + "gid": gidUpdateKeywordFunc, +} + +func uidUpdateKeywordFunc(path, value string) (os.FileInfo, error) { + uid, err := strconv.Atoi(value) + if err != nil { + return nil, err + } + if err := os.Lchown(path, uid, -1); err != nil { + return nil, err + } + return os.Lstat(path) +} + +func gidUpdateKeywordFunc(path, value string) (os.FileInfo, error) { + gid, err := strconv.Atoi(value) + if err != nil { + return nil, err + } + if err := os.Lchown(path, -1, gid); err != nil { + return nil, err + } + return os.Lstat(path) +} + +func modeUpdateKeywordFunc(path, value string) (os.FileInfo, error) { + vmode, err := strconv.ParseInt(value, 8, 32) + if err != nil { + return nil, err + } + Debugf("path: %q, value: %q, vmode: %o", path, value, vmode) + if err := os.Chmod(path, os.FileMode(vmode)); err != nil { + return nil, err + } + return os.Lstat(path) +} + +// since tar_time will only be second level precision, then when restoring the +// filepath from a tar_time, then compare the seconds first and only Chtimes if +// the seconds value is different. +func tartimeUpdateKeywordFunc(path, value string) (os.FileInfo, error) { + info, err := os.Lstat(path) + if err != nil { + return nil, err + } + + v := strings.SplitN(value, ".", 2) + if len(v) != 2 { + return nil, fmt.Errorf("expected a number like 1469104727.000000000") + } + sec, err := strconv.ParseInt(v[0], 10, 64) + if err != nil { + return nil, fmt.Errorf("expected seconds, but got %q", v[0]) + } + + // if the seconds are the same, don't do anything, because the file might + // have nanosecond value, and if using tar_time it would zero it out. + if info.ModTime().Unix() == sec { + return info, nil + } + + vtime := time.Unix(sec, 0) + if err := os.Chtimes(path, vtime, vtime); err != nil { + return nil, err + } + return os.Lstat(path) +} + +// this is nano second precision +func timeUpdateKeywordFunc(path, value string) (os.FileInfo, error) { + v := strings.SplitN(value, ".", 2) + if len(v) != 2 { + return nil, fmt.Errorf("expected a number like 1469104727.871937272") + } + nsec, err := strconv.ParseInt(v[0]+v[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("expected nano seconds, but got %q", v[0]+v[1]) + } + Debugf("arg: %q; nsec: %q", v[0]+v[1], nsec) + + vtime := time.Unix(0, nsec) + if err := os.Chtimes(path, vtime, vtime); err != nil { + return nil, err + } + return os.Lstat(path) +} diff --git a/walk.go b/walk.go index 5ddbf6c..16cafe1 100644 --- a/walk.go +++ b/walk.go @@ -23,7 +23,7 @@ var ExcludeNonDirectories = func(path string, info os.FileInfo) bool { return !info.IsDir() } -var defaultSetKeywords = []KeyVal{"type=file", "nlink=1", "flags=none", "mode=0664"} +var defaultSetKeyVals = []KeyVal{"type=file", "nlink=1", "flags=none", "mode=0664"} // Walk from root directory and assemble the DirectoryHierarchy // * `excludes` provided are used to skip paths @@ -88,7 +88,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval Name: "/set", Type: SpecialType, Pos: len(creator.DH.Entries), - Keywords: keyvalSelector(defaultSetKeywords, keywords), + Keywords: keyvalSelector(defaultSetKeyVals, keywords), } for _, keyword := range SetKeywords { err := func() error { @@ -161,7 +161,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval Name: "/set", Type: SpecialType, Pos: len(creator.DH.Entries), - Keywords: keyvalSelector(append(defaultSetKeywords, klist...), keywords), + Keywords: keyvalSelector(append(defaultSetKeyVals, klist...), keywords), } creator.curSet = &e creator.DH.Entries = append(creator.DH.Entries, e)