*: add an update/restore functionality

This allows for restoring some attributes of files from the state in an
mtree Manifest

Reported-by: Matthew Garrett <Matthewgarrett@google.com>
Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>

.travis.yml

@@ -1,6 +1,8 @@
 language: go
 go:
-  - 1.7.4
+  - 1.x
+  - 1.8.x
+  - 1.7.x
   - 1.6.3
 
 sudo: false

cmd/gomtree/main.go

@@ -16,12 +16,13 @@ import (
 
 var (
 	// Flags common with mtree(8)
-	flCreate        = flag.Bool("c", false, "create a directory hierarchy spec")
+	flCreate           = flag.Bool("c", false, "create a directory hierarchy spec")
-	flFile          = flag.String("f", "", "directory hierarchy spec to validate")
+	flFile             = flag.String("f", "", "directory hierarchy spec to validate")
-	flPath          = flag.String("p", "", "root path that the hierarchy spec is relative to")
+	flPath             = flag.String("p", "", "root path that the hierarchy spec is relative to")
-	flAddKeywords   = flag.String("K", "", "Add the specified (delimited by comma or space) keywords to the current set of keywords")
+	flAddKeywords      = flag.String("K", "", "Add the specified (delimited by comma or space) keywords to the current set of keywords")
-	flUseKeywords   = flag.String("k", "", "Use the specified (delimited by comma or space) keywords as the current set of keywords")
+	flUseKeywords      = flag.String("k", "", "Use the specified (delimited by comma or space) keywords as the current set of keywords")
-	flDirectoryOnly = flag.Bool("d", false, "Ignore everything except directory type files")
+	flDirectoryOnly    = flag.Bool("d", false, "Ignore everything except directory type files")
+	flUpdateAttributes = flag.Bool("u", false, "Modify the owner, group, permissions and xattrs of files, symbolic links and devices, to match the provided specification. This is not compatible with '-T'.")
 
 	// Flags unique to gomtree
 	flListKeywords     = flag.Bool("list-keywords", false, "List the keywords available")
@@ -208,6 +209,12 @@ func app() error {
 		excludes = append(excludes, mtree.ExcludeNonDirectories)
 	}
 
+	// -u
+	// Failing early here. Processing is done below.
+	if *flUpdateAttributes && *flTar != "" {
+		return fmt.Errorf("ERROR: -u can not be used with -T")
+	}
+
 	// -T <tar file>
 	if *flTar != "" {
 		var input io.Reader
@@ -242,6 +249,25 @@ func app() error {
 		}
 	}
 
+	// -u
+	if *flUpdateAttributes && stateDh != nil {
+		// -u
+		// this comes before the next case, intentionally.
+
+		// TODO brainstorm where to allow setting of xattrs. Maybe a new flag that allows a comma delimited list of keywords to update?
+		updateKeywords := []mtree.Keyword{"uid", "gid", "mode"}
+
+		result, err := mtree.Update(rootPath, stateDh, updateKeywords, nil)
+		if err != nil {
+			return err
+		}
+
+		if result != nil {
+			fmt.Printf("%#v\n", result)
+		}
+		return nil
+	}
+
 	// -c
 	if *flCreate {
 		fh := os.Stdout

compare.go

@@ -29,6 +29,10 @@ const (
 	// have different values (or have not been set in one of the
 	// manifests).
 	Modified DifferenceType = "modified"
+
+	// ErrorDifference represents an attempted update to the values of
+	// a keyword that failed
+	ErrorDifference DifferenceType = "errored"
 )
 
 // These functions return *type from the parameter. It's just shorthand, to
@@ -126,6 +130,7 @@ type KeyDelta struct {
 	name Keyword
 	old  string
 	new  string
+	err  error // used for update delta results
 }
 
 // Type returns the type of discrepancy encountered when comparing this key

debug.go

@@ -9,7 +9,15 @@ import (
 // DebugOutput is the where DEBUG output is written
 var DebugOutput = os.Stderr
 
-// Debugf does formatted output to DebugOutput, only if DEBUG environment variable is set
+// Debugln writes output to DebugOutput, only if DEBUG environment variable is set
+func Debugln(a ...interface{}) (n int, err error) {
+	if os.Getenv("DEBUG") != "" {
+		return fmt.Fprintf(DebugOutput, "[%d] [DEBUG] %s\n", time.Now().UnixNano(), a)
+	}
+	return 0, nil
+}
+
+// Debugf writes formatted output to DebugOutput, only if DEBUG environment variable is set
 func Debugf(format string, a ...interface{}) (n int, err error) {
 	if os.Getenv("DEBUG") != "" {
 		return fmt.Fprintf(DebugOutput, "[%d] [DEBUG] %s\n", time.Now().UnixNano(), fmt.Sprintf(format, a...))

update.go

@@ -0,0 +1,87 @@
+package mtree
+
+import (
+	"os"
+	"sort"
+)
+
+// DefaultUpdateKeywords is the default set of keywords that can take updates to the files on disk
+var DefaultUpdateKeywords = []Keyword{
+	"uid",
+	"gid",
+	"mode",
+	"time",
+	// TODO xattr
+}
+
+// Update attempts to set the attributes of root directory path, given the values of `keywords` in dh DirectoryHierarchy.
+func Update(root string, dh *DirectoryHierarchy, keywords []Keyword, fs FsEval) ([]InodeDelta, error) {
+	creator := dhCreator{DH: dh}
+	curDir, err := os.Getwd()
+	if err == nil {
+		defer os.Chdir(curDir)
+	}
+
+	if err := os.Chdir(root); err != nil {
+		return nil, err
+	}
+	sort.Sort(byPos(creator.DH.Entries))
+
+	results := []InodeDelta{}
+	for i, e := range creator.DH.Entries {
+		switch e.Type {
+		case SpecialType:
+			if e.Name == "/set" {
+				creator.curSet = &creator.DH.Entries[i]
+			} else if e.Name == "/unset" {
+				creator.curSet = nil
+			}
+			Debugf("%#v", e)
+			continue
+		case RelativeType, FullType:
+			e.Set = creator.curSet
+			pathname, err := e.Path()
+			if err != nil {
+				return nil, err
+			}
+
+			// filter the keywords to update on the file, from the keywords available for this entry:
+			var toCheck []KeyVal
+			toCheck = keyvalSelector(e.AllKeys(), keywords)
+			Debugf("toCheck(%q): %v", pathname, toCheck)
+
+			for _, kv := range toCheck {
+				if !InKeywordSlice(kv.Keyword(), keywords) {
+					continue
+				}
+				ukFunc, ok := UpdateKeywordFuncs[kv.Keyword()]
+				if !ok {
+					Debugf("no UpdateKeywordFunc for %s; skipping", kv.Keyword())
+					continue
+				}
+				if _, err := ukFunc(pathname, kv.Value()); err != nil {
+					results = append(results, InodeDelta{
+						diff: ErrorDifference,
+						path: pathname,
+						old:  e,
+						keys: []KeyDelta{
+							{
+								diff: ErrorDifference,
+								name: kv.Keyword(),
+								err:  err,
+							},
+						}})
+				}
+			}
+		}
+	}
+
+	return results, nil
+}
+
+// Result of an "update" returns the produced results
+type Result struct {
+	Path    string
+	Keyword Keyword
+	Got     string
+}

update_test.go

@@ -0,0 +1,104 @@
+// +build go1.7
+
+package mtree
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+)
+
+func TestUpdate(t *testing.T) {
+	content := []byte("I know half of you half as well as I ought to")
+	dir, err := ioutil.TempDir("", "test-check-keywords")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir) // clean up
+
+	tmpfn := filepath.Join(dir, "tmpfile")
+	if err := ioutil.WriteFile(tmpfn, content, 0666); err != nil {
+		t.Fatal(err)
+	}
+
+	// Walk this tempdir
+	dh, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Touch a file, so the mtime changes.
+	now := time.Now()
+	if err := os.Chtimes(tmpfn, now, now); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chmod(tmpfn, os.FileMode(0600)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Changing user is a little tough, but the group can be changed by a limited user to any group that the user is a member of. So just choose one that is not the current main group.
+	u, err := user.Current()
+	if err != nil {
+		t.Fatal(err)
+	}
+	ugroups, err := u.GroupIds()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, ugroup := range ugroups {
+		if ugroup == u.Gid {
+			continue
+		}
+		gid, err := strconv.Atoi(ugroup)
+		if err != nil {
+			t.Fatal(ugroup)
+		}
+		if err := os.Lchown(tmpfn, -1, gid); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Check for sanity. This ought to have failures
+	res, err := Check(dir, dh, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(res) == 0 {
+		t.Error("expected failures (like mtimes), but got none")
+	}
+	//dh.WriteTo(os.Stdout)
+
+	res, err = Update(dir, dh, DefaultUpdateKeywords, nil)
+	if err != nil {
+		t.Error(err)
+	}
+	if len(res) > 0 {
+		// pretty this shit up
+		buf, err := json.MarshalIndent(res, "", "  ")
+		if err != nil {
+			t.Errorf("%#v", res)
+		}
+		t.Error(string(buf))
+	}
+
+	// Now check that we're sane again
+	res, err = Check(dir, dh, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// should have no failures now
+	if len(res) > 0 {
+		// pretty this shit up
+		buf, err := json.MarshalIndent(res, "", "  ")
+		if err != nil {
+			t.Errorf("%#v", res)
+		}
+		t.Error(string(buf))
+	}
+
+}

updatefuncs.go

@@ -0,0 +1,108 @@
+package mtree
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// UpdateKeywordFunc is the signature for a function that will restore a file's
+// attributes. Where path is relative path to the file, and value to be
+// restored to.
+type UpdateKeywordFunc func(path string, value string) (os.FileInfo, error)
+
+// UpdateKeywordFuncs is the registered list of functions to update file attributes.
+// Keyed by the keyword as it would show up in the manifest
+var UpdateKeywordFuncs = map[Keyword]UpdateKeywordFunc{
+	"mode":     modeUpdateKeywordFunc,
+	"time":     timeUpdateKeywordFunc,
+	"tar_time": tartimeUpdateKeywordFunc,
+	"uid":      uidUpdateKeywordFunc,
+	"gid":      gidUpdateKeywordFunc,
+}
+
+func uidUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
+	uid, err := strconv.Atoi(value)
+	if err != nil {
+		return nil, err
+	}
+	if err := os.Lchown(path, uid, -1); err != nil {
+		return nil, err
+	}
+	return os.Lstat(path)
+}
+
+func gidUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
+	gid, err := strconv.Atoi(value)
+	if err != nil {
+		return nil, err
+	}
+	if err := os.Lchown(path, -1, gid); err != nil {
+		return nil, err
+	}
+	return os.Lstat(path)
+}
+
+func modeUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
+	vmode, err := strconv.ParseInt(value, 8, 32)
+	if err != nil {
+		return nil, err
+	}
+	Debugf("path: %q, value: %q, vmode: %o", path, value, vmode)
+	if err := os.Chmod(path, os.FileMode(vmode)); err != nil {
+		return nil, err
+	}
+	return os.Lstat(path)
+}
+
+// since tar_time will only be second level precision, then when restoring the
+// filepath from a tar_time, then compare the seconds first and only Chtimes if
+// the seconds value is different.
+func tartimeUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
+	info, err := os.Lstat(path)
+	if err != nil {
+		return nil, err
+	}
+
+	v := strings.SplitN(value, ".", 2)
+	if len(v) != 2 {
+		return nil, fmt.Errorf("expected a number like 1469104727.000000000")
+	}
+	sec, err := strconv.ParseInt(v[0], 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("expected seconds, but got %q", v[0])
+	}
+
+	// if the seconds are the same, don't do anything, because the file might
+	// have nanosecond value, and if using tar_time it would zero it out.
+	if info.ModTime().Unix() == sec {
+		return info, nil
+	}
+
+	vtime := time.Unix(sec, 0)
+	if err := os.Chtimes(path, vtime, vtime); err != nil {
+		return nil, err
+	}
+	return os.Lstat(path)
+}
+
+// this is nano second precision
+func timeUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
+	v := strings.SplitN(value, ".", 2)
+	if len(v) != 2 {
+		return nil, fmt.Errorf("expected a number like 1469104727.871937272")
+	}
+	nsec, err := strconv.ParseInt(v[0]+v[1], 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("expected nano seconds, but got %q", v[0]+v[1])
+	}
+	Debugf("arg: %q; nsec: %q", v[0]+v[1], nsec)
+
+	vtime := time.Unix(0, nsec)
+	if err := os.Chtimes(path, vtime, vtime); err != nil {
+		return nil, err
+	}
+	return os.Lstat(path)
+}

walk.go

@@ -23,7 +23,7 @@ var ExcludeNonDirectories = func(path string, info os.FileInfo) bool {
 	return !info.IsDir()
 }
 
-var defaultSetKeywords = []KeyVal{"type=file", "nlink=1", "flags=none", "mode=0664"}
+var defaultSetKeyVals = []KeyVal{"type=file", "nlink=1", "flags=none", "mode=0664"}
 
 // Walk from root directory and assemble the DirectoryHierarchy
 // * `excludes` provided are used to skip paths
@@ -88,7 +88,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval
 					Name:     "/set",
 					Type:     SpecialType,
 					Pos:      len(creator.DH.Entries),
-					Keywords: keyvalSelector(defaultSetKeywords, keywords),
+					Keywords: keyvalSelector(defaultSetKeyVals, keywords),
 				}
 				for _, keyword := range SetKeywords {
 					err := func() error {
@@ -161,7 +161,7 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval
 						Name:     "/set",
 						Type:     SpecialType,
 						Pos:      len(creator.DH.Entries),
-						Keywords: keyvalSelector(append(defaultSetKeywords, klist...), keywords),
+						Keywords: keyvalSelector(append(defaultSetKeyVals, klist...), keywords),
 					}
 					creator.curSet = &e
 					creator.DH.Entries = append(creator.DH.Entries, e)