From a8e4475c5ee41147b0416beae002ee1f48399d7f Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Wed, 16 Nov 2016 14:15:42 -0500 Subject: [PATCH 1/5] *: clean up * Get rid of the isErr func in main() * put main() logic closer to the top Signed-off-by: Vincent Batts --- cmd/gomtree/main.go | 535 +++++++++++++++++++++----------------------- hierarchy.go | 29 +-- version.go | 5 + 3 files changed, 269 insertions(+), 300 deletions(-) diff --git a/cmd/gomtree/main.go b/cmd/gomtree/main.go index dfac662..d7b745d 100644 --- a/cmd/gomtree/main.go +++ b/cmd/gomtree/main.go @@ -29,6 +29,255 @@ var ( flVersion = flag.Bool("version", false, "display the version of this tool") ) +func main() { + // so that defers cleanly exec + if err := app(); err != nil { + log.Fatal(err) + } +} + +func app() error { + flag.Parse() + + if *flDebug { + os.Setenv("DEBUG", "1") + } + + if *flVersion { + fmt.Printf("%s :: %s\n", mtree.AppName, mtree.Version) + return nil + } + + // -list-keywords + if *flListKeywords { + fmt.Println("Available keywords:") + for k := range mtree.KeywordFuncs { + fmt.Print(" ") + fmt.Print(k) + if mtree.Keyword(k).Default() { + fmt.Print(" (default)") + } + if !mtree.Keyword(k).Bsd() { + fmt.Print(" (not upstream)") + } + fmt.Print("\n") + } + return nil + } + + // --result-format + formatFunc, ok := formats[*flResultFormat] + if !ok { + return fmt.Errorf("invalid output format: %s", *flResultFormat) + } + + var ( + err error + tmpKeywords []string + currentKeywords []string + ) + + // -k + if *flUseKeywords != "" { + tmpKeywords = splitKeywordsArg(*flUseKeywords) + if !inSlice("type", tmpKeywords) { + tmpKeywords = append([]string{"type"}, tmpKeywords...) + } + } else { + if *flTar != "" { + tmpKeywords = mtree.DefaultTarKeywords[:] + } else { + tmpKeywords = mtree.DefaultKeywords[:] + } + } + + // -K + if *flAddKeywords != "" { + for _, kw := range splitKeywordsArg(*flAddKeywords) { + if !inSlice(kw, tmpKeywords) { + tmpKeywords = append(tmpKeywords, kw) + } + } + } + + // -bsd-keywords + if *flBsdKeywords { + for _, k := range tmpKeywords { + if mtree.Keyword(k).Bsd() { + currentKeywords = append(currentKeywords, k) + } else { + fmt.Fprintf(os.Stderr, "INFO: ignoring %q as it is not an upstream keyword\n", k) + } + } + } else { + currentKeywords = tmpKeywords + } + + // Check mutual exclusivity of keywords. + // TODO(cyphar): Abstract this inside keywords.go. + if inSlice("tar_time", currentKeywords) && inSlice("time", currentKeywords) { + return fmt.Errorf("tar_time and time are mutually exclusive keywords") + } + + // If we're doing a comparison, we always are comparing between a spec and + // state DH. If specDh is nil, we are generating a new one. + var ( + specDh *mtree.DirectoryHierarchy + stateDh *mtree.DirectoryHierarchy + specKeywords []string + ) + + // -f + if *flFile != "" && !*flCreate { + // load the hierarchy, if we're not creating a new spec + fh, err := os.Open(*flFile) + if err != nil { + return err + } + specDh, err = mtree.ParseSpec(fh) + fh.Close() + if err != nil { + return err + } + + // We can't check against more fields than in the specKeywords list, so + // currentKeywords can only have a subset of specKeywords. + specKeywords = mtree.CollectUsedKeywords(specDh) + } + + // -list-used + if *flListUsedKeywords { + if specDh == nil { + return fmt.Errorf("no specification provided. please provide a validation manifest") + } + + if *flResultFormat == "json" { + // if they're asking for json, give it to them + data := map[string][]string{*flFile: specKeywords} + buf, err := json.MarshalIndent(data, "", " ") + if err != nil { + return err + } + fmt.Println(string(buf)) + } else { + fmt.Printf("Keywords used in [%s]:\n", *flFile) + for _, kw := range specKeywords { + fmt.Printf(" %s", kw) + if _, ok := mtree.KeywordFuncs[kw]; !ok { + fmt.Print(" (unsupported)") + } + fmt.Printf("\n") + } + } + return nil + } + + if specKeywords != nil { + // If we didn't actually change the set of keywords, we can just use specKeywords. + if *flUseKeywords == "" && *flAddKeywords == "" { + currentKeywords = specKeywords + } + + for _, keyword := range currentKeywords { + // As always, time is a special case. + // TODO: Fix that. + if (keyword == "time" && inSlice("tar_time", specKeywords)) || (keyword == "tar_time" && inSlice("time", specKeywords)) { + continue + } + + if !inSlice(keyword, specKeywords) { + return fmt.Errorf("cannot verify keywords not in mtree specification: %s\n", keyword) + } + } + } + + // -p and -T are mutually exclusive + if *flPath != "" && *flTar != "" { + return fmt.Errorf("options -T and -p are mutually exclusive") + } + + // -p + var rootPath = "." + if *flPath != "" { + rootPath = *flPath + } + + // -T + if *flTar != "" { + var input io.Reader + if *flTar == "-" { + input = os.Stdin + } else { + fh, err := os.Open(*flTar) + if err != nil { + return err + } + defer fh.Close() + input = fh + } + ts := mtree.NewTarStreamer(input, currentKeywords) + + if _, err := io.Copy(ioutil.Discard, ts); err != nil && err != io.EOF { + return err + } + if err := ts.Close(); err != nil { + return err + } + var err error + stateDh, err = ts.Hierarchy() + if err != nil { + return err + } + } else { + // with a root directory + stateDh, err = mtree.Walk(rootPath, nil, currentKeywords) + if err != nil { + return err + } + } + + // -c + if *flCreate { + fh := os.Stdout + if *flFile != "" { + fh, err = os.Create(*flFile) + if err != nil { + return err + } + } + + // output stateDh + stateDh.WriteTo(fh) + return nil + } + + // This is a validation. + if specDh != nil && stateDh != nil { + var res []mtree.InodeDelta + + res, err = mtree.Compare(specDh, stateDh, currentKeywords) + if err != nil { + return err + } + if res != nil { + if isTarSpec(specDh) || *flTar != "" { + res = filterMissingKeywords(res) + } + if len(res) > 0 { + return fmt.Errorf("unexpected missing keywords: %d", len(res)) + } + + out := formatFunc(res) + if _, err := os.Stdout.Write([]byte(out)); err != nil { + return err + } + } + } else { + return fmt.Errorf("neither validating or creating a manifest. Please provide additional arguments") + } + return nil +} + var formats = map[string]func([]mtree.InodeDelta) string{ // Outputs the errors in the BSD format. "bsd": func(d []mtree.InodeDelta) string { @@ -146,292 +395,6 @@ func isTarSpec(spec *mtree.DirectoryHierarchy) bool { return false } -func main() { - flag.Parse() - - if *flDebug { - os.Setenv("DEBUG", "1") - } - - // so that defers cleanly exec - // TODO: Switch everything to being inside a function, to remove the need for isErr. - var isErr bool - defer func() { - if isErr { - os.Exit(1) - } - }() - - if *flVersion { - fmt.Printf("%s :: %s\n", os.Args[0], mtree.Version) - return - } - - // -list-keywords - if *flListKeywords { - fmt.Println("Available keywords:") - for k := range mtree.KeywordFuncs { - fmt.Print(" ") - fmt.Print(k) - if mtree.Keyword(k).Default() { - fmt.Print(" (default)") - } - if !mtree.Keyword(k).Bsd() { - fmt.Print(" (not upstream)") - } - fmt.Print("\n") - } - return - } - - // --result-format - formatFunc, ok := formats[*flResultFormat] - if !ok { - log.Printf("invalid output format: %s", *flResultFormat) - isErr = true - return - } - - var ( - err error - tmpKeywords []string - currentKeywords []string - ) - - // -k - if *flUseKeywords != "" { - tmpKeywords = splitKeywordsArg(*flUseKeywords) - if !inSlice("type", tmpKeywords) { - tmpKeywords = append([]string{"type"}, tmpKeywords...) - } - } else { - if *flTar != "" { - tmpKeywords = mtree.DefaultTarKeywords[:] - } else { - tmpKeywords = mtree.DefaultKeywords[:] - } - } - - // -K - if *flAddKeywords != "" { - for _, kw := range splitKeywordsArg(*flAddKeywords) { - if !inSlice(kw, tmpKeywords) { - tmpKeywords = append(tmpKeywords, kw) - } - } - } - - // -bsd-keywords - if *flBsdKeywords { - for _, k := range tmpKeywords { - if mtree.Keyword(k).Bsd() { - currentKeywords = append(currentKeywords, k) - } else { - fmt.Fprintf(os.Stderr, "INFO: ignoring %q as it is not an upstream keyword\n", k) - } - } - } else { - currentKeywords = tmpKeywords - } - - // Check mutual exclusivity of keywords. - // TODO(cyphar): Abstract this inside keywords.go. - if inSlice("tar_time", currentKeywords) && inSlice("time", currentKeywords) { - log.Printf("tar_time and time are mutually exclusive keywords") - isErr = true - return - } - - // If we're doing a comparison, we always are comparing between a spec and - // state DH. If specDh is nil, we are generating a new one. - var ( - specDh *mtree.DirectoryHierarchy - stateDh *mtree.DirectoryHierarchy - specKeywords []string - ) - - // -f - if *flFile != "" && !*flCreate { - // load the hierarchy, if we're not creating a new spec - fh, err := os.Open(*flFile) - if err != nil { - log.Println(err) - isErr = true - return - } - specDh, err = mtree.ParseSpec(fh) - fh.Close() - if err != nil { - log.Println(err) - isErr = true - return - } - - // We can't check against more fields than in the specKeywords list, so - // currentKeywords can only have a subset of specKeywords. - specKeywords = mtree.CollectUsedKeywords(specDh) - } - - // -list-used - if *flListUsedKeywords { - if specDh == nil { - log.Println("no specification provided. please provide a validation manifest") - isErr = true - return - } - - if *flResultFormat == "json" { - // if they're asking for json, give it to them - data := map[string][]string{*flFile: specKeywords} - buf, err := json.MarshalIndent(data, "", " ") - if err != nil { - defer os.Exit(1) - isErr = true - return - } - fmt.Println(string(buf)) - } else { - fmt.Printf("Keywords used in [%s]:\n", *flFile) - for _, kw := range specKeywords { - fmt.Printf(" %s", kw) - if _, ok := mtree.KeywordFuncs[kw]; !ok { - fmt.Print(" (unsupported)") - } - fmt.Printf("\n") - } - } - return - } - - if specKeywords != nil { - // If we didn't actually change the set of keywords, we can just use specKeywords. - if *flUseKeywords == "" && *flAddKeywords == "" { - currentKeywords = specKeywords - } - - for _, keyword := range currentKeywords { - // As always, time is a special case. - // TODO: Fix that. - if (keyword == "time" && inSlice("tar_time", specKeywords)) || (keyword == "tar_time" && inSlice("time", specKeywords)) { - continue - } - - if !inSlice(keyword, specKeywords) { - log.Printf("cannot verify keywords not in mtree specification: %s\n", keyword) - isErr = true - } - if isErr { - return - } - } - } - - // -p and -T are mutually exclusive - if *flPath != "" && *flTar != "" { - log.Println("options -T and -p are mutually exclusive") - isErr = true - return - } - - // -p - var rootPath = "." - if *flPath != "" { - rootPath = *flPath - } - - // -T - if *flTar != "" { - var input io.Reader - if *flTar == "-" { - input = os.Stdin - } else { - fh, err := os.Open(*flTar) - if err != nil { - log.Println(err) - isErr = true - return - } - defer fh.Close() - input = fh - } - ts := mtree.NewTarStreamer(input, currentKeywords) - - if _, err := io.Copy(ioutil.Discard, ts); err != nil && err != io.EOF { - log.Println(err) - isErr = true - return - } - if err := ts.Close(); err != nil { - log.Println(err) - isErr = true - return - } - var err error - stateDh, err = ts.Hierarchy() - if err != nil { - log.Println(err) - isErr = true - return - } - } else { - // with a root directory - stateDh, err = mtree.Walk(rootPath, nil, currentKeywords) - if err != nil { - log.Println(err) - isErr = true - return - } - } - - // -c - if *flCreate { - fh := os.Stdout - if *flFile != "" { - fh, err = os.Create(*flFile) - if err != nil { - log.Println(err) - isErr = true - return - } - } - - // output stateDh - stateDh.WriteTo(fh) - return - } - - // This is a validation. - if specDh != nil && stateDh != nil { - var res []mtree.InodeDelta - - res, err = mtree.Compare(specDh, stateDh, currentKeywords) - if err != nil { - log.Println(err) - isErr = true - return - } - if res != nil { - if isTarSpec(specDh) || *flTar != "" { - res = filterMissingKeywords(res) - } - if len(res) > 0 { - defer os.Exit(1) - } - - out := formatFunc(res) - if _, err := os.Stdout.Write([]byte(out)); err != nil { - log.Println(err) - isErr = true - return - } - } - } else { - log.Println("neither validating or creating a manifest. Please provide additional arguments") - isErr = true - return - } -} - func splitKeywordsArg(str string) []string { return strings.Fields(strings.Replace(str, ",", " ", -1)) } diff --git a/hierarchy.go b/hierarchy.go index 60c9192..b0d6dac 100644 --- a/hierarchy.go +++ b/hierarchy.go @@ -29,23 +29,24 @@ func (dh DirectoryHierarchy) WriteTo(w io.Writer) (n int64, err error) { // CollectUsedKeywords collects and returns all the keywords used in a // a DirectoryHierarchy func CollectUsedKeywords(dh *DirectoryHierarchy) []string { - if dh != nil { - usedkeywords := []string{} - for _, e := range dh.Entries { - switch e.Type { - case FullType, RelativeType, SpecialType: - if e.Type != SpecialType || e.Name == "/set" { - kvs := e.Keywords - for _, kv := range kvs { - kw := KeyVal(kv).Keyword() - if !inSlice(kw, usedkeywords) { - usedkeywords = append(usedkeywords, kw) - } + if dh == nil { + return nil + } + + usedkeywords := []string{} + for _, e := range dh.Entries { + switch e.Type { + case FullType, RelativeType, SpecialType: + if e.Type != SpecialType || e.Name == "/set" { + kvs := e.Keywords + for _, kv := range kvs { + kw := KeyVal(kv).Keyword() + if !inSlice(kw, usedkeywords) { + usedkeywords = append(usedkeywords, kw) } } } } - return usedkeywords } - return nil + return usedkeywords } diff --git a/version.go b/version.go index 401dd3d..c826bab 100644 --- a/version.go +++ b/version.go @@ -2,6 +2,11 @@ package mtree import "fmt" +const ( + // AppName is the name ... of this library/application + AppName = "gomtree" +) + const ( // VersionMajor is for an API incompatible changes VersionMajor = 0 From 627c6e9ddd5f5c38d25f08028cecc4ae128d4a71 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Wed, 16 Nov 2016 14:43:05 -0500 Subject: [PATCH 2/5] DirectoryHierarchy: UsedKeywords is a of the struct Signed-off-by: Vincent Batts --- check.go | 4 ++-- check_test.go | 2 +- cmd/gomtree/main.go | 2 +- hierarchy.go | 8 ++------ 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/check.go b/check.go index 5ca345d..96284d1 100644 --- a/check.go +++ b/check.go @@ -8,7 +8,7 @@ package mtree // keywords) and then doing a Compare(dh, newDh, keywords). func Check(root string, dh *DirectoryHierarchy, keywords []string) ([]InodeDelta, error) { if keywords == nil { - keywords = CollectUsedKeywords(dh) + keywords = dh.UsedKeywords() } newDh, err := Walk(root, nil, keywords) @@ -25,7 +25,7 @@ func Check(root string, dh *DirectoryHierarchy, keywords []string) ([]InodeDelta // equivalent to Compare(dh, tarDH, keywords). func TarCheck(tarDH, dh *DirectoryHierarchy, keywords []string) ([]InodeDelta, error) { if keywords == nil { - keywords = CollectUsedKeywords(dh) + keywords = dh.UsedKeywords() } return Compare(dh, tarDH, keywords) } diff --git a/check_test.go b/check_test.go index 08e9211..5cdf756 100644 --- a/check_test.go +++ b/check_test.go @@ -200,7 +200,7 @@ func TestTarTime(t *testing.T) { t.Fatal(err) } - keywords := CollectUsedKeywords(dh) + keywords := dh.UsedKeywords() // make sure "time" keyword works _, err = Check(dir, dh, keywords) diff --git a/cmd/gomtree/main.go b/cmd/gomtree/main.go index d7b745d..cd81344 100644 --- a/cmd/gomtree/main.go +++ b/cmd/gomtree/main.go @@ -142,7 +142,7 @@ func app() error { // We can't check against more fields than in the specKeywords list, so // currentKeywords can only have a subset of specKeywords. - specKeywords = mtree.CollectUsedKeywords(specDh) + specKeywords = specDh.UsedKeywords() } // -list-used diff --git a/hierarchy.go b/hierarchy.go index b0d6dac..88a007d 100644 --- a/hierarchy.go +++ b/hierarchy.go @@ -26,13 +26,9 @@ func (dh DirectoryHierarchy) WriteTo(w io.Writer) (n int64, err error) { return sum, nil } -// CollectUsedKeywords collects and returns all the keywords used in a +// UsedKeywords collects and returns all the keywords used in a // a DirectoryHierarchy -func CollectUsedKeywords(dh *DirectoryHierarchy) []string { - if dh == nil { - return nil - } - +func (dh DirectoryHierarchy) UsedKeywords() []string { usedkeywords := []string{} for _, e := range dh.Entries { switch e.Type { From 5d26726bb11e24fc7143f3fe85f6e15ea2da360d Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Wed, 16 Nov 2016 15:42:53 -0500 Subject: [PATCH 3/5] keyword: unify keyword synonyms Signed-off-by: Vincent Batts --- cmd/gomtree/main.go | 6 +++++- hierarchy.go | 2 +- hierarchy_test.go | 38 ++++++++++++++++++++++++++++++++++++++ keywords.go | 27 ++++++++++++++++++++++++++- keywords_test.go | 30 ++++++++++++++++++++++++++++++ test/cli/0004.sh | 24 ++++++++++++++++++++++++ 6 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 hierarchy_test.go create mode 100644 test/cli/0004.sh diff --git a/cmd/gomtree/main.go b/cmd/gomtree/main.go index cd81344..92c6f6a 100644 --- a/cmd/gomtree/main.go +++ b/cmd/gomtree/main.go @@ -396,7 +396,11 @@ func isTarSpec(spec *mtree.DirectoryHierarchy) bool { } func splitKeywordsArg(str string) []string { - return strings.Fields(strings.Replace(str, ",", " ", -1)) + keywords := []string{} + for _, kw := range strings.Fields(strings.Replace(str, ",", " ", -1)) { + keywords = append(keywords, mtree.KeywordSynonym(kw)) + } + return keywords } func inSlice(a string, list []string) bool { diff --git a/hierarchy.go b/hierarchy.go index 88a007d..3c8da39 100644 --- a/hierarchy.go +++ b/hierarchy.go @@ -38,7 +38,7 @@ func (dh DirectoryHierarchy) UsedKeywords() []string { for _, kv := range kvs { kw := KeyVal(kv).Keyword() if !inSlice(kw, usedkeywords) { - usedkeywords = append(usedkeywords, kw) + usedkeywords = append(usedkeywords, KeywordSynonym(kw)) } } } diff --git a/hierarchy_test.go b/hierarchy_test.go new file mode 100644 index 0000000..0a58e56 --- /dev/null +++ b/hierarchy_test.go @@ -0,0 +1,38 @@ +package mtree + +import ( + "strings" + "testing" +) + +var checklist = []struct { + blob string + set []string +}{ + {blob: ` +# machine: bananaboat +# tree: .git +# date: Wed Nov 16 14:54:17 2016 + +# . +/set type=file nlink=1 mode=0664 uid=1000 gid=100 +. size=4096 type=dir mode=0755 nlink=8 time=1479326055.423853146 + .COMMIT_EDITMSG.un~ size=1006 mode=0644 time=1479325423.450468662 sha1digest=dead0face + .TAG_EDITMSG.un~ size=1069 mode=0600 time=1471362316.801317529 sha256digest=dead0face +`, set: []string{"size", "mode", "time", "sha256digest"}}, +} + +func TestUsedKeywords(t *testing.T) { + for i, item := range checklist { + dh, err := ParseSpec(strings.NewReader(item.blob)) + if err != nil { + t.Error(err) + } + used := dh.UsedKeywords() + for _, k := range item.set { + if !inSlice(k, used) { + t.Errorf("%d: expected to find %q in %q", i, k, used) + } + } + } +} diff --git a/keywords.go b/keywords.go index fd996d3..93916f4 100644 --- a/keywords.go +++ b/keywords.go @@ -251,6 +251,31 @@ var ( } ) +// KeywordSynonym returns the canonical name for keywords that have synonyms, +// and just returns the name provided if there is no synonym. In this way it +// ought to be safe to wrap any keyword name. +func KeywordSynonym(name string) string { + switch name { + case "md5": + return "md5digest" + case "rmd160": + return "ripemd160digest" + case "rmd160digest": + return "ripemd160digest" + case "sha1": + return "sha1digest" + case "sha256": + return "sha256digest" + case "sha384": + return "sha384digest" + case "sha512": + return "sha512digest" + case "xattrs": + return "xattr" + } + return name +} + var ( modeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { permissions := info.Mode().Perm() @@ -292,7 +317,7 @@ var ( if _, err := io.Copy(h, r); err != nil { return "", err } - return fmt.Sprintf("%s=%x", name, h.Sum(nil)), nil + return fmt.Sprintf("%s=%x", KeywordSynonym(name), h.Sum(nil)), nil } } tartimeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { diff --git a/keywords_test.go b/keywords_test.go index 5860fc0..4411453 100644 --- a/keywords_test.go +++ b/keywords_test.go @@ -93,3 +93,33 @@ func TestKeywordsTimeTar(t *testing.T) { } } } + +func TestKeywordSynonym(t *testing.T) { + checklist := []struct { + give, expect string + }{ + {give: "time", expect: "time"}, + {give: "md5", expect: "md5digest"}, + {give: "md5digest", expect: "md5digest"}, + {give: "rmd160", expect: "ripemd160digest"}, + {give: "rmd160digest", expect: "ripemd160digest"}, + {give: "ripemd160digest", expect: "ripemd160digest"}, + {give: "sha1", expect: "sha1digest"}, + {give: "sha1digest", expect: "sha1digest"}, + {give: "sha256", expect: "sha256digest"}, + {give: "sha256digest", expect: "sha256digest"}, + {give: "sha384", expect: "sha384digest"}, + {give: "sha384digest", expect: "sha384digest"}, + {give: "sha512", expect: "sha512digest"}, + {give: "sha512digest", expect: "sha512digest"}, + {give: "xattr", expect: "xattr"}, + {give: "xattrs", expect: "xattr"}, + } + + for i, check := range checklist { + got := KeywordSynonym(check.give) + if got != check.expect { + t.Errorf("%d: expected %q; got %q", i, check.expect, got) + } + } +} diff --git a/test/cli/0004.sh b/test/cli/0004.sh new file mode 100644 index 0000000..b538285 --- /dev/null +++ b/test/cli/0004.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -e +#set -x + +name=$(basename $0) +root=$1 +gomtree=$(readlink -f ${root}/gomtree) +t=$(mktemp -d /tmp/go-mtree.XXXXXX) + +echo "[${name}] Running in ${t}" + +pushd ${root} +git archive --format=tar HEAD^{tree} . > ${t}/${name}.tar +mkdir -p ${t}/extract +tar -C ${t}/extract/ -xf ${t}/${name}.tar + +## This is a checking that keyword synonyms are respected +${gomtree} -k sha1digest -c -p ${t}/extract/ > ${t}/${name}.mtree +${gomtree} -k sha1 -f ${t}/${name}.mtree -p ${t}/extract/ +${gomtree} -k sha1 -c -p ${t}/extract/ > ${t}/${name}.mtree +${gomtree} -k sha1digest -f ${t}/${name}.mtree -p ${t}/extract/ + +popd +rm -rf ${t} From 4eec68be4b2611b32378fcc8cf27b1ace774c0f2 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 17 Nov 2016 19:47:31 -0500 Subject: [PATCH 4/5] *: make Keyword and KeyVal pervasive Signed-off-by: Vincent Batts --- Makefile | 4 +- check.go | 14 +- check_test.go | 2 +- cmd/gomtree/main.go | 33 ++--- compare.go | 16 +-- compare_test.go | 2 +- entry.go | 19 ++- hierarchy.go | 8 +- hierarchy_test.go | 6 +- keywordfunc.go | 168 ++++++++++++++++++++++ keywords.go | 337 +++++++++++++++++--------------------------- keywords_bsd.go | 36 ++--- keywords_linux.go | 54 +++---- keywords_test.go | 7 +- parse.go | 4 +- tar.go | 50 +++---- tar_test.go | 16 +-- walk.go | 25 ++-- 18 files changed, 434 insertions(+), 367 deletions(-) create mode 100644 keywordfunc.go diff --git a/Makefile b/Makefile index ed20a67..a428984 100644 --- a/Makefile +++ b/Makefile @@ -3,10 +3,10 @@ BUILD := gomtree CWD := $(shell pwd) SOURCE_FILES := $(shell find . -type f -name "*.go") -default: validation build +default: build validation .PHONY: validation -validation: test lint vet .cli.test +validation: .test .lint .vet .cli.test .PHONY: test test: .test diff --git a/check.go b/check.go index 96284d1..2fff36f 100644 --- a/check.go +++ b/check.go @@ -6,16 +6,20 @@ package mtree // // This is equivalent to creating a new DirectoryHierarchy with Walk(root, nil, // keywords) and then doing a Compare(dh, newDh, keywords). -func Check(root string, dh *DirectoryHierarchy, keywords []string) ([]InodeDelta, error) { +func Check(root string, dh *DirectoryHierarchy, keywords []Keyword) ([]InodeDelta, error) { if keywords == nil { - keywords = dh.UsedKeywords() + used := dh.UsedKeywords() + newDh, err := Walk(root, nil, used) + if err != nil { + return nil, err + } + return Compare(dh, newDh, used) } newDh, err := Walk(root, nil, keywords) if err != nil { return nil, err } - // TODO: Handle tar_time, if necessary. return Compare(dh, newDh, keywords) } @@ -23,9 +27,9 @@ func Check(root string, dh *DirectoryHierarchy, keywords []string) ([]InodeDelta // TarCheck is the tar equivalent of checking a file hierarchy spec against a // tar stream to determine if files have been changed. This is precisely // equivalent to Compare(dh, tarDH, keywords). -func TarCheck(tarDH, dh *DirectoryHierarchy, keywords []string) ([]InodeDelta, error) { +func TarCheck(tarDH, dh *DirectoryHierarchy, keywords []Keyword) ([]InodeDelta, error) { if keywords == nil { - keywords = dh.UsedKeywords() + return Compare(dh, tarDH, dh.UsedKeywords()) } return Compare(dh, tarDH, keywords) } diff --git a/check_test.go b/check_test.go index 5cdf756..89024dd 100644 --- a/check_test.go +++ b/check_test.go @@ -76,7 +76,7 @@ func TestCheckKeywords(t *testing.T) { } // Check again, but only sha1 and mode. This ought to pass. - res, err = Check(dir, dh, []string{"sha1", "mode"}) + res, err = Check(dir, dh, []Keyword{"sha1", "mode"}) if err != nil { t.Fatal(err) } diff --git a/cmd/gomtree/main.go b/cmd/gomtree/main.go index 92c6f6a..8c940cb 100644 --- a/cmd/gomtree/main.go +++ b/cmd/gomtree/main.go @@ -73,15 +73,15 @@ func app() error { var ( err error - tmpKeywords []string - currentKeywords []string + tmpKeywords []mtree.Keyword + currentKeywords []mtree.Keyword ) // -k if *flUseKeywords != "" { tmpKeywords = splitKeywordsArg(*flUseKeywords) - if !inSlice("type", tmpKeywords) { - tmpKeywords = append([]string{"type"}, tmpKeywords...) + if !mtree.InKeywordSlice("type", tmpKeywords) { + tmpKeywords = append([]mtree.Keyword{"type"}, tmpKeywords...) } } else { if *flTar != "" { @@ -94,7 +94,7 @@ func app() error { // -K if *flAddKeywords != "" { for _, kw := range splitKeywordsArg(*flAddKeywords) { - if !inSlice(kw, tmpKeywords) { + if !mtree.InKeywordSlice(kw, tmpKeywords) { tmpKeywords = append(tmpKeywords, kw) } } @@ -115,7 +115,7 @@ func app() error { // Check mutual exclusivity of keywords. // TODO(cyphar): Abstract this inside keywords.go. - if inSlice("tar_time", currentKeywords) && inSlice("time", currentKeywords) { + if mtree.InKeywordSlice("tar_time", currentKeywords) && mtree.InKeywordSlice("time", currentKeywords) { return fmt.Errorf("tar_time and time are mutually exclusive keywords") } @@ -124,7 +124,7 @@ func app() error { var ( specDh *mtree.DirectoryHierarchy stateDh *mtree.DirectoryHierarchy - specKeywords []string + specKeywords []mtree.Keyword ) // -f @@ -153,7 +153,7 @@ func app() error { if *flResultFormat == "json" { // if they're asking for json, give it to them - data := map[string][]string{*flFile: specKeywords} + data := map[string][]mtree.Keyword{*flFile: specKeywords} buf, err := json.MarshalIndent(data, "", " ") if err != nil { return err @@ -181,11 +181,11 @@ func app() error { for _, keyword := range currentKeywords { // As always, time is a special case. // TODO: Fix that. - if (keyword == "time" && inSlice("tar_time", specKeywords)) || (keyword == "tar_time" && inSlice("time", specKeywords)) { + if (keyword == "time" && mtree.InKeywordSlice("tar_time", specKeywords)) || (keyword == "tar_time" && mtree.InKeywordSlice("time", specKeywords)) { continue } - if !inSlice(keyword, specKeywords) { + if !mtree.InKeywordSlice(keyword, specKeywords) { return fmt.Errorf("cannot verify keywords not in mtree specification: %s\n", keyword) } } @@ -395,19 +395,10 @@ func isTarSpec(spec *mtree.DirectoryHierarchy) bool { return false } -func splitKeywordsArg(str string) []string { - keywords := []string{} +func splitKeywordsArg(str string) []mtree.Keyword { + keywords := []mtree.Keyword{} for _, kw := range strings.Fields(strings.Replace(str, ",", " ", -1)) { keywords = append(keywords, mtree.KeywordSynonym(kw)) } return keywords } - -func inSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} diff --git a/compare.go b/compare.go index 1f7b9d2..7497b46 100644 --- a/compare.go +++ b/compare.go @@ -123,7 +123,7 @@ func (i InodeDelta) String() string { // returned with InodeDelta.Diff(). type KeyDelta struct { diff DifferenceType - name string + name Keyword old string new string } @@ -136,7 +136,7 @@ func (k KeyDelta) Type() DifferenceType { // Name returns the name (the key) of the KeyDeltaVal entry in the // DirectoryHierarchy. -func (k KeyDelta) Name() string { +func (k KeyDelta) Name() Keyword { return k.name } @@ -164,7 +164,7 @@ func (k KeyDelta) New() *string { func (k KeyDelta) MarshalJSON() ([]byte, error) { return json.Marshal(struct { Type DifferenceType `json:"type"` - Name string `json:"name"` + Name Keyword `json:"name"` Old string `json:"old"` New string `json:"new"` }{ @@ -184,7 +184,7 @@ func compareEntry(oldEntry, newEntry Entry) ([]KeyDelta, error) { New *KeyVal } - diffs := map[string]*stateT{} + diffs := map[Keyword]*stateT{} // Fill the map with the old keys first. for _, kv := range oldEntry.AllKeys() { @@ -218,7 +218,7 @@ func compareEntry(oldEntry, newEntry Entry) ([]KeyDelta, error) { // We need a full list of the keys so we can deal with different keyvalue // orderings. - var kws []string + var kws []Keyword for kw := range diffs { kws = append(kws, kw) } @@ -226,7 +226,7 @@ func compareEntry(oldEntry, newEntry Entry) ([]KeyDelta, error) { // If both tar_time and time were specified in the set of keys, we have to // mess with the diffs. This is an unfortunate side-effect of tar archives. // TODO(cyphar): This really should be abstracted inside keywords.go - if inSlice("tar_time", kws) && inSlice("time", kws) { + if InKeywordSlice("tar_time", kws) && InKeywordSlice("time", kws) { // Delete "time". timeStateT := diffs["time"] delete(diffs, "time") @@ -312,7 +312,7 @@ func compareEntry(oldEntry, newEntry Entry) ([]KeyDelta, error) { // // NB: The order of the parameters matters (old, new) because Extra and // Missing are considered as different discrepancy types. -func Compare(oldDh, newDh *DirectoryHierarchy, keys []string) ([]InodeDelta, error) { +func Compare(oldDh, newDh *DirectoryHierarchy, keys []Keyword) ([]InodeDelta, error) { // Represents the new and old states for an entry. type stateT struct { Old *Entry @@ -320,7 +320,7 @@ func Compare(oldDh, newDh *DirectoryHierarchy, keys []string) ([]InodeDelta, err } // Make dealing with the keys mapping easier. - keySet := map[string]struct{}{} + keySet := map[Keyword]struct{}{} for _, key := range keys { keySet[key] = struct{}{} } diff --git a/compare_test.go b/compare_test.go index a1e9cde..23a62f9 100644 --- a/compare_test.go +++ b/compare_test.go @@ -325,7 +325,7 @@ func TestCompareKeys(t *testing.T) { } // Compare. - diffs, err := Compare(old, new, []string{"size"}) + diffs, err := Compare(old, new, []Keyword{"size"}) if err != nil { t.Fatal(err) } diff --git a/entry.go b/entry.go index 055e9a0..41d5206 100644 --- a/entry.go +++ b/entry.go @@ -21,7 +21,7 @@ type Entry struct { Pos int // order in the spec Raw string // file or directory name Name string // file or directory name - Keywords []string // TODO(vbatts) maybe a keyword typed set of values? + Keywords []KeyVal // TODO(vbatts) maybe a keyword typed set of values? Type EntryType } @@ -94,23 +94,20 @@ func (e Entry) String() string { if e.Type == DotDotType { return e.Name } - if e.Type == SpecialType || e.Type == FullType || inSlice("type=dir", e.Keywords) { - return fmt.Sprintf("%s %s", e.Name, strings.Join(e.Keywords, " ")) + if e.Type == SpecialType || e.Type == FullType || inKeyValSlice("type=dir", e.Keywords) { + return fmt.Sprintf("%s %s", e.Name, strings.Join(KeyValToString(e.Keywords), " ")) } - return fmt.Sprintf(" %s %s", e.Name, strings.Join(e.Keywords, " ")) + return fmt.Sprintf(" %s %s", e.Name, strings.Join(KeyValToString(e.Keywords), " ")) } -// AllKeys returns the full set of KeyVals for the given entry, based on the +// AllKeys returns the full set of KeyVal for the given entry, based on the // /set keys as well as the entry-local keys. Entry-local keys always take // precedence. -func (e Entry) AllKeys() KeyVals { - var kv KeyVals +func (e Entry) AllKeys() []KeyVal { if e.Set != nil { - kv = MergeSet(e.Set.Keywords, e.Keywords) - } else { - kv = NewKeyVals(e.Keywords) + return MergeKeyValSet(e.Set.Keywords, e.Keywords) } - return kv + return e.Keywords } // EntryType are the formats of lines in an mtree spec file diff --git a/hierarchy.go b/hierarchy.go index 3c8da39..3a970cd 100644 --- a/hierarchy.go +++ b/hierarchy.go @@ -28,8 +28,8 @@ func (dh DirectoryHierarchy) WriteTo(w io.Writer) (n int64, err error) { // UsedKeywords collects and returns all the keywords used in a // a DirectoryHierarchy -func (dh DirectoryHierarchy) UsedKeywords() []string { - usedkeywords := []string{} +func (dh DirectoryHierarchy) UsedKeywords() []Keyword { + usedkeywords := []Keyword{} for _, e := range dh.Entries { switch e.Type { case FullType, RelativeType, SpecialType: @@ -37,8 +37,8 @@ func (dh DirectoryHierarchy) UsedKeywords() []string { kvs := e.Keywords for _, kv := range kvs { kw := KeyVal(kv).Keyword() - if !inSlice(kw, usedkeywords) { - usedkeywords = append(usedkeywords, KeywordSynonym(kw)) + if !InKeywordSlice(kw, usedkeywords) { + usedkeywords = append(usedkeywords, KeywordSynonym(string(kw))) } } } diff --git a/hierarchy_test.go b/hierarchy_test.go index 0a58e56..7dee81a 100644 --- a/hierarchy_test.go +++ b/hierarchy_test.go @@ -7,7 +7,7 @@ import ( var checklist = []struct { blob string - set []string + set []Keyword }{ {blob: ` # machine: bananaboat @@ -19,7 +19,7 @@ var checklist = []struct { . size=4096 type=dir mode=0755 nlink=8 time=1479326055.423853146 .COMMIT_EDITMSG.un~ size=1006 mode=0644 time=1479325423.450468662 sha1digest=dead0face .TAG_EDITMSG.un~ size=1069 mode=0600 time=1471362316.801317529 sha256digest=dead0face -`, set: []string{"size", "mode", "time", "sha256digest"}}, +`, set: []Keyword{"size", "mode", "time", "sha256digest"}}, } func TestUsedKeywords(t *testing.T) { @@ -30,7 +30,7 @@ func TestUsedKeywords(t *testing.T) { } used := dh.UsedKeywords() for _, k := range item.set { - if !inSlice(k, used) { + if !InKeywordSlice(k, used) { t.Errorf("%d: expected to find %q in %q", i, k, used) } } diff --git a/keywordfunc.go b/keywordfunc.go new file mode 100644 index 0000000..d545c11 --- /dev/null +++ b/keywordfunc.go @@ -0,0 +1,168 @@ +package mtree + +import ( + "archive/tar" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "fmt" + "hash" + "io" + "os" + + "golang.org/x/crypto/ripemd160" +) + +// KeywordFunc is the type of a function called on each file to be included in +// a DirectoryHierarchy, that will produce the string output of the keyword to +// be included for the file entry. Otherwise, empty string. +// io.Reader `r` is to the file stream for the file payload. While this +// function takes an io.Reader, the caller needs to reset it to the beginning +// for each new KeywordFunc +type KeywordFunc func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) + +var ( + // KeywordFuncs is the map of all keywords (and the functions to produce them) + KeywordFuncs = map[Keyword]KeywordFunc{ + "size": sizeKeywordFunc, // The size, in bytes, of the file + "type": typeKeywordFunc, // The type of the file + "time": timeKeywordFunc, // The last modification time of the file + "link": linkKeywordFunc, // The target of the symbolic link when type=link + "uid": uidKeywordFunc, // The file owner as a numeric value + "gid": gidKeywordFunc, // The file group as a numeric value + "nlink": nlinkKeywordFunc, // The number of hard links the file is expected to have + "uname": unameKeywordFunc, // The file owner as a symbolic name + "mode": modeKeywordFunc, // The current file's permissions as a numeric (octal) or symbolic value + "cksum": cksumKeywordFunc, // The checksum of the file using the default algorithm specified by the cksum(1) utility + "md5": hasherKeywordFunc("md5digest", md5.New), // The MD5 message digest of the file + "md5digest": hasherKeywordFunc("md5digest", md5.New), // A synonym for `md5` + "rmd160": hasherKeywordFunc("ripemd160digest", ripemd160.New), // The RIPEMD160 message digest of the file + "rmd160digest": hasherKeywordFunc("ripemd160digest", ripemd160.New), // A synonym for `rmd160` + "ripemd160digest": hasherKeywordFunc("ripemd160digest", ripemd160.New), // A synonym for `rmd160` + "sha1": hasherKeywordFunc("sha1digest", sha1.New), // The SHA1 message digest of the file + "sha1digest": hasherKeywordFunc("sha1digest", sha1.New), // A synonym for `sha1` + "sha256": hasherKeywordFunc("sha256digest", sha256.New), // The SHA256 message digest of the file + "sha256digest": hasherKeywordFunc("sha256digest", sha256.New), // A synonym for `sha256` + "sha384": hasherKeywordFunc("sha384digest", sha512.New384), // The SHA384 message digest of the file + "sha384digest": hasherKeywordFunc("sha384digest", sha512.New384), // A synonym for `sha384` + "sha512": hasherKeywordFunc("sha512digest", sha512.New), // The SHA512 message digest of the file + "sha512digest": hasherKeywordFunc("sha512digest", sha512.New), // A synonym for `sha512` + + "flags": flagsKeywordFunc, // NOTE: this is a noop, but here to support the presence of the "flags" keyword. + + // This is not an upstreamed keyword, but used to vary from "time", as tar + // archives do not store nanosecond precision. So comparing on "time" will + // be only seconds level accurate. + "tar_time": tartimeKeywordFunc, // The last modification time of the file, from a tar archive mtime + + // This is not an upstreamed keyword, but a needed attribute for file validation. + // The pattern for this keyword key is prefixed by "xattr." followed by the extended attribute "namespace.key". + // The keyword value is the SHA1 digest of the extended attribute's value. + // In this way, the order of the keys does not matter, and the contents of the value is not revealed. + "xattr": xattrKeywordFunc, + "xattrs": xattrKeywordFunc, + } +) +var ( + modeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + permissions := info.Mode().Perm() + if os.ModeSetuid&info.Mode() > 0 { + permissions |= (1 << 11) + } + if os.ModeSetgid&info.Mode() > 0 { + permissions |= (1 << 10) + } + if os.ModeSticky&info.Mode() > 0 { + permissions |= (1 << 9) + } + return KeyVal(fmt.Sprintf("mode=%#o", permissions)), nil + } + sizeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + if sys, ok := info.Sys().(*tar.Header); ok { + if sys.Typeflag == tar.TypeSymlink { + return KeyVal(fmt.Sprintf("size=%d", len(sys.Linkname))), nil + } + } + return KeyVal(fmt.Sprintf("size=%d", info.Size())), nil + } + cksumKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + if !info.Mode().IsRegular() { + return emptyKV, nil + } + sum, _, err := cksum(r) + if err != nil { + return emptyKV, err + } + return KeyVal(fmt.Sprintf("cksum=%d", sum)), nil + } + hasherKeywordFunc = func(name string, newHash func() hash.Hash) KeywordFunc { + return func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + if !info.Mode().IsRegular() { + return emptyKV, nil + } + h := newHash() + if _, err := io.Copy(h, r); err != nil { + return emptyKV, err + } + return KeyVal(fmt.Sprintf("%s=%x", KeywordSynonym(name), h.Sum(nil))), nil + } + } + tartimeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + return KeyVal(fmt.Sprintf("tar_time=%d.%9.9d", info.ModTime().Unix(), 0)), nil + } + timeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + tSec := info.ModTime().Unix() + tNano := info.ModTime().Nanosecond() + return KeyVal(fmt.Sprintf("time=%d.%9.9d", tSec, tNano)), nil + } + linkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + if sys, ok := info.Sys().(*tar.Header); ok { + if sys.Linkname != "" { + linkname, err := Vis(sys.Linkname) + if err != nil { + return emptyKV, err + } + return KeyVal(fmt.Sprintf("link=%s", linkname)), nil + } + return emptyKV, nil + } + + if info.Mode()&os.ModeSymlink != 0 { + str, err := os.Readlink(path) + if err != nil { + return emptyKV, err + } + linkname, err := Vis(str) + if err != nil { + return emptyKV, err + } + return KeyVal(fmt.Sprintf("link=%s", linkname)), nil + } + return emptyKV, nil + } + typeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + if info.Mode().IsDir() { + return "type=dir", nil + } + if info.Mode().IsRegular() { + return "type=file", nil + } + if info.Mode()&os.ModeSocket != 0 { + return "type=socket", nil + } + if info.Mode()&os.ModeSymlink != 0 { + return "type=link", nil + } + if info.Mode()&os.ModeNamedPipe != 0 { + return "type=fifo", nil + } + if info.Mode()&os.ModeDevice != 0 { + if info.Mode()&os.ModeCharDevice != 0 { + return "type=char", nil + } + return "type=device", nil + } + return emptyKV, nil + } +) diff --git a/keywords.go b/keywords.go index 93916f4..b139d6d 100644 --- a/keywords.go +++ b/keywords.go @@ -1,55 +1,97 @@ package mtree import ( - "archive/tar" - "crypto/md5" - "crypto/sha1" - "crypto/sha256" - "crypto/sha512" "fmt" - "hash" - "io" - "os" "strings" - - "golang.org/x/crypto/ripemd160" ) -// KeywordFunc is the type of a function called on each file to be included in -// a DirectoryHierarchy, that will produce the string output of the keyword to -// be included for the file entry. Otherwise, empty string. -// io.Reader `r` is to the file stream for the file payload. While this -// function takes an io.Reader, the caller needs to reset it to the beginning -// for each new KeywordFunc -type KeywordFunc func(path string, info os.FileInfo, r io.Reader) (string, error) - // Keyword is the string name of a keyword, with some convenience functions for // determining whether it is a default or bsd standard keyword. type Keyword string // Default returns whether this keyword is in the default set of keywords func (k Keyword) Default() bool { - return inSlice(string(k), DefaultKeywords) + return InKeywordSlice(k, DefaultKeywords) } // Bsd returns whether this keyword is in the upstream FreeBSD mtree(8) func (k Keyword) Bsd() bool { - return inSlice(string(k), BsdKeywords) + return InKeywordSlice(k, BsdKeywords) +} + +// Synonym returns the canonical name for this keyword. This is provides the +// same functionality as KeywordSynonym() +func (k Keyword) Synonym() Keyword { + return KeywordSynonym(string(k)) +} + +// InKeywordSlice checks for the presence of `a` in `list` +func InKeywordSlice(a Keyword, list []Keyword) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} +func inKeyValSlice(a KeyVal, list []KeyVal) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} + +// ToKeywords makes a list of Keyword from a list of string +func ToKeywords(list []string) []Keyword { + ret := make([]Keyword, len(list)) + for i := range list { + ret[i] = Keyword(list[i]) + } + return ret +} + +// FromKeywords makes a list of string from a list of Keyword +func FromKeywords(list []Keyword) []string { + ret := make([]string, len(list)) + for i := range list { + ret[i] = string(list[i]) + } + return ret +} + +// KeyValToString constructs a list of string from the list of KeyVal +func KeyValToString(list []KeyVal) []string { + ret := make([]string, len(list)) + for i := range list { + ret[i] = string(list[i]) + } + return ret +} + +// StringToKeyVals constructs a list of KeyVal from the list of strings, like "keyword=value" +func StringToKeyVals(list []string) []KeyVal { + ret := make([]KeyVal, len(list)) + for i := range list { + ret[i] = KeyVal(list[i]) + } + return ret } // KeyVal is a "keyword=value" type KeyVal string // Keyword is the mapping to the available keywords -func (kv KeyVal) Keyword() string { +func (kv KeyVal) Keyword() Keyword { if !strings.Contains(string(kv), "=") { - return "" + return Keyword("") } chunks := strings.SplitN(strings.TrimSpace(string(kv)), "=", 2)[0] if !strings.Contains(chunks, ".") { - return chunks + return Keyword(chunks) } - return strings.SplitN(chunks, ".", 2)[0] + return Keyword(strings.SplitN(chunks, ".", 2)[0]) } // KeywordSuffix is really only used for xattr, as the keyword is a prefix to @@ -78,7 +120,7 @@ func (kv KeyVal) ChangeValue(newval string) string { return fmt.Sprintf("%s=%s", kv.Keyword(), newval) } -// KeyValEqual returns whether two KeyVals are equivalent. This takes +// KeyValEqual returns whether two KeyVal are equivalent. This takes // care of certain odd cases such as tar_mtime, and should be used over // using == comparisons directly unless you really know what you're // doing. @@ -87,35 +129,49 @@ func KeyValEqual(a, b KeyVal) bool { return a.Keyword() == b.Keyword() && a.Value() == b.Value() } -// keywordSelector takes an array of "keyword=value" and filters out that only the set of words -func keywordSelector(keyval, words []string) []string { - retList := []string{} +// keyvalSelector takes an array of KeyVal ("keyword=value") and filters out that only the set of keywords +func keyvalSelector(keyval []KeyVal, keyset []Keyword) []KeyVal { + retList := []KeyVal{} for _, kv := range keyval { - if inSlice(KeyVal(kv).Keyword(), words) { + if InKeywordSlice(kv.Keyword(), keyset) { retList = append(retList, kv) } } return retList } -// NewKeyVals constructs a list of KeyVal from the list of strings, like "keyword=value" -func NewKeyVals(keyvals []string) KeyVals { - kvs := make(KeyVals, len(keyvals)) - for i := range keyvals { - kvs[i] = KeyVal(keyvals[i]) +func keyValDifference(this, that []KeyVal) []KeyVal { + if len(this) == 0 { + return that } - return kvs + diff := []KeyVal{} + for _, kv := range this { + if !inKeyValSlice(kv, that) { + diff = append(diff, kv) + } + } + return diff +} +func keyValCopy(set []KeyVal) []KeyVal { + ret := make([]KeyVal, len(set)) + for i := range set { + ret[i] = set[i] + } + return ret } - -// KeyVals is a list of KeyVal -type KeyVals []KeyVal // Has the "keyword" present in the list of KeyVal, and returns the // corresponding KeyVal, else an empty string. -func (kvs KeyVals) Has(keyword string) KeyVal { - for i := range kvs { - if kvs[i].Keyword() == keyword { - return kvs[i] +func Has(keyvals []KeyVal, keyword string) KeyVal { + return HasKeyword(keyvals, Keyword(keyword)) +} + +// HasKeyword the "keyword" present in the list of KeyVal, and returns the +// corresponding KeyVal, else an empty string. +func HasKeyword(keyvals []KeyVal, keyword Keyword) KeyVal { + for i := range keyvals { + if keyvals[i].Keyword() == keyword { + return keyvals[i] } } return emptyKV @@ -125,20 +181,27 @@ var emptyKV = KeyVal("") // MergeSet takes the current setKeyVals, and then applies the entryKeyVals // such that the entry's values win. The union is returned. -func MergeSet(setKeyVals, entryKeyVals []string) KeyVals { - retList := NewKeyVals(append([]string{}, setKeyVals...)) - eKVs := NewKeyVals(entryKeyVals) - seenKeywords := []string{} +func MergeSet(setKeyVals, entryKeyVals []string) []KeyVal { + retList := StringToKeyVals(setKeyVals) + eKVs := StringToKeyVals(entryKeyVals) + return MergeKeyValSet(retList, eKVs) +} + +// MergeKeyValSet does a merge of the two sets of KeyVal, and the KeyVal of +// entryKeyVals win when there is a duplicate Keyword. +func MergeKeyValSet(setKeyVals, entryKeyVals []KeyVal) []KeyVal { + retList := keyValCopy(setKeyVals) + seenKeywords := []Keyword{} for i := range retList { word := retList[i].Keyword() - if ekv := eKVs.Has(word); ekv != emptyKV { + if ekv := HasKeyword(entryKeyVals, word); ekv != emptyKV { retList[i] = ekv } seenKeywords = append(seenKeywords, word) } - for i := range eKVs { - if !inSlice(eKVs[i].Keyword(), seenKeywords) { - retList = append(retList, eKVs[i]) + for i := range entryKeyVals { + if !InKeywordSlice(entryKeyVals[i].Keyword(), seenKeywords) { + retList = append(retList, entryKeyVals[i]) } } return retList @@ -147,7 +210,7 @@ func MergeSet(setKeyVals, entryKeyVals []string) KeyVals { var ( // DefaultKeywords has the several default keyword producers (uid, gid, // mode, nlink, type, size, mtime) - DefaultKeywords = []string{ + DefaultKeywords = []Keyword{ "size", "type", "uid", @@ -160,7 +223,7 @@ var ( // DefaultTarKeywords has keywords that should be used when creating a manifest from // an archive. Currently, evaluating the # of hardlinks has not been implemented yet - DefaultTarKeywords = []string{ + DefaultTarKeywords = []Keyword{ "size", "type", "uid", @@ -171,7 +234,7 @@ var ( } // BsdKeywords is the set of keywords that is only in the upstream FreeBSD mtree - BsdKeywords = []string{ + BsdKeywords = []Keyword{ "cksum", "device", "flags", // this one is really mostly BSD specific ... @@ -205,176 +268,36 @@ var ( } // SetKeywords is the default set of keywords calculated for a `/set` SpecialType - SetKeywords = []string{ + SetKeywords = []Keyword{ "uid", "gid", } - // KeywordFuncs is the map of all keywords (and the functions to produce them) - KeywordFuncs = map[string]KeywordFunc{ - "size": sizeKeywordFunc, // The size, in bytes, of the file - "type": typeKeywordFunc, // The type of the file - "time": timeKeywordFunc, // The last modification time of the file - "link": linkKeywordFunc, // The target of the symbolic link when type=link - "uid": uidKeywordFunc, // The file owner as a numeric value - "gid": gidKeywordFunc, // The file group as a numeric value - "nlink": nlinkKeywordFunc, // The number of hard links the file is expected to have - "uname": unameKeywordFunc, // The file owner as a symbolic name - "mode": modeKeywordFunc, // The current file's permissions as a numeric (octal) or symbolic value - "cksum": cksumKeywordFunc, // The checksum of the file using the default algorithm specified by the cksum(1) utility - "md5": hasherKeywordFunc("md5digest", md5.New), // The MD5 message digest of the file - "md5digest": hasherKeywordFunc("md5digest", md5.New), // A synonym for `md5` - "rmd160": hasherKeywordFunc("ripemd160digest", ripemd160.New), // The RIPEMD160 message digest of the file - "rmd160digest": hasherKeywordFunc("ripemd160digest", ripemd160.New), // A synonym for `rmd160` - "ripemd160digest": hasherKeywordFunc("ripemd160digest", ripemd160.New), // A synonym for `rmd160` - "sha1": hasherKeywordFunc("sha1digest", sha1.New), // The SHA1 message digest of the file - "sha1digest": hasherKeywordFunc("sha1digest", sha1.New), // A synonym for `sha1` - "sha256": hasherKeywordFunc("sha256digest", sha256.New), // The SHA256 message digest of the file - "sha256digest": hasherKeywordFunc("sha256digest", sha256.New), // A synonym for `sha256` - "sha384": hasherKeywordFunc("sha384digest", sha512.New384), // The SHA384 message digest of the file - "sha384digest": hasherKeywordFunc("sha384digest", sha512.New384), // A synonym for `sha384` - "sha512": hasherKeywordFunc("sha512digest", sha512.New), // The SHA512 message digest of the file - "sha512digest": hasherKeywordFunc("sha512digest", sha512.New), // A synonym for `sha512` - - "flags": flagsKeywordFunc, // NOTE: this is a noop, but here to support the presence of the "flags" keyword. - - // This is not an upstreamed keyword, but used to vary from "time", as tar - // archives do not store nanosecond precision. So comparing on "time" will - // be only seconds level accurate. - "tar_time": tartimeKeywordFunc, // The last modification time of the file, from a tar archive mtime - - // This is not an upstreamed keyword, but a needed attribute for file validation. - // The pattern for this keyword key is prefixed by "xattr." followed by the extended attribute "namespace.key". - // The keyword value is the SHA1 digest of the extended attribute's value. - // In this way, the order of the keys does not matter, and the contents of the value is not revealed. - "xattr": xattrKeywordFunc, - "xattrs": xattrKeywordFunc, - } ) // KeywordSynonym returns the canonical name for keywords that have synonyms, // and just returns the name provided if there is no synonym. In this way it // ought to be safe to wrap any keyword name. -func KeywordSynonym(name string) string { +func KeywordSynonym(name string) Keyword { + var retname string switch name { case "md5": - return "md5digest" + retname = "md5digest" case "rmd160": - return "ripemd160digest" + retname = "ripemd160digest" case "rmd160digest": - return "ripemd160digest" + retname = "ripemd160digest" case "sha1": - return "sha1digest" + retname = "sha1digest" case "sha256": - return "sha256digest" + retname = "sha256digest" case "sha384": - return "sha384digest" + retname = "sha384digest" case "sha512": - return "sha512digest" + retname = "sha512digest" case "xattrs": - return "xattr" + retname = "xattr" + default: + retname = name } - return name + return Keyword(retname) } - -var ( - modeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - permissions := info.Mode().Perm() - if os.ModeSetuid&info.Mode() > 0 { - permissions |= (1 << 11) - } - if os.ModeSetgid&info.Mode() > 0 { - permissions |= (1 << 10) - } - if os.ModeSticky&info.Mode() > 0 { - permissions |= (1 << 9) - } - return fmt.Sprintf("mode=%#o", permissions), nil - } - sizeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - if sys, ok := info.Sys().(*tar.Header); ok { - if sys.Typeflag == tar.TypeSymlink { - return fmt.Sprintf("size=%d", len(sys.Linkname)), nil - } - } - return fmt.Sprintf("size=%d", info.Size()), nil - } - cksumKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - if !info.Mode().IsRegular() { - return "", nil - } - sum, _, err := cksum(r) - if err != nil { - return "", err - } - return fmt.Sprintf("cksum=%d", sum), nil - } - hasherKeywordFunc = func(name string, newHash func() hash.Hash) KeywordFunc { - return func(path string, info os.FileInfo, r io.Reader) (string, error) { - if !info.Mode().IsRegular() { - return "", nil - } - h := newHash() - if _, err := io.Copy(h, r); err != nil { - return "", err - } - return fmt.Sprintf("%s=%x", KeywordSynonym(name), h.Sum(nil)), nil - } - } - tartimeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - return fmt.Sprintf("tar_time=%d.%9.9d", info.ModTime().Unix(), 0), nil - } - timeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - tSec := info.ModTime().Unix() - tNano := info.ModTime().Nanosecond() - return fmt.Sprintf("time=%d.%9.9d", tSec, tNano), nil - } - linkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - if sys, ok := info.Sys().(*tar.Header); ok { - if sys.Linkname != "" { - linkname, err := Vis(sys.Linkname) - if err != nil { - return "", err - } - return fmt.Sprintf("link=%s", linkname), nil - } - return "", nil - } - - if info.Mode()&os.ModeSymlink != 0 { - str, err := os.Readlink(path) - if err != nil { - return "", err - } - linkname, err := Vis(str) - if err != nil { - return "", err - } - return fmt.Sprintf("link=%s", linkname), nil - } - return "", nil - } - typeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - if info.Mode().IsDir() { - return "type=dir", nil - } - if info.Mode().IsRegular() { - return "type=file", nil - } - if info.Mode()&os.ModeSocket != 0 { - return "type=socket", nil - } - if info.Mode()&os.ModeSymlink != 0 { - return "type=link", nil - } - if info.Mode()&os.ModeNamedPipe != 0 { - return "type=fifo", nil - } - if info.Mode()&os.ModeDevice != 0 { - if info.Mode()&os.ModeCharDevice != 0 { - return "type=char", nil - } - return "type=device", nil - } - return "", nil - } -) diff --git a/keywords_bsd.go b/keywords_bsd.go index 3a82a9b..43a8073 100644 --- a/keywords_bsd.go +++ b/keywords_bsd.go @@ -12,46 +12,46 @@ import ( ) var ( - flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { // ideally this will pull in from here https://www.freebsd.org/cgi/man.cgi?query=chflags&sektion=2 - return "", nil + return emptyKV, nil } - unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if hdr, ok := info.Sys().(*tar.Header); ok { - return fmt.Sprintf("uname=%s", hdr.Uname), nil + return KeyVal(fmt.Sprintf("uname=%s", hdr.Uname)), nil } stat := info.Sys().(*syscall.Stat_t) u, err := user.LookupId(fmt.Sprintf("%d", stat.Uid)) if err != nil { - return "", err + return emptyKV, err } - return fmt.Sprintf("uname=%s", u.Username), nil + return KeyVal(fmt.Sprintf("uname=%s", u.Username)), nil } - uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if hdr, ok := info.Sys().(*tar.Header); ok { - return fmt.Sprintf("uid=%d", hdr.Uid), nil + return KeyVal(fmt.Sprintf("uid=%d", hdr.Uid)), nil } stat := info.Sys().(*syscall.Stat_t) - return fmt.Sprintf("uid=%d", stat.Uid), nil + return KeyVal(fmt.Sprintf("uid=%d", stat.Uid)), nil } - gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if hdr, ok := info.Sys().(*tar.Header); ok { - return fmt.Sprintf("gid=%d", hdr.Gid), nil + return KeyVal(fmt.Sprintf("gid=%d", hdr.Gid)), nil } if stat, ok := info.Sys().(*syscall.Stat_t); ok { - return fmt.Sprintf("gid=%d", stat.Gid), nil + return KeyVal(fmt.Sprintf("gid=%d", stat.Gid)), nil } - return "", nil + return emptyKV, nil } - nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if stat, ok := info.Sys().(*syscall.Stat_t); ok { - return fmt.Sprintf("nlink=%d", stat.Nlink), nil + return KeyVal(fmt.Sprintf("nlink=%d", stat.Nlink)), nil } - return "", nil + return emptyKV, nil } - xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - return "", nil + xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + return emptyKV, nil } ) diff --git a/keywords_linux.go b/keywords_linux.go index ed77015..79cf46e 100644 --- a/keywords_linux.go +++ b/keywords_linux.go @@ -17,71 +17,71 @@ import ( var ( // this is bsd specific https://www.freebsd.org/cgi/man.cgi?query=chflags&sektion=2 - flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - return "", nil + flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + return emptyKV, nil } - unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if hdr, ok := info.Sys().(*tar.Header); ok { - return fmt.Sprintf("uname=%s", hdr.Uname), nil + return KeyVal(fmt.Sprintf("uname=%s", hdr.Uname)), nil } stat := info.Sys().(*syscall.Stat_t) u, err := user.LookupId(fmt.Sprintf("%d", stat.Uid)) if err != nil { - return "", err + return emptyKV, err } - return fmt.Sprintf("uname=%s", u.Username), nil + return KeyVal(fmt.Sprintf("uname=%s", u.Username)), nil } - uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if hdr, ok := info.Sys().(*tar.Header); ok { - return fmt.Sprintf("uid=%d", hdr.Uid), nil + return KeyVal(fmt.Sprintf("uid=%d", hdr.Uid)), nil } stat := info.Sys().(*syscall.Stat_t) - return fmt.Sprintf("uid=%d", stat.Uid), nil + return KeyVal(fmt.Sprintf("uid=%d", stat.Uid)), nil } - gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if hdr, ok := info.Sys().(*tar.Header); ok { - return fmt.Sprintf("gid=%d", hdr.Gid), nil + return KeyVal(fmt.Sprintf("gid=%d", hdr.Gid)), nil } if stat, ok := info.Sys().(*syscall.Stat_t); ok { - return fmt.Sprintf("gid=%d", stat.Gid), nil + return KeyVal(fmt.Sprintf("gid=%d", stat.Gid)), nil } - return "", nil + return emptyKV, nil } - nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if stat, ok := info.Sys().(*syscall.Stat_t); ok { - return fmt.Sprintf("nlink=%d", stat.Nlink), nil + return KeyVal(fmt.Sprintf("nlink=%d", stat.Nlink)), nil } - return "", nil + return emptyKV, nil } - xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if hdr, ok := info.Sys().(*tar.Header); ok { if len(hdr.Xattrs) == 0 { - return "", nil + return emptyKV, nil } - klist := []string{} + klist := []KeyVal{} for k, v := range hdr.Xattrs { - klist = append(klist, fmt.Sprintf("xattr.%s=%s", k, base64.StdEncoding.EncodeToString([]byte(v)))) + klist = append(klist, KeyVal(fmt.Sprintf("xattr.%s=%s", k, base64.StdEncoding.EncodeToString([]byte(v))))) } - return strings.Join(klist, " "), nil + return KeyVal(strings.Join(KeyValToString(klist), " ")), nil } if !info.Mode().IsRegular() && !info.Mode().IsDir() { - return "", nil + return emptyKV, nil } xlist, err := xattr.List(path) if err != nil { - return "", err + return emptyKV, err } - klist := make([]string, len(xlist)) + klist := make([]KeyVal, len(xlist)) for i := range xlist { data, err := xattr.Get(path, xlist[i]) if err != nil { - return "", err + return emptyKV, err } - klist[i] = fmt.Sprintf("xattr.%s=%s", xlist[i], base64.StdEncoding.EncodeToString(data)) + klist[i] = KeyVal(fmt.Sprintf("xattr.%s=%s", xlist[i], base64.StdEncoding.EncodeToString(data))) } - return strings.Join(klist, " "), nil + return KeyVal(strings.Join(KeyValToString(klist), " ")), nil } ) diff --git a/keywords_test.go b/keywords_test.go index 4411453..b8add8c 100644 --- a/keywords_test.go +++ b/keywords_test.go @@ -54,7 +54,7 @@ func TestKeywordsTimeNano(t *testing.T) { {857125628319, 0}, } { mtime := time.Unix(test.sec, test.nsec) - expected := fmt.Sprintf("time=%d.%9.9d", test.sec, test.nsec) + expected := KeyVal(fmt.Sprintf("time=%d.%9.9d", test.sec, test.nsec)) got, err := timeKeywordFunc("", fakeFileInfo{ mtime: mtime, }, nil) @@ -81,7 +81,7 @@ func TestKeywordsTimeTar(t *testing.T) { {857125628319, 0}, } { mtime := time.Unix(test.sec, test.nsec) - expected := fmt.Sprintf("tar_time=%d.%9.9d", test.sec, 0) + expected := KeyVal(fmt.Sprintf("tar_time=%d.%9.9d", test.sec, 0)) got, err := tartimeKeywordFunc("", fakeFileInfo{ mtime: mtime, }, nil) @@ -96,7 +96,8 @@ func TestKeywordsTimeTar(t *testing.T) { func TestKeywordSynonym(t *testing.T) { checklist := []struct { - give, expect string + give string + expect Keyword }{ {give: "time", expect: "time"}, {give: "md5", expect: "md5digest"}, diff --git a/parse.go b/parse.go index 9e1f9bb..36a7163 100644 --- a/parse.go +++ b/parse.go @@ -48,7 +48,7 @@ func ParseSpec(r io.Reader) (*DirectoryHierarchy, error) { // parse the options f := strings.Fields(str) e.Name = f[0] - e.Keywords = f[1:] + e.Keywords = StringToKeyVals(f[1:]) if e.Name == "/set" { creator.curSet = &e } else if e.Name == "/unset" { @@ -80,7 +80,7 @@ func ParseSpec(r io.Reader) (*DirectoryHierarchy, error) { } else { e.Type = RelativeType } - e.Keywords = f[1:] + e.Keywords = StringToKeyVals(f[1:]) // TODO: gather keywords if using tar stream e.Parent = creator.curDir for i := range e.Keywords { diff --git a/tar.go b/tar.go index 8b8b034..adfaea5 100644 --- a/tar.go +++ b/tar.go @@ -17,11 +17,15 @@ type Streamer interface { Hierarchy() (*DirectoryHierarchy, error) } -var tarDefaultSetKeywords = []string{"type=file", "flags=none", "mode=0664"} +var tarDefaultSetKeywords = []KeyVal{ + "type=file", + "flags=none", + "mode=0664", +} // NewTarStreamer streams a tar archive and creates a file hierarchy based off // of the tar metadata headers -func NewTarStreamer(r io.Reader, keywords []string) Streamer { +func NewTarStreamer(r io.Reader, keywords []Keyword) Streamer { pR, pW := io.Pipe() ts := &tarStream{ pipeReader: pR, @@ -45,17 +49,17 @@ type tarStream struct { pipeWriter *io.PipeWriter teeReader io.Reader tarReader *tar.Reader - keywords []string + keywords []Keyword err error } func (ts *tarStream) readHeaders() { // remove "time" keyword - notimekws := []string{} + notimekws := []Keyword{} for _, kw := range ts.keywords { - if !inSlice(kw, notimekws) { + if !InKeywordSlice(kw, notimekws) { if kw == "time" { - if !inSlice("tar_time", ts.keywords) { + if !InKeywordSlice("tar_time", ts.keywords) { notimekws = append(notimekws, "tar_time") } } else { @@ -74,7 +78,7 @@ func (ts *tarStream) readHeaders() { Type: CommentType, }, Set: nil, - Keywords: []string{"type=dir"}, + Keywords: []KeyVal{"type=dir"}, } metadataEntries := signatureEntries("") for _, e := range metadataEntries { @@ -242,7 +246,7 @@ func populateTree(root, e *Entry, hdr *tar.Header) error { Name: encoded, Type: RelativeType, Parent: parent, - Keywords: []string{"type=dir"}, // temp data + Keywords: []KeyVal{"type=dir"}, // temp data Set: nil, // temp data } pathname, err := newEntry.Path() @@ -276,7 +280,7 @@ func populateTree(root, e *Entry, hdr *tar.Header) error { // root: the "head" of the sub-tree to flatten // creator: a dhCreator that helps with the '/set' keyword // keywords: keywords specified by the user that should be evaluated -func flatten(root *Entry, creator *dhCreator, keywords []string) { +func flatten(root *Entry, creator *dhCreator, keywords []Keyword) { if root == nil || creator == nil { return } @@ -292,18 +296,19 @@ func flatten(root *Entry, creator *dhCreator, keywords []string) { if root.Set != nil { // Check if we need a new set + consolidatedKeys := keyvalSelector(append(tarDefaultSetKeywords, root.Set.Keywords...), keywords) if creator.curSet == nil { creator.curSet = &Entry{ Type: SpecialType, Name: "/set", - Keywords: keywordSelector(append(tarDefaultSetKeywords, root.Set.Keywords...), keywords), + Keywords: consolidatedKeys, Pos: len(creator.DH.Entries), } creator.DH.Entries = append(creator.DH.Entries, *creator.curSet) } else { needNewSet := false for _, k := range root.Set.Keywords { - if !inSlice(k, creator.curSet.Keywords) { + if !inKeyValSlice(k, creator.curSet.Keywords) { needNewSet = true break } @@ -313,7 +318,7 @@ func flatten(root *Entry, creator *dhCreator, keywords []string) { Name: "/set", Type: SpecialType, Pos: len(creator.DH.Entries), - Keywords: keywordSelector(append(tarDefaultSetKeywords, root.Set.Keywords...), keywords), + Keywords: consolidatedKeys, } creator.DH.Entries = append(creator.DH.Entries, *creator.curSet) } @@ -331,7 +336,7 @@ func flatten(root *Entry, creator *dhCreator, keywords []string) { } root.Set = creator.curSet if creator.curSet != nil { - root.Keywords = setDifference(root.Keywords, creator.curSet.Keywords) + root.Keywords = keyValDifference(root.Keywords, creator.curSet.Keywords) } root.Pos = len(creator.DH.Entries) creator.DH.Entries = append(creator.DH.Entries, *root) @@ -376,11 +381,11 @@ func resolveHardlinks(root *Entry, hardlinks map[string][]string, countlinks boo } linkfile.Keywords = basefile.Keywords if countlinks { - linkfile.Keywords = append(linkfile.Keywords, fmt.Sprintf("nlink=%d", len(links)+1)) + linkfile.Keywords = append(linkfile.Keywords, KeyVal(fmt.Sprintf("nlink=%d", len(links)+1))) } } if countlinks { - basefile.Keywords = append(basefile.Keywords, fmt.Sprintf("nlink=%d", len(links)+1)) + basefile.Keywords = append(basefile.Keywords, KeyVal(fmt.Sprintf("nlink=%d", len(links)+1))) } } } @@ -410,19 +415,6 @@ func filter(root *Entry, p func(*Entry) bool) []Entry { return nil } -func setDifference(this, that []string) []string { - if len(this) == 0 { - return that - } - diff := []string{} - for _, kv := range this { - if !inSlice(kv, that) { - diff = append(diff, kv) - } - } - return diff -} - func (ts *tarStream) setErr(err error) { ts.err = err } @@ -444,7 +436,7 @@ func (ts *tarStream) Hierarchy() (*DirectoryHierarchy, error) { if ts.root == nil { return nil, fmt.Errorf("root Entry not found, nothing to flatten") } - resolveHardlinks(ts.root, ts.hardlinks, inSlice("nlink", ts.keywords)) + resolveHardlinks(ts.root, ts.hardlinks, InKeywordSlice(Keyword("nlink"), ts.keywords)) flatten(ts.root, &ts.creator, ts.keywords) return ts.creator.DH, nil } diff --git a/tar_test.go b/tar_test.go index f11763f..15df52b 100644 --- a/tar_test.go +++ b/tar_test.go @@ -128,7 +128,7 @@ func TestArchiveCreation(t *testing.T) { if err != nil { t.Fatal(err) } - str := NewTarStreamer(fh, []string{"sha1"}) + str := NewTarStreamer(fh, []Keyword{"sha1"}) if _, err := io.Copy(ioutil.Discard, str); err != nil && err != io.EOF { t.Fatal(err) @@ -145,7 +145,7 @@ func TestArchiveCreation(t *testing.T) { } // Test the tar manifest against the actual directory - res, err := Check("./testdata/collection", tdh, []string{"sha1"}) + res, err := Check("./testdata/collection", tdh, []Keyword{"sha1"}) if err != nil { t.Fatal(err) } @@ -158,7 +158,7 @@ func TestArchiveCreation(t *testing.T) { } // Test the tar manifest against itself - res, err = TarCheck(tdh, tdh, []string{"sha1"}) + res, err = TarCheck(tdh, tdh, []Keyword{"sha1"}) if err != nil { t.Fatal(err) } @@ -170,11 +170,11 @@ func TestArchiveCreation(t *testing.T) { } // Validate the directory manifest against the archive - dh, err := Walk("./testdata/collection", nil, []string{"sha1"}) + dh, err := Walk("./testdata/collection", nil, []Keyword{"sha1"}) if err != nil { t.Fatal(err) } - res, err = TarCheck(tdh, dh, []string{"sha1"}) + res, err = TarCheck(tdh, dh, []Keyword{"sha1"}) if err != nil { t.Fatal(err) } @@ -212,7 +212,7 @@ func TestTreeTraversal(t *testing.T) { t.Fatal(err) } - res, err := TarCheck(tdh, tdh, []string{"sha1"}) + res, err := TarCheck(tdh, tdh, []Keyword{"sha1"}) if err != nil { t.Fatal(err) } @@ -224,7 +224,7 @@ func TestTreeTraversal(t *testing.T) { } // top-level "." directory will contain contents of traversal.tar - res, err = Check("./testdata/.", tdh, []string{"sha1"}) + res, err = Check("./testdata/.", tdh, []Keyword{"sha1"}) if err != nil { t.Fatal(err) } @@ -262,7 +262,7 @@ func TestTreeTraversal(t *testing.T) { } // Implied top-level "." directory will contain the contents of singlefile.tar - res, err = Check("./testdata/.", tdh, []string{"sha1"}) + res, err = Check("./testdata/.", tdh, []Keyword{"sha1"}) if err != nil { t.Fatal(err) } diff --git a/walk.go b/walk.go index 8e0763f..13a4fc8 100644 --- a/walk.go +++ b/walk.go @@ -15,12 +15,12 @@ import ( // returns true, then the path is not included in the spec. type ExcludeFunc func(path string, info os.FileInfo) bool -var defaultSetKeywords = []string{"type=file", "nlink=1", "flags=none", "mode=0664"} +var defaultSetKeywords = []KeyVal{"type=file", "nlink=1", "flags=none", "mode=0664"} // Walk from root directory and assemble the DirectoryHierarchy. excludes // provided are used to skip paths. keywords are the set to collect from the // walked paths. The recommended default list is DefaultKeywords. -func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHierarchy, error) { +func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHierarchy, error) { creator := dhCreator{DH: &DirectoryHierarchy{}} // insert signature and metadata comments first (user, machine, tree, date) metadataEntries := signatureEntries(root) @@ -32,7 +32,7 @@ func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHie if err != nil { return err } - for _, ex := range exlcudes { + for _, ex := range excludes { if ex(path, info) { return nil } @@ -71,7 +71,7 @@ func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHie Name: "/set", Type: SpecialType, Pos: len(creator.DH.Entries), - Keywords: keywordSelector(defaultSetKeywords, keywords), + Keywords: keyvalSelector(defaultSetKeywords, keywords), } for _, keyword := range SetKeywords { err := func() error { @@ -103,7 +103,7 @@ func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHie creator.DH.Entries = append(creator.DH.Entries, e) } else if creator.curSet != nil { // check the attributes of the /set keywords and re-set if changed - klist := []string{} + klist := []KeyVal{} for _, keyword := range SetKeywords { err := func() error { var r io.Reader @@ -135,7 +135,7 @@ func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHie needNewSet := false for _, k := range klist { - if !inSlice(k, creator.curSet.Keywords) { + if !inKeyValSlice(k, creator.curSet.Keywords) { needNewSet = true } } @@ -144,7 +144,7 @@ func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHie Name: "/set", Type: SpecialType, Pos: len(creator.DH.Entries), - Keywords: keywordSelector(append(defaultSetKeywords, klist...), keywords), + Keywords: keyvalSelector(append(defaultSetKeywords, klist...), keywords), } creator.curSet = &e creator.DH.Entries = append(creator.DH.Entries, e) @@ -181,7 +181,7 @@ func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHie if err != nil { return err } - if str != "" && !inSlice(str, creator.curSet.Keywords) { + if str != "" && !inKeyValSlice(str, creator.curSet.Keywords) { e.Keywords = append(e.Keywords, str) } return nil @@ -209,15 +209,6 @@ func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHie return creator.DH, err } -func inSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} - // startWalk walks the file tree rooted at root, calling walkFn for each file or // directory in the tree, including root. All errors that arise visiting files // and directories are filtered by walkFn. The files are walked in lexical From 21723a3974bc47e3950708caf5cfdf4a65086efc Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 17 Nov 2016 21:29:55 -0500 Subject: [PATCH 5/5] *: fix comparison of missing keywords Adding another test validated from the FreeBSD workflow. Just because the keywords requested to be validated are not present in the manifest, it is not an error. Also, if the keywords from a new manifest are not present in a prior manifest, then only compare the common keywords. Fixes https://github.com/vbatts/go-mtree/issues/86 Signed-off-by: Vincent Batts --- check_test.go | 2 +- cmd/gomtree/main.go | 10 +++------- compare.go | 22 +++++++++++++--------- test/cli/0005.sh | 19 +++++++++++++++++++ 4 files changed, 36 insertions(+), 17 deletions(-) create mode 100644 test/cli/0005.sh diff --git a/check_test.go b/check_test.go index 89024dd..6627b53 100644 --- a/check_test.go +++ b/check_test.go @@ -69,7 +69,7 @@ func TestCheckKeywords(t *testing.T) { t.Fatal(err) } if len(res) != 1 { - t.Errorf("expected to get 1 delta on changed mtimes, but did not") + t.Fatal("expected to get 1 delta on changed mtimes, but did not") } if res[0].Type() != Modified { t.Errorf("expected to get modified delta on changed mtimes, but did not") diff --git a/cmd/gomtree/main.go b/cmd/gomtree/main.go index 8c940cb..0d3d488 100644 --- a/cmd/gomtree/main.go +++ b/cmd/gomtree/main.go @@ -184,10 +184,6 @@ func app() error { if (keyword == "time" && mtree.InKeywordSlice("tar_time", specKeywords)) || (keyword == "tar_time" && mtree.InKeywordSlice("time", specKeywords)) { continue } - - if !mtree.InKeywordSlice(keyword, specKeywords) { - return fmt.Errorf("cannot verify keywords not in mtree specification: %s\n", keyword) - } } } @@ -263,9 +259,9 @@ func app() error { if isTarSpec(specDh) || *flTar != "" { res = filterMissingKeywords(res) } - if len(res) > 0 { - return fmt.Errorf("unexpected missing keywords: %d", len(res)) - } + //if len(res) > 0 { + //return fmt.Errorf("unexpected missing keywords: %d", len(res)) + //} out := formatFunc(res) if _, err := os.Stdout.Write([]byte(out)); err != nil { diff --git a/compare.go b/compare.go index 7497b46..b2340b3 100644 --- a/compare.go +++ b/compare.go @@ -185,10 +185,16 @@ func compareEntry(oldEntry, newEntry Entry) ([]KeyDelta, error) { } diffs := map[Keyword]*stateT{} + oldKeys := oldEntry.AllKeys() + newKeys := newEntry.AllKeys() // Fill the map with the old keys first. - for _, kv := range oldEntry.AllKeys() { + for _, kv := range oldKeys { key := kv.Keyword() + // only add this diff if the new keys has this keyword + if key != "tar_time" && key != "time" && HasKeyword(newKeys, key) == emptyKV { + continue + } // Cannot take &kv because it's the iterator. copy := new(KeyVal) @@ -202,8 +208,12 @@ func compareEntry(oldEntry, newEntry Entry) ([]KeyDelta, error) { } // Then fill the new keys. - for _, kv := range newEntry.AllKeys() { + for _, kv := range newKeys { key := kv.Keyword() + // only add this diff if the old keys has this keyword + if key != "tar_time" && key != "time" && HasKeyword(oldKeys, key) == emptyKV { + continue + } // Cannot take &kv because it's the iterator. copy := new(KeyVal) @@ -319,12 +329,6 @@ func Compare(oldDh, newDh *DirectoryHierarchy, keys []Keyword) ([]InodeDelta, er New *Entry } - // Make dealing with the keys mapping easier. - keySet := map[Keyword]struct{}{} - for _, key := range keys { - keySet[key] = struct{}{} - } - // To deal with different orderings of the entries, use a path-keyed // map to make sure we don't start comparing unrelated entries. diffs := map[string]*stateT{} @@ -405,7 +409,7 @@ func Compare(oldDh, newDh *DirectoryHierarchy, keys []Keyword) ([]InodeDelta, er if keys != nil { var filterChanged []KeyDelta for _, keyDiff := range changed { - if _, ok := keySet[keyDiff.name]; ok { + if InKeywordSlice(keyDiff.name, keys) { filterChanged = append(filterChanged, keyDiff) } } diff --git a/test/cli/0005.sh b/test/cli/0005.sh new file mode 100644 index 0000000..48e025b --- /dev/null +++ b/test/cli/0005.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +name=$(basename $0) +root=$1 +gomtree=$(readlink -f ${root}/gomtree) +t=$(mktemp -d /tmp/go-mtree.XXXXXX) + +echo "[${name}] Running in ${t}" + +pushd ${root} +mkdir -p ${t}/extract +git archive --format=tar HEAD^{tree} . | tar -C ${t}/extract/ -x + +${gomtree} -k sha1digest -c -p ${t}/extract/ > ${t}/${name}.mtree +${gomtree} -f ${t}/${name}.mtree -k md5digest -p ${t}/extract/ + +popd +rm -rf ${t}