From 4eec68be4b2611b32378fcc8cf27b1ace774c0f2 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 17 Nov 2016 19:47:31 -0500 Subject: [PATCH] *: make Keyword and KeyVal pervasive Signed-off-by: Vincent Batts --- Makefile | 4 +- check.go | 14 +- check_test.go | 2 +- cmd/gomtree/main.go | 33 ++--- compare.go | 16 +-- compare_test.go | 2 +- entry.go | 19 ++- hierarchy.go | 8 +- hierarchy_test.go | 6 +- keywordfunc.go | 168 ++++++++++++++++++++++ keywords.go | 337 +++++++++++++++++--------------------------- keywords_bsd.go | 36 ++--- keywords_linux.go | 54 +++---- keywords_test.go | 7 +- parse.go | 4 +- tar.go | 50 +++---- tar_test.go | 16 +-- walk.go | 25 ++-- 18 files changed, 434 insertions(+), 367 deletions(-) create mode 100644 keywordfunc.go diff --git a/Makefile b/Makefile index ed20a67..a428984 100644 --- a/Makefile +++ b/Makefile @@ -3,10 +3,10 @@ BUILD := gomtree CWD := $(shell pwd) SOURCE_FILES := $(shell find . -type f -name "*.go") -default: validation build +default: build validation .PHONY: validation -validation: test lint vet .cli.test +validation: .test .lint .vet .cli.test .PHONY: test test: .test diff --git a/check.go b/check.go index 96284d1..2fff36f 100644 --- a/check.go +++ b/check.go @@ -6,16 +6,20 @@ package mtree // // This is equivalent to creating a new DirectoryHierarchy with Walk(root, nil, // keywords) and then doing a Compare(dh, newDh, keywords). -func Check(root string, dh *DirectoryHierarchy, keywords []string) ([]InodeDelta, error) { +func Check(root string, dh *DirectoryHierarchy, keywords []Keyword) ([]InodeDelta, error) { if keywords == nil { - keywords = dh.UsedKeywords() + used := dh.UsedKeywords() + newDh, err := Walk(root, nil, used) + if err != nil { + return nil, err + } + return Compare(dh, newDh, used) } newDh, err := Walk(root, nil, keywords) if err != nil { return nil, err } - // TODO: Handle tar_time, if necessary. return Compare(dh, newDh, keywords) } @@ -23,9 +27,9 @@ func Check(root string, dh *DirectoryHierarchy, keywords []string) ([]InodeDelta // TarCheck is the tar equivalent of checking a file hierarchy spec against a // tar stream to determine if files have been changed. This is precisely // equivalent to Compare(dh, tarDH, keywords). -func TarCheck(tarDH, dh *DirectoryHierarchy, keywords []string) ([]InodeDelta, error) { +func TarCheck(tarDH, dh *DirectoryHierarchy, keywords []Keyword) ([]InodeDelta, error) { if keywords == nil { - keywords = dh.UsedKeywords() + return Compare(dh, tarDH, dh.UsedKeywords()) } return Compare(dh, tarDH, keywords) } diff --git a/check_test.go b/check_test.go index 5cdf756..89024dd 100644 --- a/check_test.go +++ b/check_test.go @@ -76,7 +76,7 @@ func TestCheckKeywords(t *testing.T) { } // Check again, but only sha1 and mode. This ought to pass. - res, err = Check(dir, dh, []string{"sha1", "mode"}) + res, err = Check(dir, dh, []Keyword{"sha1", "mode"}) if err != nil { t.Fatal(err) } diff --git a/cmd/gomtree/main.go b/cmd/gomtree/main.go index 92c6f6a..8c940cb 100644 --- a/cmd/gomtree/main.go +++ b/cmd/gomtree/main.go @@ -73,15 +73,15 @@ func app() error { var ( err error - tmpKeywords []string - currentKeywords []string + tmpKeywords []mtree.Keyword + currentKeywords []mtree.Keyword ) // -k if *flUseKeywords != "" { tmpKeywords = splitKeywordsArg(*flUseKeywords) - if !inSlice("type", tmpKeywords) { - tmpKeywords = append([]string{"type"}, tmpKeywords...) + if !mtree.InKeywordSlice("type", tmpKeywords) { + tmpKeywords = append([]mtree.Keyword{"type"}, tmpKeywords...) } } else { if *flTar != "" { @@ -94,7 +94,7 @@ func app() error { // -K if *flAddKeywords != "" { for _, kw := range splitKeywordsArg(*flAddKeywords) { - if !inSlice(kw, tmpKeywords) { + if !mtree.InKeywordSlice(kw, tmpKeywords) { tmpKeywords = append(tmpKeywords, kw) } } @@ -115,7 +115,7 @@ func app() error { // Check mutual exclusivity of keywords. // TODO(cyphar): Abstract this inside keywords.go. - if inSlice("tar_time", currentKeywords) && inSlice("time", currentKeywords) { + if mtree.InKeywordSlice("tar_time", currentKeywords) && mtree.InKeywordSlice("time", currentKeywords) { return fmt.Errorf("tar_time and time are mutually exclusive keywords") } @@ -124,7 +124,7 @@ func app() error { var ( specDh *mtree.DirectoryHierarchy stateDh *mtree.DirectoryHierarchy - specKeywords []string + specKeywords []mtree.Keyword ) // -f @@ -153,7 +153,7 @@ func app() error { if *flResultFormat == "json" { // if they're asking for json, give it to them - data := map[string][]string{*flFile: specKeywords} + data := map[string][]mtree.Keyword{*flFile: specKeywords} buf, err := json.MarshalIndent(data, "", " ") if err != nil { return err @@ -181,11 +181,11 @@ func app() error { for _, keyword := range currentKeywords { // As always, time is a special case. // TODO: Fix that. - if (keyword == "time" && inSlice("tar_time", specKeywords)) || (keyword == "tar_time" && inSlice("time", specKeywords)) { + if (keyword == "time" && mtree.InKeywordSlice("tar_time", specKeywords)) || (keyword == "tar_time" && mtree.InKeywordSlice("time", specKeywords)) { continue } - if !inSlice(keyword, specKeywords) { + if !mtree.InKeywordSlice(keyword, specKeywords) { return fmt.Errorf("cannot verify keywords not in mtree specification: %s\n", keyword) } } @@ -395,19 +395,10 @@ func isTarSpec(spec *mtree.DirectoryHierarchy) bool { return false } -func splitKeywordsArg(str string) []string { - keywords := []string{} +func splitKeywordsArg(str string) []mtree.Keyword { + keywords := []mtree.Keyword{} for _, kw := range strings.Fields(strings.Replace(str, ",", " ", -1)) { keywords = append(keywords, mtree.KeywordSynonym(kw)) } return keywords } - -func inSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} diff --git a/compare.go b/compare.go index 1f7b9d2..7497b46 100644 --- a/compare.go +++ b/compare.go @@ -123,7 +123,7 @@ func (i InodeDelta) String() string { // returned with InodeDelta.Diff(). type KeyDelta struct { diff DifferenceType - name string + name Keyword old string new string } @@ -136,7 +136,7 @@ func (k KeyDelta) Type() DifferenceType { // Name returns the name (the key) of the KeyDeltaVal entry in the // DirectoryHierarchy. -func (k KeyDelta) Name() string { +func (k KeyDelta) Name() Keyword { return k.name } @@ -164,7 +164,7 @@ func (k KeyDelta) New() *string { func (k KeyDelta) MarshalJSON() ([]byte, error) { return json.Marshal(struct { Type DifferenceType `json:"type"` - Name string `json:"name"` + Name Keyword `json:"name"` Old string `json:"old"` New string `json:"new"` }{ @@ -184,7 +184,7 @@ func compareEntry(oldEntry, newEntry Entry) ([]KeyDelta, error) { New *KeyVal } - diffs := map[string]*stateT{} + diffs := map[Keyword]*stateT{} // Fill the map with the old keys first. for _, kv := range oldEntry.AllKeys() { @@ -218,7 +218,7 @@ func compareEntry(oldEntry, newEntry Entry) ([]KeyDelta, error) { // We need a full list of the keys so we can deal with different keyvalue // orderings. - var kws []string + var kws []Keyword for kw := range diffs { kws = append(kws, kw) } @@ -226,7 +226,7 @@ func compareEntry(oldEntry, newEntry Entry) ([]KeyDelta, error) { // If both tar_time and time were specified in the set of keys, we have to // mess with the diffs. This is an unfortunate side-effect of tar archives. // TODO(cyphar): This really should be abstracted inside keywords.go - if inSlice("tar_time", kws) && inSlice("time", kws) { + if InKeywordSlice("tar_time", kws) && InKeywordSlice("time", kws) { // Delete "time". timeStateT := diffs["time"] delete(diffs, "time") @@ -312,7 +312,7 @@ func compareEntry(oldEntry, newEntry Entry) ([]KeyDelta, error) { // // NB: The order of the parameters matters (old, new) because Extra and // Missing are considered as different discrepancy types. -func Compare(oldDh, newDh *DirectoryHierarchy, keys []string) ([]InodeDelta, error) { +func Compare(oldDh, newDh *DirectoryHierarchy, keys []Keyword) ([]InodeDelta, error) { // Represents the new and old states for an entry. type stateT struct { Old *Entry @@ -320,7 +320,7 @@ func Compare(oldDh, newDh *DirectoryHierarchy, keys []string) ([]InodeDelta, err } // Make dealing with the keys mapping easier. - keySet := map[string]struct{}{} + keySet := map[Keyword]struct{}{} for _, key := range keys { keySet[key] = struct{}{} } diff --git a/compare_test.go b/compare_test.go index a1e9cde..23a62f9 100644 --- a/compare_test.go +++ b/compare_test.go @@ -325,7 +325,7 @@ func TestCompareKeys(t *testing.T) { } // Compare. - diffs, err := Compare(old, new, []string{"size"}) + diffs, err := Compare(old, new, []Keyword{"size"}) if err != nil { t.Fatal(err) } diff --git a/entry.go b/entry.go index 055e9a0..41d5206 100644 --- a/entry.go +++ b/entry.go @@ -21,7 +21,7 @@ type Entry struct { Pos int // order in the spec Raw string // file or directory name Name string // file or directory name - Keywords []string // TODO(vbatts) maybe a keyword typed set of values? + Keywords []KeyVal // TODO(vbatts) maybe a keyword typed set of values? Type EntryType } @@ -94,23 +94,20 @@ func (e Entry) String() string { if e.Type == DotDotType { return e.Name } - if e.Type == SpecialType || e.Type == FullType || inSlice("type=dir", e.Keywords) { - return fmt.Sprintf("%s %s", e.Name, strings.Join(e.Keywords, " ")) + if e.Type == SpecialType || e.Type == FullType || inKeyValSlice("type=dir", e.Keywords) { + return fmt.Sprintf("%s %s", e.Name, strings.Join(KeyValToString(e.Keywords), " ")) } - return fmt.Sprintf(" %s %s", e.Name, strings.Join(e.Keywords, " ")) + return fmt.Sprintf(" %s %s", e.Name, strings.Join(KeyValToString(e.Keywords), " ")) } -// AllKeys returns the full set of KeyVals for the given entry, based on the +// AllKeys returns the full set of KeyVal for the given entry, based on the // /set keys as well as the entry-local keys. Entry-local keys always take // precedence. -func (e Entry) AllKeys() KeyVals { - var kv KeyVals +func (e Entry) AllKeys() []KeyVal { if e.Set != nil { - kv = MergeSet(e.Set.Keywords, e.Keywords) - } else { - kv = NewKeyVals(e.Keywords) + return MergeKeyValSet(e.Set.Keywords, e.Keywords) } - return kv + return e.Keywords } // EntryType are the formats of lines in an mtree spec file diff --git a/hierarchy.go b/hierarchy.go index 3c8da39..3a970cd 100644 --- a/hierarchy.go +++ b/hierarchy.go @@ -28,8 +28,8 @@ func (dh DirectoryHierarchy) WriteTo(w io.Writer) (n int64, err error) { // UsedKeywords collects and returns all the keywords used in a // a DirectoryHierarchy -func (dh DirectoryHierarchy) UsedKeywords() []string { - usedkeywords := []string{} +func (dh DirectoryHierarchy) UsedKeywords() []Keyword { + usedkeywords := []Keyword{} for _, e := range dh.Entries { switch e.Type { case FullType, RelativeType, SpecialType: @@ -37,8 +37,8 @@ func (dh DirectoryHierarchy) UsedKeywords() []string { kvs := e.Keywords for _, kv := range kvs { kw := KeyVal(kv).Keyword() - if !inSlice(kw, usedkeywords) { - usedkeywords = append(usedkeywords, KeywordSynonym(kw)) + if !InKeywordSlice(kw, usedkeywords) { + usedkeywords = append(usedkeywords, KeywordSynonym(string(kw))) } } } diff --git a/hierarchy_test.go b/hierarchy_test.go index 0a58e56..7dee81a 100644 --- a/hierarchy_test.go +++ b/hierarchy_test.go @@ -7,7 +7,7 @@ import ( var checklist = []struct { blob string - set []string + set []Keyword }{ {blob: ` # machine: bananaboat @@ -19,7 +19,7 @@ var checklist = []struct { . size=4096 type=dir mode=0755 nlink=8 time=1479326055.423853146 .COMMIT_EDITMSG.un~ size=1006 mode=0644 time=1479325423.450468662 sha1digest=dead0face .TAG_EDITMSG.un~ size=1069 mode=0600 time=1471362316.801317529 sha256digest=dead0face -`, set: []string{"size", "mode", "time", "sha256digest"}}, +`, set: []Keyword{"size", "mode", "time", "sha256digest"}}, } func TestUsedKeywords(t *testing.T) { @@ -30,7 +30,7 @@ func TestUsedKeywords(t *testing.T) { } used := dh.UsedKeywords() for _, k := range item.set { - if !inSlice(k, used) { + if !InKeywordSlice(k, used) { t.Errorf("%d: expected to find %q in %q", i, k, used) } } diff --git a/keywordfunc.go b/keywordfunc.go new file mode 100644 index 0000000..d545c11 --- /dev/null +++ b/keywordfunc.go @@ -0,0 +1,168 @@ +package mtree + +import ( + "archive/tar" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "fmt" + "hash" + "io" + "os" + + "golang.org/x/crypto/ripemd160" +) + +// KeywordFunc is the type of a function called on each file to be included in +// a DirectoryHierarchy, that will produce the string output of the keyword to +// be included for the file entry. Otherwise, empty string. +// io.Reader `r` is to the file stream for the file payload. While this +// function takes an io.Reader, the caller needs to reset it to the beginning +// for each new KeywordFunc +type KeywordFunc func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) + +var ( + // KeywordFuncs is the map of all keywords (and the functions to produce them) + KeywordFuncs = map[Keyword]KeywordFunc{ + "size": sizeKeywordFunc, // The size, in bytes, of the file + "type": typeKeywordFunc, // The type of the file + "time": timeKeywordFunc, // The last modification time of the file + "link": linkKeywordFunc, // The target of the symbolic link when type=link + "uid": uidKeywordFunc, // The file owner as a numeric value + "gid": gidKeywordFunc, // The file group as a numeric value + "nlink": nlinkKeywordFunc, // The number of hard links the file is expected to have + "uname": unameKeywordFunc, // The file owner as a symbolic name + "mode": modeKeywordFunc, // The current file's permissions as a numeric (octal) or symbolic value + "cksum": cksumKeywordFunc, // The checksum of the file using the default algorithm specified by the cksum(1) utility + "md5": hasherKeywordFunc("md5digest", md5.New), // The MD5 message digest of the file + "md5digest": hasherKeywordFunc("md5digest", md5.New), // A synonym for `md5` + "rmd160": hasherKeywordFunc("ripemd160digest", ripemd160.New), // The RIPEMD160 message digest of the file + "rmd160digest": hasherKeywordFunc("ripemd160digest", ripemd160.New), // A synonym for `rmd160` + "ripemd160digest": hasherKeywordFunc("ripemd160digest", ripemd160.New), // A synonym for `rmd160` + "sha1": hasherKeywordFunc("sha1digest", sha1.New), // The SHA1 message digest of the file + "sha1digest": hasherKeywordFunc("sha1digest", sha1.New), // A synonym for `sha1` + "sha256": hasherKeywordFunc("sha256digest", sha256.New), // The SHA256 message digest of the file + "sha256digest": hasherKeywordFunc("sha256digest", sha256.New), // A synonym for `sha256` + "sha384": hasherKeywordFunc("sha384digest", sha512.New384), // The SHA384 message digest of the file + "sha384digest": hasherKeywordFunc("sha384digest", sha512.New384), // A synonym for `sha384` + "sha512": hasherKeywordFunc("sha512digest", sha512.New), // The SHA512 message digest of the file + "sha512digest": hasherKeywordFunc("sha512digest", sha512.New), // A synonym for `sha512` + + "flags": flagsKeywordFunc, // NOTE: this is a noop, but here to support the presence of the "flags" keyword. + + // This is not an upstreamed keyword, but used to vary from "time", as tar + // archives do not store nanosecond precision. So comparing on "time" will + // be only seconds level accurate. + "tar_time": tartimeKeywordFunc, // The last modification time of the file, from a tar archive mtime + + // This is not an upstreamed keyword, but a needed attribute for file validation. + // The pattern for this keyword key is prefixed by "xattr." followed by the extended attribute "namespace.key". + // The keyword value is the SHA1 digest of the extended attribute's value. + // In this way, the order of the keys does not matter, and the contents of the value is not revealed. + "xattr": xattrKeywordFunc, + "xattrs": xattrKeywordFunc, + } +) +var ( + modeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + permissions := info.Mode().Perm() + if os.ModeSetuid&info.Mode() > 0 { + permissions |= (1 << 11) + } + if os.ModeSetgid&info.Mode() > 0 { + permissions |= (1 << 10) + } + if os.ModeSticky&info.Mode() > 0 { + permissions |= (1 << 9) + } + return KeyVal(fmt.Sprintf("mode=%#o", permissions)), nil + } + sizeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + if sys, ok := info.Sys().(*tar.Header); ok { + if sys.Typeflag == tar.TypeSymlink { + return KeyVal(fmt.Sprintf("size=%d", len(sys.Linkname))), nil + } + } + return KeyVal(fmt.Sprintf("size=%d", info.Size())), nil + } + cksumKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + if !info.Mode().IsRegular() { + return emptyKV, nil + } + sum, _, err := cksum(r) + if err != nil { + return emptyKV, err + } + return KeyVal(fmt.Sprintf("cksum=%d", sum)), nil + } + hasherKeywordFunc = func(name string, newHash func() hash.Hash) KeywordFunc { + return func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + if !info.Mode().IsRegular() { + return emptyKV, nil + } + h := newHash() + if _, err := io.Copy(h, r); err != nil { + return emptyKV, err + } + return KeyVal(fmt.Sprintf("%s=%x", KeywordSynonym(name), h.Sum(nil))), nil + } + } + tartimeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + return KeyVal(fmt.Sprintf("tar_time=%d.%9.9d", info.ModTime().Unix(), 0)), nil + } + timeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + tSec := info.ModTime().Unix() + tNano := info.ModTime().Nanosecond() + return KeyVal(fmt.Sprintf("time=%d.%9.9d", tSec, tNano)), nil + } + linkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + if sys, ok := info.Sys().(*tar.Header); ok { + if sys.Linkname != "" { + linkname, err := Vis(sys.Linkname) + if err != nil { + return emptyKV, err + } + return KeyVal(fmt.Sprintf("link=%s", linkname)), nil + } + return emptyKV, nil + } + + if info.Mode()&os.ModeSymlink != 0 { + str, err := os.Readlink(path) + if err != nil { + return emptyKV, err + } + linkname, err := Vis(str) + if err != nil { + return emptyKV, err + } + return KeyVal(fmt.Sprintf("link=%s", linkname)), nil + } + return emptyKV, nil + } + typeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + if info.Mode().IsDir() { + return "type=dir", nil + } + if info.Mode().IsRegular() { + return "type=file", nil + } + if info.Mode()&os.ModeSocket != 0 { + return "type=socket", nil + } + if info.Mode()&os.ModeSymlink != 0 { + return "type=link", nil + } + if info.Mode()&os.ModeNamedPipe != 0 { + return "type=fifo", nil + } + if info.Mode()&os.ModeDevice != 0 { + if info.Mode()&os.ModeCharDevice != 0 { + return "type=char", nil + } + return "type=device", nil + } + return emptyKV, nil + } +) diff --git a/keywords.go b/keywords.go index 93916f4..b139d6d 100644 --- a/keywords.go +++ b/keywords.go @@ -1,55 +1,97 @@ package mtree import ( - "archive/tar" - "crypto/md5" - "crypto/sha1" - "crypto/sha256" - "crypto/sha512" "fmt" - "hash" - "io" - "os" "strings" - - "golang.org/x/crypto/ripemd160" ) -// KeywordFunc is the type of a function called on each file to be included in -// a DirectoryHierarchy, that will produce the string output of the keyword to -// be included for the file entry. Otherwise, empty string. -// io.Reader `r` is to the file stream for the file payload. While this -// function takes an io.Reader, the caller needs to reset it to the beginning -// for each new KeywordFunc -type KeywordFunc func(path string, info os.FileInfo, r io.Reader) (string, error) - // Keyword is the string name of a keyword, with some convenience functions for // determining whether it is a default or bsd standard keyword. type Keyword string // Default returns whether this keyword is in the default set of keywords func (k Keyword) Default() bool { - return inSlice(string(k), DefaultKeywords) + return InKeywordSlice(k, DefaultKeywords) } // Bsd returns whether this keyword is in the upstream FreeBSD mtree(8) func (k Keyword) Bsd() bool { - return inSlice(string(k), BsdKeywords) + return InKeywordSlice(k, BsdKeywords) +} + +// Synonym returns the canonical name for this keyword. This is provides the +// same functionality as KeywordSynonym() +func (k Keyword) Synonym() Keyword { + return KeywordSynonym(string(k)) +} + +// InKeywordSlice checks for the presence of `a` in `list` +func InKeywordSlice(a Keyword, list []Keyword) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} +func inKeyValSlice(a KeyVal, list []KeyVal) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} + +// ToKeywords makes a list of Keyword from a list of string +func ToKeywords(list []string) []Keyword { + ret := make([]Keyword, len(list)) + for i := range list { + ret[i] = Keyword(list[i]) + } + return ret +} + +// FromKeywords makes a list of string from a list of Keyword +func FromKeywords(list []Keyword) []string { + ret := make([]string, len(list)) + for i := range list { + ret[i] = string(list[i]) + } + return ret +} + +// KeyValToString constructs a list of string from the list of KeyVal +func KeyValToString(list []KeyVal) []string { + ret := make([]string, len(list)) + for i := range list { + ret[i] = string(list[i]) + } + return ret +} + +// StringToKeyVals constructs a list of KeyVal from the list of strings, like "keyword=value" +func StringToKeyVals(list []string) []KeyVal { + ret := make([]KeyVal, len(list)) + for i := range list { + ret[i] = KeyVal(list[i]) + } + return ret } // KeyVal is a "keyword=value" type KeyVal string // Keyword is the mapping to the available keywords -func (kv KeyVal) Keyword() string { +func (kv KeyVal) Keyword() Keyword { if !strings.Contains(string(kv), "=") { - return "" + return Keyword("") } chunks := strings.SplitN(strings.TrimSpace(string(kv)), "=", 2)[0] if !strings.Contains(chunks, ".") { - return chunks + return Keyword(chunks) } - return strings.SplitN(chunks, ".", 2)[0] + return Keyword(strings.SplitN(chunks, ".", 2)[0]) } // KeywordSuffix is really only used for xattr, as the keyword is a prefix to @@ -78,7 +120,7 @@ func (kv KeyVal) ChangeValue(newval string) string { return fmt.Sprintf("%s=%s", kv.Keyword(), newval) } -// KeyValEqual returns whether two KeyVals are equivalent. This takes +// KeyValEqual returns whether two KeyVal are equivalent. This takes // care of certain odd cases such as tar_mtime, and should be used over // using == comparisons directly unless you really know what you're // doing. @@ -87,35 +129,49 @@ func KeyValEqual(a, b KeyVal) bool { return a.Keyword() == b.Keyword() && a.Value() == b.Value() } -// keywordSelector takes an array of "keyword=value" and filters out that only the set of words -func keywordSelector(keyval, words []string) []string { - retList := []string{} +// keyvalSelector takes an array of KeyVal ("keyword=value") and filters out that only the set of keywords +func keyvalSelector(keyval []KeyVal, keyset []Keyword) []KeyVal { + retList := []KeyVal{} for _, kv := range keyval { - if inSlice(KeyVal(kv).Keyword(), words) { + if InKeywordSlice(kv.Keyword(), keyset) { retList = append(retList, kv) } } return retList } -// NewKeyVals constructs a list of KeyVal from the list of strings, like "keyword=value" -func NewKeyVals(keyvals []string) KeyVals { - kvs := make(KeyVals, len(keyvals)) - for i := range keyvals { - kvs[i] = KeyVal(keyvals[i]) +func keyValDifference(this, that []KeyVal) []KeyVal { + if len(this) == 0 { + return that } - return kvs + diff := []KeyVal{} + for _, kv := range this { + if !inKeyValSlice(kv, that) { + diff = append(diff, kv) + } + } + return diff +} +func keyValCopy(set []KeyVal) []KeyVal { + ret := make([]KeyVal, len(set)) + for i := range set { + ret[i] = set[i] + } + return ret } - -// KeyVals is a list of KeyVal -type KeyVals []KeyVal // Has the "keyword" present in the list of KeyVal, and returns the // corresponding KeyVal, else an empty string. -func (kvs KeyVals) Has(keyword string) KeyVal { - for i := range kvs { - if kvs[i].Keyword() == keyword { - return kvs[i] +func Has(keyvals []KeyVal, keyword string) KeyVal { + return HasKeyword(keyvals, Keyword(keyword)) +} + +// HasKeyword the "keyword" present in the list of KeyVal, and returns the +// corresponding KeyVal, else an empty string. +func HasKeyword(keyvals []KeyVal, keyword Keyword) KeyVal { + for i := range keyvals { + if keyvals[i].Keyword() == keyword { + return keyvals[i] } } return emptyKV @@ -125,20 +181,27 @@ var emptyKV = KeyVal("") // MergeSet takes the current setKeyVals, and then applies the entryKeyVals // such that the entry's values win. The union is returned. -func MergeSet(setKeyVals, entryKeyVals []string) KeyVals { - retList := NewKeyVals(append([]string{}, setKeyVals...)) - eKVs := NewKeyVals(entryKeyVals) - seenKeywords := []string{} +func MergeSet(setKeyVals, entryKeyVals []string) []KeyVal { + retList := StringToKeyVals(setKeyVals) + eKVs := StringToKeyVals(entryKeyVals) + return MergeKeyValSet(retList, eKVs) +} + +// MergeKeyValSet does a merge of the two sets of KeyVal, and the KeyVal of +// entryKeyVals win when there is a duplicate Keyword. +func MergeKeyValSet(setKeyVals, entryKeyVals []KeyVal) []KeyVal { + retList := keyValCopy(setKeyVals) + seenKeywords := []Keyword{} for i := range retList { word := retList[i].Keyword() - if ekv := eKVs.Has(word); ekv != emptyKV { + if ekv := HasKeyword(entryKeyVals, word); ekv != emptyKV { retList[i] = ekv } seenKeywords = append(seenKeywords, word) } - for i := range eKVs { - if !inSlice(eKVs[i].Keyword(), seenKeywords) { - retList = append(retList, eKVs[i]) + for i := range entryKeyVals { + if !InKeywordSlice(entryKeyVals[i].Keyword(), seenKeywords) { + retList = append(retList, entryKeyVals[i]) } } return retList @@ -147,7 +210,7 @@ func MergeSet(setKeyVals, entryKeyVals []string) KeyVals { var ( // DefaultKeywords has the several default keyword producers (uid, gid, // mode, nlink, type, size, mtime) - DefaultKeywords = []string{ + DefaultKeywords = []Keyword{ "size", "type", "uid", @@ -160,7 +223,7 @@ var ( // DefaultTarKeywords has keywords that should be used when creating a manifest from // an archive. Currently, evaluating the # of hardlinks has not been implemented yet - DefaultTarKeywords = []string{ + DefaultTarKeywords = []Keyword{ "size", "type", "uid", @@ -171,7 +234,7 @@ var ( } // BsdKeywords is the set of keywords that is only in the upstream FreeBSD mtree - BsdKeywords = []string{ + BsdKeywords = []Keyword{ "cksum", "device", "flags", // this one is really mostly BSD specific ... @@ -205,176 +268,36 @@ var ( } // SetKeywords is the default set of keywords calculated for a `/set` SpecialType - SetKeywords = []string{ + SetKeywords = []Keyword{ "uid", "gid", } - // KeywordFuncs is the map of all keywords (and the functions to produce them) - KeywordFuncs = map[string]KeywordFunc{ - "size": sizeKeywordFunc, // The size, in bytes, of the file - "type": typeKeywordFunc, // The type of the file - "time": timeKeywordFunc, // The last modification time of the file - "link": linkKeywordFunc, // The target of the symbolic link when type=link - "uid": uidKeywordFunc, // The file owner as a numeric value - "gid": gidKeywordFunc, // The file group as a numeric value - "nlink": nlinkKeywordFunc, // The number of hard links the file is expected to have - "uname": unameKeywordFunc, // The file owner as a symbolic name - "mode": modeKeywordFunc, // The current file's permissions as a numeric (octal) or symbolic value - "cksum": cksumKeywordFunc, // The checksum of the file using the default algorithm specified by the cksum(1) utility - "md5": hasherKeywordFunc("md5digest", md5.New), // The MD5 message digest of the file - "md5digest": hasherKeywordFunc("md5digest", md5.New), // A synonym for `md5` - "rmd160": hasherKeywordFunc("ripemd160digest", ripemd160.New), // The RIPEMD160 message digest of the file - "rmd160digest": hasherKeywordFunc("ripemd160digest", ripemd160.New), // A synonym for `rmd160` - "ripemd160digest": hasherKeywordFunc("ripemd160digest", ripemd160.New), // A synonym for `rmd160` - "sha1": hasherKeywordFunc("sha1digest", sha1.New), // The SHA1 message digest of the file - "sha1digest": hasherKeywordFunc("sha1digest", sha1.New), // A synonym for `sha1` - "sha256": hasherKeywordFunc("sha256digest", sha256.New), // The SHA256 message digest of the file - "sha256digest": hasherKeywordFunc("sha256digest", sha256.New), // A synonym for `sha256` - "sha384": hasherKeywordFunc("sha384digest", sha512.New384), // The SHA384 message digest of the file - "sha384digest": hasherKeywordFunc("sha384digest", sha512.New384), // A synonym for `sha384` - "sha512": hasherKeywordFunc("sha512digest", sha512.New), // The SHA512 message digest of the file - "sha512digest": hasherKeywordFunc("sha512digest", sha512.New), // A synonym for `sha512` - - "flags": flagsKeywordFunc, // NOTE: this is a noop, but here to support the presence of the "flags" keyword. - - // This is not an upstreamed keyword, but used to vary from "time", as tar - // archives do not store nanosecond precision. So comparing on "time" will - // be only seconds level accurate. - "tar_time": tartimeKeywordFunc, // The last modification time of the file, from a tar archive mtime - - // This is not an upstreamed keyword, but a needed attribute for file validation. - // The pattern for this keyword key is prefixed by "xattr." followed by the extended attribute "namespace.key". - // The keyword value is the SHA1 digest of the extended attribute's value. - // In this way, the order of the keys does not matter, and the contents of the value is not revealed. - "xattr": xattrKeywordFunc, - "xattrs": xattrKeywordFunc, - } ) // KeywordSynonym returns the canonical name for keywords that have synonyms, // and just returns the name provided if there is no synonym. In this way it // ought to be safe to wrap any keyword name. -func KeywordSynonym(name string) string { +func KeywordSynonym(name string) Keyword { + var retname string switch name { case "md5": - return "md5digest" + retname = "md5digest" case "rmd160": - return "ripemd160digest" + retname = "ripemd160digest" case "rmd160digest": - return "ripemd160digest" + retname = "ripemd160digest" case "sha1": - return "sha1digest" + retname = "sha1digest" case "sha256": - return "sha256digest" + retname = "sha256digest" case "sha384": - return "sha384digest" + retname = "sha384digest" case "sha512": - return "sha512digest" + retname = "sha512digest" case "xattrs": - return "xattr" + retname = "xattr" + default: + retname = name } - return name + return Keyword(retname) } - -var ( - modeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - permissions := info.Mode().Perm() - if os.ModeSetuid&info.Mode() > 0 { - permissions |= (1 << 11) - } - if os.ModeSetgid&info.Mode() > 0 { - permissions |= (1 << 10) - } - if os.ModeSticky&info.Mode() > 0 { - permissions |= (1 << 9) - } - return fmt.Sprintf("mode=%#o", permissions), nil - } - sizeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - if sys, ok := info.Sys().(*tar.Header); ok { - if sys.Typeflag == tar.TypeSymlink { - return fmt.Sprintf("size=%d", len(sys.Linkname)), nil - } - } - return fmt.Sprintf("size=%d", info.Size()), nil - } - cksumKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - if !info.Mode().IsRegular() { - return "", nil - } - sum, _, err := cksum(r) - if err != nil { - return "", err - } - return fmt.Sprintf("cksum=%d", sum), nil - } - hasherKeywordFunc = func(name string, newHash func() hash.Hash) KeywordFunc { - return func(path string, info os.FileInfo, r io.Reader) (string, error) { - if !info.Mode().IsRegular() { - return "", nil - } - h := newHash() - if _, err := io.Copy(h, r); err != nil { - return "", err - } - return fmt.Sprintf("%s=%x", KeywordSynonym(name), h.Sum(nil)), nil - } - } - tartimeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - return fmt.Sprintf("tar_time=%d.%9.9d", info.ModTime().Unix(), 0), nil - } - timeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - tSec := info.ModTime().Unix() - tNano := info.ModTime().Nanosecond() - return fmt.Sprintf("time=%d.%9.9d", tSec, tNano), nil - } - linkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - if sys, ok := info.Sys().(*tar.Header); ok { - if sys.Linkname != "" { - linkname, err := Vis(sys.Linkname) - if err != nil { - return "", err - } - return fmt.Sprintf("link=%s", linkname), nil - } - return "", nil - } - - if info.Mode()&os.ModeSymlink != 0 { - str, err := os.Readlink(path) - if err != nil { - return "", err - } - linkname, err := Vis(str) - if err != nil { - return "", err - } - return fmt.Sprintf("link=%s", linkname), nil - } - return "", nil - } - typeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - if info.Mode().IsDir() { - return "type=dir", nil - } - if info.Mode().IsRegular() { - return "type=file", nil - } - if info.Mode()&os.ModeSocket != 0 { - return "type=socket", nil - } - if info.Mode()&os.ModeSymlink != 0 { - return "type=link", nil - } - if info.Mode()&os.ModeNamedPipe != 0 { - return "type=fifo", nil - } - if info.Mode()&os.ModeDevice != 0 { - if info.Mode()&os.ModeCharDevice != 0 { - return "type=char", nil - } - return "type=device", nil - } - return "", nil - } -) diff --git a/keywords_bsd.go b/keywords_bsd.go index 3a82a9b..43a8073 100644 --- a/keywords_bsd.go +++ b/keywords_bsd.go @@ -12,46 +12,46 @@ import ( ) var ( - flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { // ideally this will pull in from here https://www.freebsd.org/cgi/man.cgi?query=chflags&sektion=2 - return "", nil + return emptyKV, nil } - unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if hdr, ok := info.Sys().(*tar.Header); ok { - return fmt.Sprintf("uname=%s", hdr.Uname), nil + return KeyVal(fmt.Sprintf("uname=%s", hdr.Uname)), nil } stat := info.Sys().(*syscall.Stat_t) u, err := user.LookupId(fmt.Sprintf("%d", stat.Uid)) if err != nil { - return "", err + return emptyKV, err } - return fmt.Sprintf("uname=%s", u.Username), nil + return KeyVal(fmt.Sprintf("uname=%s", u.Username)), nil } - uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if hdr, ok := info.Sys().(*tar.Header); ok { - return fmt.Sprintf("uid=%d", hdr.Uid), nil + return KeyVal(fmt.Sprintf("uid=%d", hdr.Uid)), nil } stat := info.Sys().(*syscall.Stat_t) - return fmt.Sprintf("uid=%d", stat.Uid), nil + return KeyVal(fmt.Sprintf("uid=%d", stat.Uid)), nil } - gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if hdr, ok := info.Sys().(*tar.Header); ok { - return fmt.Sprintf("gid=%d", hdr.Gid), nil + return KeyVal(fmt.Sprintf("gid=%d", hdr.Gid)), nil } if stat, ok := info.Sys().(*syscall.Stat_t); ok { - return fmt.Sprintf("gid=%d", stat.Gid), nil + return KeyVal(fmt.Sprintf("gid=%d", stat.Gid)), nil } - return "", nil + return emptyKV, nil } - nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if stat, ok := info.Sys().(*syscall.Stat_t); ok { - return fmt.Sprintf("nlink=%d", stat.Nlink), nil + return KeyVal(fmt.Sprintf("nlink=%d", stat.Nlink)), nil } - return "", nil + return emptyKV, nil } - xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - return "", nil + xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + return emptyKV, nil } ) diff --git a/keywords_linux.go b/keywords_linux.go index ed77015..79cf46e 100644 --- a/keywords_linux.go +++ b/keywords_linux.go @@ -17,71 +17,71 @@ import ( var ( // this is bsd specific https://www.freebsd.org/cgi/man.cgi?query=chflags&sektion=2 - flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { - return "", nil + flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { + return emptyKV, nil } - unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if hdr, ok := info.Sys().(*tar.Header); ok { - return fmt.Sprintf("uname=%s", hdr.Uname), nil + return KeyVal(fmt.Sprintf("uname=%s", hdr.Uname)), nil } stat := info.Sys().(*syscall.Stat_t) u, err := user.LookupId(fmt.Sprintf("%d", stat.Uid)) if err != nil { - return "", err + return emptyKV, err } - return fmt.Sprintf("uname=%s", u.Username), nil + return KeyVal(fmt.Sprintf("uname=%s", u.Username)), nil } - uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if hdr, ok := info.Sys().(*tar.Header); ok { - return fmt.Sprintf("uid=%d", hdr.Uid), nil + return KeyVal(fmt.Sprintf("uid=%d", hdr.Uid)), nil } stat := info.Sys().(*syscall.Stat_t) - return fmt.Sprintf("uid=%d", stat.Uid), nil + return KeyVal(fmt.Sprintf("uid=%d", stat.Uid)), nil } - gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if hdr, ok := info.Sys().(*tar.Header); ok { - return fmt.Sprintf("gid=%d", hdr.Gid), nil + return KeyVal(fmt.Sprintf("gid=%d", hdr.Gid)), nil } if stat, ok := info.Sys().(*syscall.Stat_t); ok { - return fmt.Sprintf("gid=%d", stat.Gid), nil + return KeyVal(fmt.Sprintf("gid=%d", stat.Gid)), nil } - return "", nil + return emptyKV, nil } - nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if stat, ok := info.Sys().(*syscall.Stat_t); ok { - return fmt.Sprintf("nlink=%d", stat.Nlink), nil + return KeyVal(fmt.Sprintf("nlink=%d", stat.Nlink)), nil } - return "", nil + return emptyKV, nil } - xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (string, error) { + xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) { if hdr, ok := info.Sys().(*tar.Header); ok { if len(hdr.Xattrs) == 0 { - return "", nil + return emptyKV, nil } - klist := []string{} + klist := []KeyVal{} for k, v := range hdr.Xattrs { - klist = append(klist, fmt.Sprintf("xattr.%s=%s", k, base64.StdEncoding.EncodeToString([]byte(v)))) + klist = append(klist, KeyVal(fmt.Sprintf("xattr.%s=%s", k, base64.StdEncoding.EncodeToString([]byte(v))))) } - return strings.Join(klist, " "), nil + return KeyVal(strings.Join(KeyValToString(klist), " ")), nil } if !info.Mode().IsRegular() && !info.Mode().IsDir() { - return "", nil + return emptyKV, nil } xlist, err := xattr.List(path) if err != nil { - return "", err + return emptyKV, err } - klist := make([]string, len(xlist)) + klist := make([]KeyVal, len(xlist)) for i := range xlist { data, err := xattr.Get(path, xlist[i]) if err != nil { - return "", err + return emptyKV, err } - klist[i] = fmt.Sprintf("xattr.%s=%s", xlist[i], base64.StdEncoding.EncodeToString(data)) + klist[i] = KeyVal(fmt.Sprintf("xattr.%s=%s", xlist[i], base64.StdEncoding.EncodeToString(data))) } - return strings.Join(klist, " "), nil + return KeyVal(strings.Join(KeyValToString(klist), " ")), nil } ) diff --git a/keywords_test.go b/keywords_test.go index 4411453..b8add8c 100644 --- a/keywords_test.go +++ b/keywords_test.go @@ -54,7 +54,7 @@ func TestKeywordsTimeNano(t *testing.T) { {857125628319, 0}, } { mtime := time.Unix(test.sec, test.nsec) - expected := fmt.Sprintf("time=%d.%9.9d", test.sec, test.nsec) + expected := KeyVal(fmt.Sprintf("time=%d.%9.9d", test.sec, test.nsec)) got, err := timeKeywordFunc("", fakeFileInfo{ mtime: mtime, }, nil) @@ -81,7 +81,7 @@ func TestKeywordsTimeTar(t *testing.T) { {857125628319, 0}, } { mtime := time.Unix(test.sec, test.nsec) - expected := fmt.Sprintf("tar_time=%d.%9.9d", test.sec, 0) + expected := KeyVal(fmt.Sprintf("tar_time=%d.%9.9d", test.sec, 0)) got, err := tartimeKeywordFunc("", fakeFileInfo{ mtime: mtime, }, nil) @@ -96,7 +96,8 @@ func TestKeywordsTimeTar(t *testing.T) { func TestKeywordSynonym(t *testing.T) { checklist := []struct { - give, expect string + give string + expect Keyword }{ {give: "time", expect: "time"}, {give: "md5", expect: "md5digest"}, diff --git a/parse.go b/parse.go index 9e1f9bb..36a7163 100644 --- a/parse.go +++ b/parse.go @@ -48,7 +48,7 @@ func ParseSpec(r io.Reader) (*DirectoryHierarchy, error) { // parse the options f := strings.Fields(str) e.Name = f[0] - e.Keywords = f[1:] + e.Keywords = StringToKeyVals(f[1:]) if e.Name == "/set" { creator.curSet = &e } else if e.Name == "/unset" { @@ -80,7 +80,7 @@ func ParseSpec(r io.Reader) (*DirectoryHierarchy, error) { } else { e.Type = RelativeType } - e.Keywords = f[1:] + e.Keywords = StringToKeyVals(f[1:]) // TODO: gather keywords if using tar stream e.Parent = creator.curDir for i := range e.Keywords { diff --git a/tar.go b/tar.go index 8b8b034..adfaea5 100644 --- a/tar.go +++ b/tar.go @@ -17,11 +17,15 @@ type Streamer interface { Hierarchy() (*DirectoryHierarchy, error) } -var tarDefaultSetKeywords = []string{"type=file", "flags=none", "mode=0664"} +var tarDefaultSetKeywords = []KeyVal{ + "type=file", + "flags=none", + "mode=0664", +} // NewTarStreamer streams a tar archive and creates a file hierarchy based off // of the tar metadata headers -func NewTarStreamer(r io.Reader, keywords []string) Streamer { +func NewTarStreamer(r io.Reader, keywords []Keyword) Streamer { pR, pW := io.Pipe() ts := &tarStream{ pipeReader: pR, @@ -45,17 +49,17 @@ type tarStream struct { pipeWriter *io.PipeWriter teeReader io.Reader tarReader *tar.Reader - keywords []string + keywords []Keyword err error } func (ts *tarStream) readHeaders() { // remove "time" keyword - notimekws := []string{} + notimekws := []Keyword{} for _, kw := range ts.keywords { - if !inSlice(kw, notimekws) { + if !InKeywordSlice(kw, notimekws) { if kw == "time" { - if !inSlice("tar_time", ts.keywords) { + if !InKeywordSlice("tar_time", ts.keywords) { notimekws = append(notimekws, "tar_time") } } else { @@ -74,7 +78,7 @@ func (ts *tarStream) readHeaders() { Type: CommentType, }, Set: nil, - Keywords: []string{"type=dir"}, + Keywords: []KeyVal{"type=dir"}, } metadataEntries := signatureEntries("") for _, e := range metadataEntries { @@ -242,7 +246,7 @@ func populateTree(root, e *Entry, hdr *tar.Header) error { Name: encoded, Type: RelativeType, Parent: parent, - Keywords: []string{"type=dir"}, // temp data + Keywords: []KeyVal{"type=dir"}, // temp data Set: nil, // temp data } pathname, err := newEntry.Path() @@ -276,7 +280,7 @@ func populateTree(root, e *Entry, hdr *tar.Header) error { // root: the "head" of the sub-tree to flatten // creator: a dhCreator that helps with the '/set' keyword // keywords: keywords specified by the user that should be evaluated -func flatten(root *Entry, creator *dhCreator, keywords []string) { +func flatten(root *Entry, creator *dhCreator, keywords []Keyword) { if root == nil || creator == nil { return } @@ -292,18 +296,19 @@ func flatten(root *Entry, creator *dhCreator, keywords []string) { if root.Set != nil { // Check if we need a new set + consolidatedKeys := keyvalSelector(append(tarDefaultSetKeywords, root.Set.Keywords...), keywords) if creator.curSet == nil { creator.curSet = &Entry{ Type: SpecialType, Name: "/set", - Keywords: keywordSelector(append(tarDefaultSetKeywords, root.Set.Keywords...), keywords), + Keywords: consolidatedKeys, Pos: len(creator.DH.Entries), } creator.DH.Entries = append(creator.DH.Entries, *creator.curSet) } else { needNewSet := false for _, k := range root.Set.Keywords { - if !inSlice(k, creator.curSet.Keywords) { + if !inKeyValSlice(k, creator.curSet.Keywords) { needNewSet = true break } @@ -313,7 +318,7 @@ func flatten(root *Entry, creator *dhCreator, keywords []string) { Name: "/set", Type: SpecialType, Pos: len(creator.DH.Entries), - Keywords: keywordSelector(append(tarDefaultSetKeywords, root.Set.Keywords...), keywords), + Keywords: consolidatedKeys, } creator.DH.Entries = append(creator.DH.Entries, *creator.curSet) } @@ -331,7 +336,7 @@ func flatten(root *Entry, creator *dhCreator, keywords []string) { } root.Set = creator.curSet if creator.curSet != nil { - root.Keywords = setDifference(root.Keywords, creator.curSet.Keywords) + root.Keywords = keyValDifference(root.Keywords, creator.curSet.Keywords) } root.Pos = len(creator.DH.Entries) creator.DH.Entries = append(creator.DH.Entries, *root) @@ -376,11 +381,11 @@ func resolveHardlinks(root *Entry, hardlinks map[string][]string, countlinks boo } linkfile.Keywords = basefile.Keywords if countlinks { - linkfile.Keywords = append(linkfile.Keywords, fmt.Sprintf("nlink=%d", len(links)+1)) + linkfile.Keywords = append(linkfile.Keywords, KeyVal(fmt.Sprintf("nlink=%d", len(links)+1))) } } if countlinks { - basefile.Keywords = append(basefile.Keywords, fmt.Sprintf("nlink=%d", len(links)+1)) + basefile.Keywords = append(basefile.Keywords, KeyVal(fmt.Sprintf("nlink=%d", len(links)+1))) } } } @@ -410,19 +415,6 @@ func filter(root *Entry, p func(*Entry) bool) []Entry { return nil } -func setDifference(this, that []string) []string { - if len(this) == 0 { - return that - } - diff := []string{} - for _, kv := range this { - if !inSlice(kv, that) { - diff = append(diff, kv) - } - } - return diff -} - func (ts *tarStream) setErr(err error) { ts.err = err } @@ -444,7 +436,7 @@ func (ts *tarStream) Hierarchy() (*DirectoryHierarchy, error) { if ts.root == nil { return nil, fmt.Errorf("root Entry not found, nothing to flatten") } - resolveHardlinks(ts.root, ts.hardlinks, inSlice("nlink", ts.keywords)) + resolveHardlinks(ts.root, ts.hardlinks, InKeywordSlice(Keyword("nlink"), ts.keywords)) flatten(ts.root, &ts.creator, ts.keywords) return ts.creator.DH, nil } diff --git a/tar_test.go b/tar_test.go index f11763f..15df52b 100644 --- a/tar_test.go +++ b/tar_test.go @@ -128,7 +128,7 @@ func TestArchiveCreation(t *testing.T) { if err != nil { t.Fatal(err) } - str := NewTarStreamer(fh, []string{"sha1"}) + str := NewTarStreamer(fh, []Keyword{"sha1"}) if _, err := io.Copy(ioutil.Discard, str); err != nil && err != io.EOF { t.Fatal(err) @@ -145,7 +145,7 @@ func TestArchiveCreation(t *testing.T) { } // Test the tar manifest against the actual directory - res, err := Check("./testdata/collection", tdh, []string{"sha1"}) + res, err := Check("./testdata/collection", tdh, []Keyword{"sha1"}) if err != nil { t.Fatal(err) } @@ -158,7 +158,7 @@ func TestArchiveCreation(t *testing.T) { } // Test the tar manifest against itself - res, err = TarCheck(tdh, tdh, []string{"sha1"}) + res, err = TarCheck(tdh, tdh, []Keyword{"sha1"}) if err != nil { t.Fatal(err) } @@ -170,11 +170,11 @@ func TestArchiveCreation(t *testing.T) { } // Validate the directory manifest against the archive - dh, err := Walk("./testdata/collection", nil, []string{"sha1"}) + dh, err := Walk("./testdata/collection", nil, []Keyword{"sha1"}) if err != nil { t.Fatal(err) } - res, err = TarCheck(tdh, dh, []string{"sha1"}) + res, err = TarCheck(tdh, dh, []Keyword{"sha1"}) if err != nil { t.Fatal(err) } @@ -212,7 +212,7 @@ func TestTreeTraversal(t *testing.T) { t.Fatal(err) } - res, err := TarCheck(tdh, tdh, []string{"sha1"}) + res, err := TarCheck(tdh, tdh, []Keyword{"sha1"}) if err != nil { t.Fatal(err) } @@ -224,7 +224,7 @@ func TestTreeTraversal(t *testing.T) { } // top-level "." directory will contain contents of traversal.tar - res, err = Check("./testdata/.", tdh, []string{"sha1"}) + res, err = Check("./testdata/.", tdh, []Keyword{"sha1"}) if err != nil { t.Fatal(err) } @@ -262,7 +262,7 @@ func TestTreeTraversal(t *testing.T) { } // Implied top-level "." directory will contain the contents of singlefile.tar - res, err = Check("./testdata/.", tdh, []string{"sha1"}) + res, err = Check("./testdata/.", tdh, []Keyword{"sha1"}) if err != nil { t.Fatal(err) } diff --git a/walk.go b/walk.go index 8e0763f..13a4fc8 100644 --- a/walk.go +++ b/walk.go @@ -15,12 +15,12 @@ import ( // returns true, then the path is not included in the spec. type ExcludeFunc func(path string, info os.FileInfo) bool -var defaultSetKeywords = []string{"type=file", "nlink=1", "flags=none", "mode=0664"} +var defaultSetKeywords = []KeyVal{"type=file", "nlink=1", "flags=none", "mode=0664"} // Walk from root directory and assemble the DirectoryHierarchy. excludes // provided are used to skip paths. keywords are the set to collect from the // walked paths. The recommended default list is DefaultKeywords. -func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHierarchy, error) { +func Walk(root string, excludes []ExcludeFunc, keywords []Keyword) (*DirectoryHierarchy, error) { creator := dhCreator{DH: &DirectoryHierarchy{}} // insert signature and metadata comments first (user, machine, tree, date) metadataEntries := signatureEntries(root) @@ -32,7 +32,7 @@ func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHie if err != nil { return err } - for _, ex := range exlcudes { + for _, ex := range excludes { if ex(path, info) { return nil } @@ -71,7 +71,7 @@ func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHie Name: "/set", Type: SpecialType, Pos: len(creator.DH.Entries), - Keywords: keywordSelector(defaultSetKeywords, keywords), + Keywords: keyvalSelector(defaultSetKeywords, keywords), } for _, keyword := range SetKeywords { err := func() error { @@ -103,7 +103,7 @@ func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHie creator.DH.Entries = append(creator.DH.Entries, e) } else if creator.curSet != nil { // check the attributes of the /set keywords and re-set if changed - klist := []string{} + klist := []KeyVal{} for _, keyword := range SetKeywords { err := func() error { var r io.Reader @@ -135,7 +135,7 @@ func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHie needNewSet := false for _, k := range klist { - if !inSlice(k, creator.curSet.Keywords) { + if !inKeyValSlice(k, creator.curSet.Keywords) { needNewSet = true } } @@ -144,7 +144,7 @@ func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHie Name: "/set", Type: SpecialType, Pos: len(creator.DH.Entries), - Keywords: keywordSelector(append(defaultSetKeywords, klist...), keywords), + Keywords: keyvalSelector(append(defaultSetKeywords, klist...), keywords), } creator.curSet = &e creator.DH.Entries = append(creator.DH.Entries, e) @@ -181,7 +181,7 @@ func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHie if err != nil { return err } - if str != "" && !inSlice(str, creator.curSet.Keywords) { + if str != "" && !inKeyValSlice(str, creator.curSet.Keywords) { e.Keywords = append(e.Keywords, str) } return nil @@ -209,15 +209,6 @@ func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHie return creator.DH, err } -func inSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} - // startWalk walks the file tree rooted at root, calling walkFn for each file or // directory in the tree, including root. All errors that arise visiting files // and directories are filtered by walkFn. The files are walked in lexical