From ca06fee1d8deafa7ffb8d5e08f568854013ab90c Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Mon, 25 Jul 2016 18:29:10 -0400 Subject: [PATCH 1/8] main: don't dupliate keywords When adding keywords with -K, we don't want to duplicate keywords in the keywords slice. Signed-off-by: Stephen Chung --- cmd/gomtree/main.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/gomtree/main.go b/cmd/gomtree/main.go index 705c41c..3b48b0d 100644 --- a/cmd/gomtree/main.go +++ b/cmd/gomtree/main.go @@ -102,7 +102,11 @@ func main() { } // -K if *flAddKeywords != "" { - currentKeywords = append(currentKeywords, splitKeywordsArg(*flAddKeywords)...) + for _, kw := range splitKeywordsArg(*flAddKeywords) { + if !inSlice(kw, currentKeywords) { + currentKeywords = append(currentKeywords, kw) + } + } } // -f From b161688aadcc320dfd78672ed2f06745005257f1 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Mon, 25 Jul 2016 20:35:38 -0400 Subject: [PATCH 2/8] cmd: gomtree no arguments my tar_stream_tar_time PR accidentally put the functionality for when gomtree has no arguments inside an unreachable block. Signed-off-by: Stephen Chung --- cmd/gomtree/main.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/gomtree/main.go b/cmd/gomtree/main.go index 3b48b0d..25777ef 100644 --- a/cmd/gomtree/main.go +++ b/cmd/gomtree/main.go @@ -227,12 +227,12 @@ func main() { fmt.Printf("%s missing\n", missingpath) } } - } else { - log.Println("neither validating or creating a manifest. Please provide additional arguments") - isErr = true - defer os.Exit(1) - return } + } else { + log.Println("neither validating or creating a manifest. Please provide additional arguments") + isErr = true + defer os.Exit(1) + return } } From 2f80b2c54e891ab4e0ce0a5af584b69b00c50690 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Mon, 25 Jul 2016 20:04:55 -0400 Subject: [PATCH 3/8] tar: explicitly close files after populateTree Files don't close properly when `defer`ing inside a for loop, since the surrounding function is still iterating in a for loop. To fix this, just close the files explicitly after `populateTree()` in `readHeaders()` Signed-off-by: Stephen Chung --- tar.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tar.go b/tar.go index 9455853..9ea759d 100644 --- a/tar.go +++ b/tar.go @@ -93,9 +93,6 @@ func (ts *tarStream) readHeaders() { ts.pipeReader.CloseWithError(err) return } - defer tmpFile.Close() - defer os.Remove(tmpFile.Name()) - // Alright, it's either file or directory encodedName, err := Vis(filepath.Base(hdr.Name)) if err != nil { @@ -175,6 +172,8 @@ func (ts *tarStream) readHeaders() { } } populateTree(&root, &e, hdr, ts) + tmpFile.Close() + os.Remove(tmpFile.Name()) } } From a596aefc434d30a7d4a3e188e20376591e5b0087 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Wed, 20 Jul 2016 13:28:08 -0400 Subject: [PATCH 4/8] debug: add an mtree.Debugf and -debug flag Signed-off-by: Vincent Batts --- cmd/gomtree/main.go | 6 ++++++ debug.go | 18 ++++++++++++++++++ keywords.go | 3 ++- keywords_linux.go | 2 -- 4 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 debug.go diff --git a/cmd/gomtree/main.go b/cmd/gomtree/main.go index 25777ef..6563cbc 100644 --- a/cmd/gomtree/main.go +++ b/cmd/gomtree/main.go @@ -23,6 +23,8 @@ var ( flListKeywords = flag.Bool("list-keywords", false, "List the keywords available") flResultFormat = flag.String("result-format", "bsd", "output the validation results using the given format (bsd, json, path)") flTar = flag.String("T", "", "use tar archive to create or validate a directory hierarchy spec") + + flDebug = flag.Bool("debug", false, "output debug info to STDERR") ) var formats = map[string]func(*mtree.Result) string{ @@ -57,6 +59,10 @@ var formats = map[string]func(*mtree.Result) string{ func main() { flag.Parse() + if *flDebug { + os.Setenv("DEBUG", "1") + } + // so that defers cleanly exec var isErr bool defer func() { diff --git a/debug.go b/debug.go new file mode 100644 index 0000000..832035e --- /dev/null +++ b/debug.go @@ -0,0 +1,18 @@ +package mtree + +import ( + "fmt" + "os" + "time" +) + +// DebugOutput is the where DEBUG output is written +var DebugOutput = os.Stderr + +// Debugf does formatted output to DebugOutput, only if DEBUG environment variable is set +func Debugf(format string, a ...interface{}) (n int, err error) { + if os.Getenv("DEBUG") != "" { + return fmt.Fprintf(DebugOutput, "[%d] [DEBUG] %s\n", time.Now().UnixNano(), fmt.Sprintf(format, a...)) + } + return 0, nil +} diff --git a/keywords.go b/keywords.go index 4119fc0..136f679 100644 --- a/keywords.go +++ b/keywords.go @@ -185,7 +185,8 @@ var ( // The pattern for this keyword key is prefixed by "xattr." followed by the extended attribute "namespace.key". // The keyword value is the SHA1 digest of the extended attribute's value. // In this way, the order of the keys does not matter, and the contents of the value is not revealed. - "xattr": xattrKeywordFunc, + "xattr": xattrKeywordFunc, + "xattrs": xattrKeywordFunc, } ) diff --git a/keywords_linux.go b/keywords_linux.go index 26bfc40..177c89f 100644 --- a/keywords_linux.go +++ b/keywords_linux.go @@ -1,5 +1,3 @@ -// +build linux - package mtree import ( From dcc5afbd33692fd1c3c685089c5c0e00a0eb4397 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 26 Jul 2016 13:55:25 -0400 Subject: [PATCH 5/8] xattr: fix the failure on empty list Signed-off-by: Vincent Batts --- xattr/xattr.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/xattr/xattr.go b/xattr/xattr.go index 3e2b53b..d6ad9ce 100644 --- a/xattr/xattr.go +++ b/xattr/xattr.go @@ -29,7 +29,14 @@ func List(path string) ([]string, error) { if err != nil { return nil, err } - return strings.Split(strings.TrimRight(string(dest[:i]), nilByte), nilByte), nil + + // If the returned list is empty, return nil instead of []string{""} + str := string(dest[:i]) + if str == "" { + return nil, nil + } + + return strings.Split(strings.TrimRight(str, nilByte), nilByte), nil } const nilByte = "\x00" From 8e19ce80ecdf502e46bd4c9e9a55236269d78a95 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 26 Jul 2016 14:21:44 -0400 Subject: [PATCH 6/8] keywords: include list of upstream keywords Signed-off-by: Vincent Batts --- cmd/gomtree/main.go | 17 +++++++++------ keywords.go | 50 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/cmd/gomtree/main.go b/cmd/gomtree/main.go index 6563cbc..d520dba 100644 --- a/cmd/gomtree/main.go +++ b/cmd/gomtree/main.go @@ -71,20 +71,24 @@ func main() { } }() - // -l + // -list-keywords if *flListKeywords { fmt.Println("Available keywords:") for k := range mtree.KeywordFuncs { - if inSlice(k, mtree.DefaultKeywords) { - fmt.Println(" ", k, " (default)") - } else { - fmt.Println(" ", k) + fmt.Print(" ") + fmt.Print(k) + if mtree.Keyword(k).Default() { + fmt.Print(" (default)") } + if !mtree.Keyword(k).Bsd() { + fmt.Print(" (not upstream)") + } + fmt.Print("\n") } return } - // --output + // --result-format formatFunc, ok := formats[*flResultFormat] if !ok { log.Printf("invalid output format: %s", *flResultFormat) @@ -169,6 +173,7 @@ func main() { return } } + // -c if *flCreate { // create a directory hierarchy diff --git a/keywords.go b/keywords.go index 136f679..2228aca 100644 --- a/keywords.go +++ b/keywords.go @@ -23,6 +23,20 @@ import ( // for each new KeywordFunc type KeywordFunc func(path string, info os.FileInfo, r io.Reader) (string, error) +// Keyword is the string name of a keyword, with some convenience functions for +// determining whether it is a default or bsd standard keyword. +type Keyword string + +// Default returns whether this keyword is in the default set of keywords +func (k Keyword) Default() bool { + return inSlice(string(k), DefaultKeywords) +} + +// Bsd returns whether this keyword is in the upstream FreeBSD mtree(8) +func (k Keyword) Bsd() bool { + return inSlice(string(k), BsdKeywords) +} + // KeyVal is a "keyword=value" type KeyVal string @@ -134,6 +148,7 @@ var ( "nlink", "time", } + // DefaultTarKeywords has keywords that should be used when creating a manifest from // an archive. Currently, evaluating the # of hardlinks has not been implemented yet DefaultTarKeywords = []string{ @@ -145,6 +160,41 @@ var ( "link", "tar_time", } + + // BsdKeywords is the set of keywords that is only in the upstream FreeBSD mtree + BsdKeywords = []string{ + "cksum", + "device", + "flags", + "ignore", + "gid", + "gname", + "link", + "md5", + "md5digest", + "mode", + "nlink", + "nochange", + "optional", + "ripemd160digest", + "rmd160", + "rmd160digest", + "sha1", + "sha1digest", + "sha256", + "sha256digest", + "sha384", + "sha384digest", + "sha512", + "sha512digest", + "size", + "tags", + "time", + "type", + "uid", + "uname", + } + // SetKeywords is the default set of keywords calculated for a `/set` SpecialType SetKeywords = []string{ "uid", From 7b7d96ca781286cd665bfc70c22d21fe502f6dc8 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 26 Jul 2016 14:31:39 -0400 Subject: [PATCH 7/8] main: add -bsd-keywords To support either producing or checking manifests that are compatible with upstream FreeBSD mtree(8) keywords, this flag will only operate on upstream keywords. Completely ignoring non-upstream keywords, though printing out an INFO to stderr for information purposes. Example: ```bash INFO: ignoring "xattrs" as it is not an upstream keyword /set type=file nlink=1 mode=0664 uid=1000 gid=100 . size=4096 type=dir mode=0755 nlink=2 time=1469556206.235575511 [...] ``` Signed-off-by: Vincent Batts --- cmd/gomtree/main.go | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/cmd/gomtree/main.go b/cmd/gomtree/main.go index d520dba..8af7f55 100644 --- a/cmd/gomtree/main.go +++ b/cmd/gomtree/main.go @@ -23,6 +23,7 @@ var ( flListKeywords = flag.Bool("list-keywords", false, "List the keywords available") flResultFormat = flag.String("result-format", "bsd", "output the validation results using the given format (bsd, json, path)") flTar = flag.String("T", "", "use tar archive to create or validate a directory hierarchy spec") + flBsdKeywords = flag.Bool("bsd-keywords", false, "only operate on keywords that are supported by upstream mtree(8)") flDebug = flag.Bool("debug", false, "output debug info to STDERR") ) @@ -96,29 +97,46 @@ func main() { return } - var currentKeywords []string + var ( + tmpKeywords []string + currentKeywords []string + ) // -k if *flUseKeywords != "" { - currentKeywords = splitKeywordsArg(*flUseKeywords) - if !inSlice("type", currentKeywords) { - currentKeywords = append([]string{"type"}, currentKeywords...) + tmpKeywords = splitKeywordsArg(*flUseKeywords) + if !inSlice("type", tmpKeywords) { + tmpKeywords = append([]string{"type"}, tmpKeywords...) } } else { if *flTar != "" { - currentKeywords = mtree.DefaultTarKeywords[:] + tmpKeywords = mtree.DefaultTarKeywords[:] } else { - currentKeywords = mtree.DefaultKeywords[:] + tmpKeywords = mtree.DefaultKeywords[:] } } + // -K if *flAddKeywords != "" { for _, kw := range splitKeywordsArg(*flAddKeywords) { - if !inSlice(kw, currentKeywords) { - currentKeywords = append(currentKeywords, kw) + if !inSlice(kw, tmpKeywords) { + tmpKeywords = append(tmpKeywords, kw) } } } + // -bsd-keywords + if *flBsdKeywords { + for _, k := range tmpKeywords { + if mtree.Keyword(k).Bsd() { + currentKeywords = append(currentKeywords, k) + } else { + fmt.Fprintf(os.Stderr, "INFO: ignoring %q as it is not an upstream keyword\n", k) + } + } + } else { + currentKeywords = tmpKeywords + } + // -f var dh *mtree.DirectoryHierarchy if *flFile != "" && !*flCreate { From ab1cad064de8b26eb5bb994a065e50f7a0ef6ba9 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Wed, 27 Jul 2016 11:40:11 -0400 Subject: [PATCH 8/8] readme: update details about go-mtree in readme Include details about tar compatibility, as well as other aspects that have changed Signed-off-by: Stephen Chung --- README.md | 102 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 73 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index d252b4e..f21a836 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ files are often a critical aspect of the file, holding ACLs, capabilities, etc. While FreeBSD filesystem do support `extattr`, this feature has not made its way into their `mtree`. -This implementation of mtree supports an additional "keyword" of `xattr`. If -you include this keyword, then the FreeBSD `mtree` will fail as it is an -unknown keyword to that implementation. +This implementation of mtree supports a few non-upstream "keyword"s. These are: +`xattr` and `tar_time`. If you include these keywords, then the FreeBSD `mtree` +will fail as they are unknown keywords to that implementation. ### Typical form @@ -70,6 +70,35 @@ avoids issues with encoding, as well as issues of information leaking. The designation of SHA1 is arbitrary and seen as a general "good enough" assertion of the value. +### Tar form + +```mtree +# . +/set type=file mode=0664 uid=1000 gid=1000 +. type=dir mode=0775 tar_time=1468430408.000000000 + +# samedir +samedir type=dir mode=0775 tar_time=1468000972.000000000 + file2 size=0 tar_time=1467999782.000000000 + file1 size=0 tar_time=1467999781.000000000 + +[...] +``` + +While `go-mtree` serves mainly as a library for upstream `mtree` support, +`go-mtree` is also compatible with [tar archives][tar] (which is not an upstream feature). +This means that we can now create and validate a manifest by specifying a tar file. + +Notice that for the output of creating a validation manifest from a tar file, the default behavior +for evaluating a notion of time is to use the `tar_time` keyword. In the +"directory hierarchy" format of mtree, `time` is being evaluated with +nanosecond precision. However, GNU tar truncates a file's modification time +to 1-second precision. That is, if a file's full modification time is +123456789.123456789, the `tar_time` equivalent would be 123456789.000000000. +This way, if you validate a manifest created using a tar file against an +actual root directory, there will be no complaints from `go-mtree` so long as the +1-second precision time of a file in the root directory is the same. + ## Usage @@ -83,13 +112,25 @@ To use the command line tool, first [build it](#Building), then the following. This will also include the sha512 digest of the files. ```bash -gomtree -c -K sha512digest -p . > /tmp/mtree.txt +gomtree -c -K sha512digest -p . > /tmp/root.mtree +``` + +With a tar file: + +```bash +gomtree -c -K sha512digest -T sometarfile.tar > /tmp/tar.mtree ``` ### Validate a manifest ```bash -gomtree -p . -f /tmp/mtree.txt +gomtree -p . -f /tmp/root.mtree +``` + +With a tar file: + +```bash +gomtree -T sometarfile.tar -f /tmp/root.mtree ``` ### See the supported keywords @@ -97,30 +138,32 @@ gomtree -p . -f /tmp/mtree.txt ```bash gomtree -list-keywords Available keywords: - rmd160 - ripemd160digest - type (default) - link (default) - cksum - md5 - md5digest - sha256digest - sha512 - time (default) - gid (default) - mode (default) - sha1 - sha1digest - size (default) - uid (default) - sha256 - sha384 - sha512digest - nlink (default) - uname - rmd160digest - sha384digest - xattr + uname + sha1 + sha1digest + sha256digest + xattrs (not upstream) + link (default) + nlink (default) + md5digest + rmd160digest + mode (default) + cksum + md5 + rmd160 + type (default) + time (default) + uid (default) + gid (default) + sha256 + sha384 + sha512 + xattr (not upstream) + tar_time (not upstream) + size (default) + ripemd160digest + sha384digest + sha512digest ``` @@ -146,3 +189,4 @@ go build . [archiecobbs/mtree-port]: https://github.com/archiecobbs/mtree-port [godoc]: https://godoc.org/github.com/vbatts/go-mtree [sha1]: https://tools.ietf.org/html/rfc3174 +[tar]: http://man7.org/linux/man-pages/man1/tar.1.html