From c0a5cb25ecc12c60572d4dafa01df4bdadbf8ae1 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 17 Nov 2016 22:43:02 -0500 Subject: [PATCH] -d: exclude non-directories Adding flag, and supporting functionality for exluding entries that are non-directories Signed-off-by: Vincent Batts --- cmd/gomtree/main.go | 21 ++++++++++++++------- compare_test.go | 2 +- tar.go | 12 +++++++++++- tar_test.go | 42 ++++++++++++++++++++++++++++++++++++------ test/cli/0006.sh | 33 +++++++++++++++++++++++++++++++++ walk.go | 5 +++++ walk_test.go | 17 +++++++++++++++++ 7 files changed, 117 insertions(+), 15 deletions(-) create mode 100644 test/cli/0006.sh diff --git a/cmd/gomtree/main.go b/cmd/gomtree/main.go index 9461638..e66b425 100644 --- a/cmd/gomtree/main.go +++ b/cmd/gomtree/main.go @@ -16,11 +16,12 @@ import ( var ( // Flags common with mtree(8) - flCreate = flag.Bool("c", false, "create a directory hierarchy spec") - flFile = flag.String("f", "", "directory hierarchy spec to validate") - flPath = flag.String("p", "", "root path that the hierarchy spec is relative to") - flAddKeywords = flag.String("K", "", "Add the specified (delimited by comma or space) keywords to the current set of keywords") - flUseKeywords = flag.String("k", "", "Use the specified (delimited by comma or space) keywords as the current set of keywords") + flCreate = flag.Bool("c", false, "create a directory hierarchy spec") + flFile = flag.String("f", "", "directory hierarchy spec to validate") + flPath = flag.String("p", "", "root path that the hierarchy spec is relative to") + flAddKeywords = flag.String("K", "", "Add the specified (delimited by comma or space) keywords to the current set of keywords") + flUseKeywords = flag.String("k", "", "Use the specified (delimited by comma or space) keywords as the current set of keywords") + flDirectoryOnly = flag.Bool("d", false, "Ignore everything except directory type files") // Flags unique to gomtree flListKeywords = flag.Bool("list-keywords", false, "List the keywords available") @@ -201,6 +202,12 @@ func app() error { rootPath = *flPath } + excludes := []mtree.ExcludeFunc{} + // -d + if *flDirectoryOnly { + excludes = append(excludes, mtree.ExcludeNonDirectories) + } + // -T if *flTar != "" { var input io.Reader @@ -214,7 +221,7 @@ func app() error { defer fh.Close() input = fh } - ts := mtree.NewTarStreamer(input, currentKeywords) + ts := mtree.NewTarStreamer(input, excludes, currentKeywords) if _, err := io.Copy(ioutil.Discard, ts); err != nil && err != io.EOF { return err @@ -229,7 +236,7 @@ func app() error { } } else { // with a root directory - stateDh, err = mtree.Walk(rootPath, nil, currentKeywords) + stateDh, err = mtree.Walk(rootPath, excludes, currentKeywords) if err != nil { return err } diff --git a/compare_test.go b/compare_test.go index 23a62f9..cfba566 100644 --- a/compare_test.go +++ b/compare_test.go @@ -391,7 +391,7 @@ func TestTarCompare(t *testing.T) { t.Fatal(err) } - str := NewTarStreamer(bytes.NewBuffer(ts), append(DefaultTarKeywords, "sha1")) + str := NewTarStreamer(bytes.NewBuffer(ts), nil, append(DefaultTarKeywords, "sha1")) if _, err = io.Copy(ioutil.Discard, str); err != nil && err != io.EOF { t.Fatal(err) } diff --git a/tar.go b/tar.go index adfaea5..8165841 100644 --- a/tar.go +++ b/tar.go @@ -25,7 +25,7 @@ var tarDefaultSetKeywords = []KeyVal{ // NewTarStreamer streams a tar archive and creates a file hierarchy based off // of the tar metadata headers -func NewTarStreamer(r io.Reader, keywords []Keyword) Streamer { +func NewTarStreamer(r io.Reader, excludes []ExcludeFunc, keywords []Keyword) Streamer { pR, pW := io.Pipe() ts := &tarStream{ pipeReader: pR, @@ -35,6 +35,7 @@ func NewTarStreamer(r io.Reader, keywords []Keyword) Streamer { tarReader: tar.NewReader(pR), keywords: keywords, hardlinks: map[string][]string{}, + excludes: excludes, } go ts.readHeaders() @@ -50,6 +51,7 @@ type tarStream struct { teeReader io.Reader tarReader *tar.Reader keywords []Keyword + excludes []ExcludeFunc err error } @@ -85,12 +87,20 @@ func (ts *tarStream) readHeaders() { e.Pos = len(ts.creator.DH.Entries) ts.creator.DH.Entries = append(ts.creator.DH.Entries, e) } +hdrloop: for { hdr, err := ts.tarReader.Next() if err != nil { ts.pipeReader.CloseWithError(err) return } + + for _, ex := range ts.excludes { + if ex(hdr.Name, hdr.FileInfo()) { + continue hdrloop + } + } + // Because the content of the file may need to be read by several // KeywordFuncs, it needs to be an io.Seeker as well. So, just reading from // ts.tarReader is not enough. diff --git a/tar_test.go b/tar_test.go index 15df52b..8ae89a0 100644 --- a/tar_test.go +++ b/tar_test.go @@ -16,7 +16,7 @@ func ExampleStreamer() { if err != nil { // handle error ... } - str := NewTarStreamer(fh, nil) + str := NewTarStreamer(fh, nil, nil) if err := extractTar("/tmp/dir", str); err != nil { // handle error ... } @@ -59,7 +59,7 @@ func TestTar(t *testing.T) { if err != nil { t.Fatal(err) } - str := NewTarStreamer(fh, append(DefaultKeywords, "sha1")) + str := NewTarStreamer(fh, nil, append(DefaultKeywords, "sha1")) if _, err := io.Copy(ioutil.Discard, str); err != nil && err != io.EOF { t.Fatal(err) @@ -128,7 +128,7 @@ func TestArchiveCreation(t *testing.T) { if err != nil { t.Fatal(err) } - str := NewTarStreamer(fh, []Keyword{"sha1"}) + str := NewTarStreamer(fh, nil, []Keyword{"sha1"}) if _, err := io.Copy(ioutil.Discard, str); err != nil && err != io.EOF { t.Fatal(err) @@ -196,7 +196,7 @@ func TestTreeTraversal(t *testing.T) { if err != nil { t.Fatal(err) } - str := NewTarStreamer(fh, DefaultTarKeywords) + str := NewTarStreamer(fh, nil, DefaultTarKeywords) if _, err = io.Copy(ioutil.Discard, str); err != nil && err != io.EOF { t.Fatal(err) @@ -249,7 +249,7 @@ func TestTreeTraversal(t *testing.T) { if err != nil { t.Fatal(err) } - str = NewTarStreamer(fh, DefaultTarKeywords) + str = NewTarStreamer(fh, nil, DefaultTarKeywords) if _, err = io.Copy(ioutil.Discard, str); err != nil && err != io.EOF { t.Fatal(err) } @@ -287,7 +287,7 @@ func TestHardlinks(t *testing.T) { if err != nil { t.Fatal(err) } - str := NewTarStreamer(fh, append(DefaultTarKeywords, "nlink")) + str := NewTarStreamer(fh, nil, append(DefaultTarKeywords, "nlink")) if _, err = io.Copy(ioutil.Discard, str); err != nil && err != io.EOF { t.Fatal(err) @@ -369,3 +369,33 @@ func makeTarStream(ff []fakeFile) ([]byte, error) { } return buf.Bytes(), nil } + +func TestArchiveExcludeNonDirectory(t *testing.T) { + fh, err := os.Open("./testdata/collection.tar") + if err != nil { + t.Fatal(err) + } + str := NewTarStreamer(fh, []ExcludeFunc{ExcludeNonDirectories}, []Keyword{"type"}) + + if _, err := io.Copy(ioutil.Discard, str); err != nil && err != io.EOF { + t.Fatal(err) + } + if err := str.Close(); err != nil { + t.Fatal(err) + } + fh.Close() + // get DirectoryHierarcy struct from walking the tar archive + tdh, err := str.Hierarchy() + if err != nil { + t.Fatal(err) + } + for i := range tdh.Entries { + for _, keyval := range tdh.Entries[i].AllKeys() { + if tdh.Entries[i].Type == FullType || tdh.Entries[i].Type == RelativeType { + if keyval.Keyword() == "type" && keyval.Value() != "dir" { + t.Errorf("expected only directories, but %q is a %q", tdh.Entries[i].Name, keyval.Value()) + } + } + } + } +} diff --git a/test/cli/0006.sh b/test/cli/0006.sh new file mode 100644 index 0000000..611353a --- /dev/null +++ b/test/cli/0006.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -e + +name=$(basename $0) +root=$1 +gomtree=$(readlink -f ${root}/gomtree) +t=$(mktemp -d /tmp/go-mtree.XXXXXX) + +echo "[${name}] Running in ${t}" +# This test is for basic running check of manifest, and check against tar and file system +# + +pushd ${root} + +git archive --format=tar HEAD^{tree} . > ${t}/${name}.tar + +prev_umask=$(umask) +umask 0 # this is so the tar command can set the mode's properly +mkdir -p ${t}/extract +tar -C ${t}/extract/ -xf ${t}/${name}.tar +umask ${prev_umask} + +# create manifest from tar, ignoring non directories +${gomtree} -d -c -k type -T ${t}/${name}.tar > ${t}/${name}.mtree + +# check tar-manifest against the tar +${gomtree} -d -f ${t}/${name}.mtree -T ${t}/${name}.tar + +# check filesystem-manifest against the filesystem +${gomtree} -f ${t}/${name}.mtree -p ${t}/extract/ + +popd +rm -rf ${t} diff --git a/walk.go b/walk.go index 13a4fc8..5209d6e 100644 --- a/walk.go +++ b/walk.go @@ -15,6 +15,11 @@ import ( // returns true, then the path is not included in the spec. type ExcludeFunc func(path string, info os.FileInfo) bool +// ExcludeNonDirectories is an ExcludeFunc for excluding all paths that are not directories +var ExcludeNonDirectories = func(path string, info os.FileInfo) bool { + return !info.IsDir() +} + var defaultSetKeywords = []KeyVal{"type=file", "nlink=1", "flags=none", "mode=0664"} // Walk from root directory and assemble the DirectoryHierarchy. excludes diff --git a/walk_test.go b/walk_test.go index 75c404a..d40ea90 100644 --- a/walk_test.go +++ b/walk_test.go @@ -35,3 +35,20 @@ func TestWalk(t *testing.T) { } } } + +func TestWalkDirectory(t *testing.T) { + dh, err := Walk(".", []ExcludeFunc{ExcludeNonDirectories}, []Keyword{"type"}) + if err != nil { + t.Fatal(err) + } + + for i := range dh.Entries { + for _, keyval := range dh.Entries[i].AllKeys() { + if dh.Entries[i].Type == FullType || dh.Entries[i].Type == RelativeType { + if keyval.Keyword() == "type" && keyval.Value() != "dir" { + t.Errorf("expected only directories, but %q is a %q", dh.Entries[i].Name, keyval.Value()) + } + } + } + } +}