-d: exclude non-directories

Adding flag, and supporting functionality for exluding entries that are
non-directories

Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>

cmd/gomtree/main.go

@@ -16,11 +16,12 @@ import (
 
 var (
 	// Flags common with mtree(8)
-	flCreate      = flag.Bool("c", false, "create a directory hierarchy spec")
+	flCreate        = flag.Bool("c", false, "create a directory hierarchy spec")
-	flFile        = flag.String("f", "", "directory hierarchy spec to validate")
+	flFile          = flag.String("f", "", "directory hierarchy spec to validate")
-	flPath        = flag.String("p", "", "root path that the hierarchy spec is relative to")
+	flPath          = flag.String("p", "", "root path that the hierarchy spec is relative to")
-	flAddKeywords = flag.String("K", "", "Add the specified (delimited by comma or space) keywords to the current set of keywords")
+	flAddKeywords   = flag.String("K", "", "Add the specified (delimited by comma or space) keywords to the current set of keywords")
-	flUseKeywords = flag.String("k", "", "Use the specified (delimited by comma or space) keywords as the current set of keywords")
+	flUseKeywords   = flag.String("k", "", "Use the specified (delimited by comma or space) keywords as the current set of keywords")
+	flDirectoryOnly = flag.Bool("d", false, "Ignore everything except directory type files")
 
 	// Flags unique to gomtree
 	flListKeywords     = flag.Bool("list-keywords", false, "List the keywords available")
@@ -201,6 +202,12 @@ func app() error {
 		rootPath = *flPath
 	}
 
+	excludes := []mtree.ExcludeFunc{}
+	// -d
+	if *flDirectoryOnly {
+		excludes = append(excludes, mtree.ExcludeNonDirectories)
+	}
+
 	// -T <tar file>
 	if *flTar != "" {
 		var input io.Reader
@@ -214,7 +221,7 @@ func app() error {
 			defer fh.Close()
 			input = fh
 		}
-		ts := mtree.NewTarStreamer(input, currentKeywords)
+		ts := mtree.NewTarStreamer(input, excludes, currentKeywords)
 
 		if _, err := io.Copy(ioutil.Discard, ts); err != nil && err != io.EOF {
 			return err
@@ -229,7 +236,7 @@ func app() error {
 		}
 	} else {
 		// with a root directory
-		stateDh, err = mtree.Walk(rootPath, nil, currentKeywords)
+		stateDh, err = mtree.Walk(rootPath, excludes, currentKeywords)
 		if err != nil {
 			return err
 		}

compare_test.go

@@ -391,7 +391,7 @@ func TestTarCompare(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	str := NewTarStreamer(bytes.NewBuffer(ts), append(DefaultTarKeywords, "sha1"))
+	str := NewTarStreamer(bytes.NewBuffer(ts), nil, append(DefaultTarKeywords, "sha1"))
 	if _, err = io.Copy(ioutil.Discard, str); err != nil && err != io.EOF {
 		t.Fatal(err)
 	}

tar.go

@@ -25,7 +25,7 @@ var tarDefaultSetKeywords = []KeyVal{
 
 // NewTarStreamer streams a tar archive and creates a file hierarchy based off
 // of the tar metadata headers
-func NewTarStreamer(r io.Reader, keywords []Keyword) Streamer {
+func NewTarStreamer(r io.Reader, excludes []ExcludeFunc, keywords []Keyword) Streamer {
 	pR, pW := io.Pipe()
 	ts := &tarStream{
 		pipeReader: pR,
@@ -35,6 +35,7 @@ func NewTarStreamer(r io.Reader, keywords []Keyword) Streamer {
 		tarReader:  tar.NewReader(pR),
 		keywords:   keywords,
 		hardlinks:  map[string][]string{},
+		excludes:   excludes,
 	}
 
 	go ts.readHeaders()
@@ -50,6 +51,7 @@ type tarStream struct {
 	teeReader  io.Reader
 	tarReader  *tar.Reader
 	keywords   []Keyword
+	excludes   []ExcludeFunc
 	err        error
 }
 
@@ -85,12 +87,20 @@ func (ts *tarStream) readHeaders() {
 		e.Pos = len(ts.creator.DH.Entries)
 		ts.creator.DH.Entries = append(ts.creator.DH.Entries, e)
 	}
+hdrloop:
 	for {
 		hdr, err := ts.tarReader.Next()
 		if err != nil {
 			ts.pipeReader.CloseWithError(err)
 			return
 		}
+
+		for _, ex := range ts.excludes {
+			if ex(hdr.Name, hdr.FileInfo()) {
+				continue hdrloop
+			}
+		}
+
 		// Because the content of the file may need to be read by several
 		// KeywordFuncs, it needs to be an io.Seeker as well. So, just reading from
 		// ts.tarReader is not enough.

tar_test.go

@@ -16,7 +16,7 @@ func ExampleStreamer() {
 	if err != nil {
 		// handle error ...
 	}
-	str := NewTarStreamer(fh, nil)
+	str := NewTarStreamer(fh, nil, nil)
 	if err := extractTar("/tmp/dir", str); err != nil {
 		// handle error ...
 	}
@@ -59,7 +59,7 @@ func TestTar(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	str := NewTarStreamer(fh, append(DefaultKeywords, "sha1"))
+	str := NewTarStreamer(fh, nil, append(DefaultKeywords, "sha1"))
 
 	if _, err := io.Copy(ioutil.Discard, str); err != nil && err != io.EOF {
 		t.Fatal(err)
@@ -128,7 +128,7 @@ func TestArchiveCreation(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	str := NewTarStreamer(fh, []Keyword{"sha1"})
+	str := NewTarStreamer(fh, nil, []Keyword{"sha1"})
 
 	if _, err := io.Copy(ioutil.Discard, str); err != nil && err != io.EOF {
 		t.Fatal(err)
@@ -196,7 +196,7 @@ func TestTreeTraversal(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	str := NewTarStreamer(fh, DefaultTarKeywords)
+	str := NewTarStreamer(fh, nil, DefaultTarKeywords)
 
 	if _, err = io.Copy(ioutil.Discard, str); err != nil && err != io.EOF {
 		t.Fatal(err)
@@ -249,7 +249,7 @@ func TestTreeTraversal(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	str = NewTarStreamer(fh, DefaultTarKeywords)
+	str = NewTarStreamer(fh, nil, DefaultTarKeywords)
 	if _, err = io.Copy(ioutil.Discard, str); err != nil && err != io.EOF {
 		t.Fatal(err)
 	}
@@ -287,7 +287,7 @@ func TestHardlinks(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	str := NewTarStreamer(fh, append(DefaultTarKeywords, "nlink"))
+	str := NewTarStreamer(fh, nil, append(DefaultTarKeywords, "nlink"))
 
 	if _, err = io.Copy(ioutil.Discard, str); err != nil && err != io.EOF {
 		t.Fatal(err)
@@ -369,3 +369,33 @@ func makeTarStream(ff []fakeFile) ([]byte, error) {
 	}
 	return buf.Bytes(), nil
 }
+
+func TestArchiveExcludeNonDirectory(t *testing.T) {
+	fh, err := os.Open("./testdata/collection.tar")
+	if err != nil {
+		t.Fatal(err)
+	}
+	str := NewTarStreamer(fh, []ExcludeFunc{ExcludeNonDirectories}, []Keyword{"type"})
+
+	if _, err := io.Copy(ioutil.Discard, str); err != nil && err != io.EOF {
+		t.Fatal(err)
+	}
+	if err := str.Close(); err != nil {
+		t.Fatal(err)
+	}
+	fh.Close()
+	// get DirectoryHierarcy struct from walking the tar archive
+	tdh, err := str.Hierarchy()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := range tdh.Entries {
+		for _, keyval := range tdh.Entries[i].AllKeys() {
+			if tdh.Entries[i].Type == FullType || tdh.Entries[i].Type == RelativeType {
+				if keyval.Keyword() == "type" && keyval.Value() != "dir" {
+					t.Errorf("expected only directories, but %q is a %q", tdh.Entries[i].Name, keyval.Value())
+				}
+			}
+		}
+	}
+}

test/cli/0006.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+
+name=$(basename $0)
+root=$1
+gomtree=$(readlink -f ${root}/gomtree)
+t=$(mktemp -d /tmp/go-mtree.XXXXXX)
+
+echo "[${name}] Running in ${t}"
+# This test is for basic running check of manifest, and check against tar and file system
+#
+
+pushd ${root}
+
+git archive --format=tar HEAD^{tree} . > ${t}/${name}.tar
+
+prev_umask=$(umask)
+umask 0 # this is so the tar command can set the mode's properly
+mkdir -p ${t}/extract
+tar -C ${t}/extract/ -xf ${t}/${name}.tar
+umask ${prev_umask}
+
+# create manifest from tar, ignoring non directories
+${gomtree} -d -c -k type -T ${t}/${name}.tar > ${t}/${name}.mtree
+
+# check tar-manifest against the tar
+${gomtree} -d -f ${t}/${name}.mtree -T ${t}/${name}.tar
+
+# check filesystem-manifest against the filesystem
+${gomtree} -f ${t}/${name}.mtree -p ${t}/extract/
+
+popd
+rm -rf ${t}

walk.go

@@ -15,6 +15,11 @@ import (
 // returns true, then the path is not included in the spec.
 type ExcludeFunc func(path string, info os.FileInfo) bool
 
+// ExcludeNonDirectories is an ExcludeFunc for excluding all paths that are not directories
+var ExcludeNonDirectories = func(path string, info os.FileInfo) bool {
+	return !info.IsDir()
+}
+
 var defaultSetKeywords = []KeyVal{"type=file", "nlink=1", "flags=none", "mode=0664"}
 
 // Walk from root directory and assemble the DirectoryHierarchy. excludes

walk_test.go

@@ -35,3 +35,20 @@ func TestWalk(t *testing.T) {
 		}
 	}
 }
+
+func TestWalkDirectory(t *testing.T) {
+	dh, err := Walk(".", []ExcludeFunc{ExcludeNonDirectories}, []Keyword{"type"})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := range dh.Entries {
+		for _, keyval := range dh.Entries[i].AllKeys() {
+			if dh.Entries[i].Type == FullType || dh.Entries[i].Type == RelativeType {
+				if keyval.Keyword() == "type" && keyval.Value() != "dir" {
+					t.Errorf("expected only directories, but %q is a %q", dh.Entries[i].Name, keyval.Value())
+				}
+			}
+		}
+	}
+}