tar: create and validate a manifest from a tar stream

This commit contains added features to go-mtree that allows
user to create an mtree spec with '-T' option when specifying
a tar archive. Users can also validate an mtree spec against
a tar archive with an mtree spec. Also for the test archive,
there is a mixture of files, and folders (empty & non-empty),
and symlinks (broken & unbroken).

Signed-off-by: Stephen Chung <schung@redhat.com>

tar.go

@@ -4,19 +4,19 @@ import (
 	"archive/tar"
 	"io"
 	"io/ioutil"
-	"log"
 	"os"
 	"path/filepath"
 	"strings"
 )
 
-// Streamer interface that wraps an io.ReadCloser with a function that will
-// return it's Hierarchy
+// Streamer creates a file hierarchy out of a tar stream
 type Streamer interface {
 	io.ReadCloser
 	Hierarchy() (*DirectoryHierarchy, error)
 }
 
+var tarDefaultSetKeywords = []string{"type=file", "flags=none", "mode=0664"}
+
 // NewTarStreamer streams a tar archive and creates a file hierarchy based off
 // of the tar metadata headers
 func NewTarStreamer(r io.Reader, keywords []string) Streamer {
@@ -46,15 +46,28 @@ type tarStream struct {
 func (ts *tarStream) readHeaders() {
 	// We have to start with the directory we're in, and anything beyond these
 	// items is determined at the time a tar is extracted.
-	e := Entry{
-		Name:     ".",
-		Keywords: []string{"size=0", "type=dir"},
+	rootComment := Entry{
+		Raw:  "# .",
+		Type: CommentType,
+	}
+	root := Entry{
+		Name: ".",
+		Type: RelativeType,
+		Prev: &rootComment,
+		Set: &Entry{
+			Name: "meta-set",
+			Type: SpecialType,
+		},
+	}
+	metadataEntries := signatureEntries("<user specified tar archive>")
+	for _, e := range metadataEntries {
+		e.Pos = len(ts.creator.DH.Entries)
+		ts.creator.DH.Entries = append(ts.creator.DH.Entries, e)
 	}
-	ts.creator.curDir = &e
-	ts.creator.DH.Entries = append(ts.creator.DH.Entries, e)
 	for {
 		hdr, err := ts.tarReader.Next()
 		if err != nil {
+			flatten(&root, ts)
 			ts.pipeReader.CloseWithError(err)
 			return
 		}
@@ -84,17 +97,30 @@ func (ts *tarStream) readHeaders() {
 		// Alright, it's either file or directory
 		e := Entry{
 			Name: filepath.Base(hdr.Name),
-			Pos:  len(ts.creator.DH.Entries),
 			Type: RelativeType,
 		}
+
 		// now collect keywords on the file
 		for _, keyword := range ts.keywords {
 			if keyFunc, ok := KeywordFuncs[keyword]; ok {
+				// We can't extract directories on to disk, so "size" keyword
+				// is irrelevant for now
+				if hdr.FileInfo().IsDir() && keyword == "size" {
+					continue
+				}
+
+				if string(hdr.Typeflag) == string('1') {
+					// TODO: get number of hardlinks for a file
+				}
 				val, err := keyFunc(hdr.Name, hdr.FileInfo(), tmpFile)
 				if err != nil {
 					ts.setErr(err)
 				}
-				e.Keywords = append(e.Keywords, val)
+
+				// for good measure, check that we actually get a value for a keyword
+				if val != "" {
+					e.Keywords = append(e.Keywords, val)
+				}
 
 				// don't forget to reset the reader
 				if _, err := tmpFile.Seek(0, 0); err != nil {
@@ -105,39 +131,40 @@ func (ts *tarStream) readHeaders() {
 				}
 			}
 		}
+
+		// collect meta-set keywords for a directory so that we can build the
+		// actual sets in `flatten`
+		if hdr.FileInfo().IsDir() {
+			s := Entry{
+				Name: "meta-set",
+				Type: SpecialType,
+			}
+			for _, setKW := range SetKeywords {
+				if keyFunc, ok := KeywordFuncs[setKW]; ok {
+					val, err := keyFunc(hdr.Name, hdr.FileInfo(), tmpFile)
+					if err != nil {
+						ts.setErr(err)
+					}
+					if val != "" {
+						s.Keywords = append(s.Keywords, val)
+					}
+					if _, err := tmpFile.Seek(0, 0); err != nil {
+						tmpFile.Close()
+						os.Remove(tmpFile.Name())
+						ts.pipeReader.CloseWithError(err)
+					}
+				}
+			}
+			if filepath.Dir(filepath.Clean(hdr.Name)) == "." {
+				root.Set = &s
+			} else {
+				e.Set = &s
+			}
+		}
+		populateTree(&root, &e, hdr, ts)
+
 		tmpFile.Close()
 		os.Remove(tmpFile.Name())
-
-		// compare directories, to determine parent of the current entry
-		cd := compareDir(filepath.Dir(hdr.Name), ts.creator.curDir.Path())
-		switch {
-		case cd == sameDir:
-			e.Parent = ts.creator.curDir
-			if e.Parent != nil {
-				e.Parent.Children = append(e.Parent.Children, &e)
-			}
-		case cd == parentDir:
-			e.Parent = ts.creator.curDir.Parent
-			if e.Parent != nil {
-				e.Parent.Children = append(e.Parent.Children, &e)
-			}
-		}
-
-		if hdr.FileInfo().IsDir() {
-			ts.creator.curDir = &e
-		}
-		// TODO getting the parent child relationship of these entries!
-		if hdr.FileInfo().IsDir() {
-			log.Println(strings.Split(hdr.Name, "/"), strings.Split(ts.creator.curDir.Path(), "/"))
-		}
-
-		ts.creator.DH.Entries = append(ts.creator.DH.Entries, e)
-
-		// Now is the wacky part of building out the entries. Since we can not
-		// control how the archive was assembled, can only take in the order given.
-		// Using `/set` will be tough. Hopefully i can do the directory stepping
-		// with relative paths, but even then I may get a new directory, and not
-		// the files first, but its directories first. :-\
 	}
 }
 
@@ -150,6 +177,167 @@ const (
 	parentDir
 )
 
+// populateTree creates a file tree hierarchy using an Entry's Parent and
+// Children fields. When examining the Entry e to insert in the tree, we
+// determine if the path to that Entry exists yet. If it does, insert it in the
+// appropriate position in the tree. If not, create a path with "placeholder"
+// directories, and then insert the Entry. populateTree does not consider
+// symbolic links yet.
+func populateTree(root, e *Entry, hdr *tar.Header, ts *tarStream) {
+	isDir := hdr.FileInfo().IsDir()
+	wd := filepath.Clean(hdr.Name)
+
+	if !isDir {
+		// If entry is a file, we only want the directory it's in.
+		wd = filepath.Dir(wd)
+	}
+	if filepath.Dir(wd) == "." {
+		if isDir {
+			root.Keywords = e.Keywords
+		} else {
+			root.Children = append([]*Entry{e}, root.Children...)
+			e.Parent = root
+		}
+		return
+	}
+
+	dirNames := strings.Split(wd, "/")
+	parent := root
+	for _, name := range dirNames[1:] {
+		if node := parent.Descend(name); node == nil {
+			// Entry for directory doesn't exist in tree relative to root
+			var newEntry *Entry
+			if isDir {
+				newEntry = e
+			} else {
+				newEntry = &Entry{
+					Name: name,
+					Type: RelativeType,
+				}
+			}
+			newEntry.Parent = parent
+			parent.Children = append(parent.Children, newEntry)
+			parent = newEntry
+		} else {
+			// Entry for directory exists in tree, just keep going
+			parent = node
+		}
+	}
+	if !isDir {
+		parent.Children = append([]*Entry{e}, parent.Children...)
+		e.Parent = parent
+	} else {
+		commentEntry := Entry{
+			Raw:  "# " + e.Path(),
+			Type: CommentType,
+		}
+		e.Prev = &commentEntry
+	}
+}
+
+// After constructing the tree from the tar stream, we want to "flatten" this
+// tree by appending Entry's into ts.creator.DH.Entries in an appropriate
+// manner to simplify writing the output with ts.creator.DH.WriteTo
+// root: the "head" of the sub-tree to flatten
+// ts  : tarStream to keep track of Entry's
+func flatten(root *Entry, ts *tarStream) {
+	if root.Prev != nil {
+		// root.Prev != nil implies root is a directory
+		ts.creator.DH.Entries = append(ts.creator.DH.Entries,
+			Entry{
+				Type: BlankType,
+				Pos:  len(ts.creator.DH.Entries),
+			})
+		root.Prev.Pos = len(ts.creator.DH.Entries)
+		ts.creator.DH.Entries = append(ts.creator.DH.Entries, *root.Prev)
+
+		// Check if we need a new set
+		if ts.creator.curSet == nil {
+			ts.creator.curSet = &Entry{
+				Type:     SpecialType,
+				Name:     "/set",
+				Keywords: keywordSelector(append(tarDefaultSetKeywords, root.Set.Keywords...), ts.keywords),
+				Pos:      len(ts.creator.DH.Entries),
+			}
+			ts.creator.DH.Entries = append(ts.creator.DH.Entries, *ts.creator.curSet)
+		} else {
+			needNewSet := false
+			for _, k := range root.Set.Keywords {
+				if !inSlice(k, ts.creator.curSet.Keywords) {
+					needNewSet = true
+					break
+				}
+			}
+			if needNewSet {
+				ts.creator.curSet = &Entry{
+					Name:     "/set",
+					Type:     SpecialType,
+					Pos:      len(ts.creator.DH.Entries),
+					Keywords: keywordSelector(append(tarDefaultSetKeywords, root.Set.Keywords...), ts.keywords),
+				}
+				ts.creator.DH.Entries = append(ts.creator.DH.Entries, *ts.creator.curSet)
+			}
+		}
+	}
+	root.Set = ts.creator.curSet
+	root.Keywords = setDifference(root.Keywords, ts.creator.curSet.Keywords)
+	root.Pos = len(ts.creator.DH.Entries)
+	ts.creator.DH.Entries = append(ts.creator.DH.Entries, *root)
+
+	for _, c := range root.Children {
+		flatten(c, ts)
+	}
+
+	if root.Prev != nil {
+		// Show a comment when stepping out
+		root.Prev.Pos = len(ts.creator.DH.Entries)
+		ts.creator.DH.Entries = append(ts.creator.DH.Entries, *root.Prev)
+		dotEntry := Entry{
+			Type: DotDotType,
+			Name: "..",
+			Pos:  len(ts.creator.DH.Entries),
+		}
+		ts.creator.DH.Entries = append(ts.creator.DH.Entries, dotEntry)
+	}
+}
+
+// filter takes in a pointer to an Entry, and returns a slice of Entry's that
+// satisfy the predicate p
+func filter(root *Entry, p func(*Entry) bool) []Entry {
+	var validEntrys []Entry
+	if len(root.Children) > 0 || root.Prev != nil {
+		for _, c := range root.Children {
+			// if an Entry is a directory, filter the directory
+			if c.Prev != nil {
+				validEntrys = append(validEntrys, filter(c, p)...)
+			}
+			if p(c) {
+				if c.Prev == nil {
+					// prepend directories
+					validEntrys = append([]Entry{*c}, validEntrys...)
+				} else {
+					validEntrys = append(validEntrys, *c)
+				}
+			}
+		}
+		return validEntrys
+	}
+	return nil
+}
+
+func setDifference(this, that []string) []string {
+	if len(this) == 0 {
+		return that
+	}
+	diff := []string{}
+	for _, kv := range this {
+		if !inSlice(kv, that) {
+			diff = append(diff, kv)
+		}
+	}
+	return diff
+}
+
 func compareDir(curDir, prevDir string) relationship {
 	curDir = filepath.Clean(curDir)
 	prevDir = filepath.Clean(prevDir)