*: close to producing a validating hierarchy

Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>

hierarchy.go

@@ -46,8 +46,17 @@ func (e Entry) String() string {
 	if e.Raw != "" {
 		return e.Raw
 	}
+	if e.Type == BlankType {
+		return ""
+	}
+	if e.Type == DotDotType {
+		return e.Name
+	}
 	// TODO(vbatts) if type is RelativeType and a keyword of not type=dir
-	return fmt.Sprintf("%s %s", e.Name, strings.Join(e.Keywords, " "))
+	if e.Type == SpecialType || e.Type == FullType || inSlice("type=dir", e.Keywords) {
+		return fmt.Sprintf("%s %s", e.Name, strings.Join(e.Keywords, " "))
+	}
+	return fmt.Sprintf("    %s %s", e.Name, strings.Join(e.Keywords, " "))
 }
 
 // EntryType are the formats of lines in an mtree spec file

keywords.go

@@ -35,7 +35,6 @@ var (
 	SetKeywords = []string{
 		"uid",
 		"gid",
-		"mode",
 	}
 	// KeywordFuncs is the map of all keywords (and the functions to produce them)
 	KeywordFuncs = map[string]KeywordFunc{
@@ -73,7 +72,7 @@ var (
 
 var (
 	modeKeywordFunc = func(path string, info os.FileInfo) (string, error) {
-		return fmt.Sprintf("mode=%#o", info.Mode()), nil
+		return fmt.Sprintf("mode=%#o", info.Mode().Perm()), nil
 	}
 	sizeKeywordFunc = func(path string, info os.FileInfo) (string, error) {
 		return fmt.Sprintf("size=%d", info.Size()), nil

walk.go

@@ -3,6 +3,7 @@ package mtree
 import (
 	"os"
 	"path/filepath"
+	"sort"
 )
 
 // ExcludeFunc is the type of function called on each path walked to determine
@@ -10,14 +11,20 @@ import (
 // returns true, then the path is not included in the spec.
 type ExcludeFunc func(path string, info os.FileInfo) bool
 
+type dhCreator struct {
+	DH        *DirectoryHierarchy
+	curSet    []string
+	depth     int
+	curDirNum int
+}
+
 //
 // To be able to do a "walk" that produces an outcome with `/set ...` would
 // need a more linear walk, which this can not ensure.
 func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHierarchy, error) {
-	dh := DirectoryHierarchy{}
+	creator := dhCreator{DH: &DirectoryHierarchy{}}
-	count := 0
+	// TODO insert signature and metadata comments first (user, machine, tree, date)
-	// TODO insert signature and metadata comments first
+	err := startWalk(&creator, root, func(path string, info os.FileInfo, err error) error {
-	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -26,20 +33,170 @@ func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHie
 				return nil
 			}
 		}
-		e := Entry{}
+
-		//e.Name = filepath.Base(path)
+		// handle the /set SpecialType
-		e.Name = path
+		if info.IsDir() {
-		e.Pos = count
+			if len(creator.curSet) == 0 {
+				// set the initial /set keywords
+				e := Entry{
+					Name:     "/set",
+					Type:     SpecialType,
+					Pos:      len(creator.DH.Entries),
+					Keywords: []string{"type=file", "nlink=1", "flags=none", "mode=0664"},
+				}
+				for _, keyword := range SetKeywords {
+					if str, err := KeywordFuncs[keyword](path, info); err == nil && str != "" {
+						e.Keywords = append(e.Keywords, str)
+					} else if err != nil {
+						return err
+					}
+				}
+				creator.curSet = e.Keywords
+				creator.DH.Entries = append(creator.DH.Entries, e)
+			} else if len(creator.curSet) > 0 {
+				// check the attributes of the /set keywords and re-set if changed
+				klist := []string{}
+				for _, keyword := range SetKeywords {
+					if str, err := KeywordFuncs[keyword](path, info); err == nil && str != "" {
+						klist = append(klist, str)
+					} else if err != nil {
+						return err
+					}
+				}
+
+				needNewSet := false
+				for _, k := range klist {
+					if !inSlice(k, creator.curSet) {
+						needNewSet = true
+					}
+				}
+
+				if needNewSet {
+					creator.curSet = append([]string{"type=file", "nlink=1", "flags=none", "mode=0664"}, klist...)
+					creator.DH.Entries = append(creator.DH.Entries, Entry{
+						Name:     "/set",
+						Type:     SpecialType,
+						Pos:      len(creator.DH.Entries),
+						Keywords: creator.curSet,
+					})
+				} else {
+					creator.DH.Entries = append(creator.DH.Entries, Entry{
+						Type: BlankType,
+						Pos:  len(creator.DH.Entries),
+					})
+				}
+			}
+		}
+
+		e := Entry{
+			Name: filepath.Base(path),
+			Pos:  len(creator.DH.Entries),
+		}
 		for _, keyword := range keywords {
 			if str, err := KeywordFuncs[keyword](path, info); err == nil && str != "" {
-				e.Keywords = append(e.Keywords, str)
+				if !inSlice(str, creator.curSet) {
+					e.Keywords = append(e.Keywords, str)
+				}
 			} else if err != nil {
 				return err
 			}
 		}
-		dh.Entries = append(dh.Entries, e)
+		creator.DH.Entries = append(creator.DH.Entries, e)
-		count++
 		return nil
 	})
-	return &dh, err
+	return creator.DH, err
+}
+
+func inSlice(a string, list []string) bool {
+	for _, b := range list {
+		if b == a {
+			return true
+		}
+	}
+	return false
+}
+
+// startWalk walks the file tree rooted at root, calling walkFn for each file or
+// directory in the tree, including root. All errors that arise visiting files
+// and directories are filtered by walkFn. The files are walked in lexical
+// order, which makes the output deterministic but means that for very
+// large directories Walk can be inefficient.
+// Walk does not follow symbolic links.
+func startWalk(c *dhCreator, root string, walkFn filepath.WalkFunc) error {
+	info, err := os.Lstat(root)
+	if err != nil {
+		return walkFn(root, nil, err)
+	}
+	return walk(c, root, info, walkFn)
+}
+
+// walk recursively descends path, calling w.
+func walk(c *dhCreator, path string, info os.FileInfo, walkFn filepath.WalkFunc) error {
+	err := walkFn(path, info, nil)
+	if err != nil {
+		if info.IsDir() && err == filepath.SkipDir {
+			return nil
+		}
+		return err
+	}
+
+	if !info.IsDir() {
+		return nil
+	}
+
+	names, err := readOrderedDirNames(path)
+	if err != nil {
+		return walkFn(path, info, err)
+	}
+
+	for _, name := range names {
+		filename := filepath.Join(path, name)
+		fileInfo, err := os.Lstat(filename)
+		if err != nil {
+			if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir {
+				return err
+			}
+		} else {
+			err = walk(c, filename, fileInfo, walkFn)
+			if err != nil {
+				if !fileInfo.IsDir() || err != filepath.SkipDir {
+					return err
+				}
+			}
+		}
+	}
+	c.DH.Entries = append(c.DH.Entries, Entry{
+		Name: "..",
+		Type: DotDotType,
+		Pos:  len(c.DH.Entries),
+	})
+	return nil
+}
+
+// readOrderedDirNames reads the directory and returns a sorted list of all
+// entries with non-directories first, followed by directories.
+func readOrderedDirNames(dirname string) ([]string, error) {
+	f, err := os.Open(dirname)
+	if err != nil {
+		return nil, err
+	}
+	infos, err := f.Readdir(-1)
+	f.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	names := []string{}
+	dirnames := []string{}
+	for _, info := range infos {
+		if info.IsDir() {
+			dirnames = append(dirnames, info.Name())
+			continue
+		}
+		names = append(names, info.Name())
+	}
+	sort.Strings(names)
+	sort.Strings(dirnames)
+	return append(names, dirnames...), nil
+
 }

walk_test.go

@@ -1,16 +1,33 @@
 package mtree
 
 import (
-	"os"
+	"io/ioutil"
 	"testing"
 )
 
 func TestWalk(t *testing.T) {
-	dh, err := Walk(".", nil, append(DefaultKeywords, "xattr"))
+	dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"))
 	if err != nil {
 		t.Fatal(err)
 	}
-	if _, err = dh.WriteTo(os.Stdout); err != nil {
+
+	fh, err := ioutil.TempFile("", "walk.")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = dh.WriteTo(fh); err != nil {
 		t.Error(err)
 	}
+	fh.Close()
+	t.Fatal(fh.Name())
+	//os.Remove(fh.Name())
+}
+
+func TestReadNames(t *testing.T) {
+	names, err := readOrderedDirNames(".")
+	if err != nil {
+		t.Error(err)
+	}
+	t.Errorf("names: %q", names)
 }