From d1ddeb8e3d6efd1a3557458a698a7b5221ec5e00 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 17 Mar 2016 17:16:46 -0400 Subject: [PATCH] *: close to producing a validating hierarchy Signed-off-by: Vincent Batts --- hierarchy.go | 11 +++- keywords.go | 3 +- walk.go | 181 +++++++++++++++++++++++++++++++++++++++++++++++---- walk_test.go | 23 ++++++- 4 files changed, 200 insertions(+), 18 deletions(-) diff --git a/hierarchy.go b/hierarchy.go index 09d06d7..e81ca3c 100644 --- a/hierarchy.go +++ b/hierarchy.go @@ -46,8 +46,17 @@ func (e Entry) String() string { if e.Raw != "" { return e.Raw } + if e.Type == BlankType { + return "" + } + if e.Type == DotDotType { + return e.Name + } // TODO(vbatts) if type is RelativeType and a keyword of not type=dir - return fmt.Sprintf("%s %s", e.Name, strings.Join(e.Keywords, " ")) + if e.Type == SpecialType || e.Type == FullType || inSlice("type=dir", e.Keywords) { + return fmt.Sprintf("%s %s", e.Name, strings.Join(e.Keywords, " ")) + } + return fmt.Sprintf(" %s %s", e.Name, strings.Join(e.Keywords, " ")) } // EntryType are the formats of lines in an mtree spec file diff --git a/keywords.go b/keywords.go index 1cfc4ac..aab084a 100644 --- a/keywords.go +++ b/keywords.go @@ -35,7 +35,6 @@ var ( SetKeywords = []string{ "uid", "gid", - "mode", } // KeywordFuncs is the map of all keywords (and the functions to produce them) KeywordFuncs = map[string]KeywordFunc{ @@ -73,7 +72,7 @@ var ( var ( modeKeywordFunc = func(path string, info os.FileInfo) (string, error) { - return fmt.Sprintf("mode=%#o", info.Mode()), nil + return fmt.Sprintf("mode=%#o", info.Mode().Perm()), nil } sizeKeywordFunc = func(path string, info os.FileInfo) (string, error) { return fmt.Sprintf("size=%d", info.Size()), nil diff --git a/walk.go b/walk.go index 1964d27..c598cd1 100644 --- a/walk.go +++ b/walk.go @@ -3,6 +3,7 @@ package mtree import ( "os" "path/filepath" + "sort" ) // ExcludeFunc is the type of function called on each path walked to determine @@ -10,14 +11,20 @@ import ( // returns true, then the path is not included in the spec. type ExcludeFunc func(path string, info os.FileInfo) bool +type dhCreator struct { + DH *DirectoryHierarchy + curSet []string + depth int + curDirNum int +} + // // To be able to do a "walk" that produces an outcome with `/set ...` would // need a more linear walk, which this can not ensure. func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHierarchy, error) { - dh := DirectoryHierarchy{} - count := 0 - // TODO insert signature and metadata comments first - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + creator := dhCreator{DH: &DirectoryHierarchy{}} + // TODO insert signature and metadata comments first (user, machine, tree, date) + err := startWalk(&creator, root, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -26,20 +33,170 @@ func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHie return nil } } - e := Entry{} - //e.Name = filepath.Base(path) - e.Name = path - e.Pos = count + + // handle the /set SpecialType + if info.IsDir() { + if len(creator.curSet) == 0 { + // set the initial /set keywords + e := Entry{ + Name: "/set", + Type: SpecialType, + Pos: len(creator.DH.Entries), + Keywords: []string{"type=file", "nlink=1", "flags=none", "mode=0664"}, + } + for _, keyword := range SetKeywords { + if str, err := KeywordFuncs[keyword](path, info); err == nil && str != "" { + e.Keywords = append(e.Keywords, str) + } else if err != nil { + return err + } + } + creator.curSet = e.Keywords + creator.DH.Entries = append(creator.DH.Entries, e) + } else if len(creator.curSet) > 0 { + // check the attributes of the /set keywords and re-set if changed + klist := []string{} + for _, keyword := range SetKeywords { + if str, err := KeywordFuncs[keyword](path, info); err == nil && str != "" { + klist = append(klist, str) + } else if err != nil { + return err + } + } + + needNewSet := false + for _, k := range klist { + if !inSlice(k, creator.curSet) { + needNewSet = true + } + } + + if needNewSet { + creator.curSet = append([]string{"type=file", "nlink=1", "flags=none", "mode=0664"}, klist...) + creator.DH.Entries = append(creator.DH.Entries, Entry{ + Name: "/set", + Type: SpecialType, + Pos: len(creator.DH.Entries), + Keywords: creator.curSet, + }) + } else { + creator.DH.Entries = append(creator.DH.Entries, Entry{ + Type: BlankType, + Pos: len(creator.DH.Entries), + }) + } + } + } + + e := Entry{ + Name: filepath.Base(path), + Pos: len(creator.DH.Entries), + } for _, keyword := range keywords { if str, err := KeywordFuncs[keyword](path, info); err == nil && str != "" { - e.Keywords = append(e.Keywords, str) + if !inSlice(str, creator.curSet) { + e.Keywords = append(e.Keywords, str) + } } else if err != nil { return err } } - dh.Entries = append(dh.Entries, e) - count++ + creator.DH.Entries = append(creator.DH.Entries, e) return nil }) - return &dh, err + return creator.DH, err +} + +func inSlice(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} + +// startWalk walks the file tree rooted at root, calling walkFn for each file or +// directory in the tree, including root. All errors that arise visiting files +// and directories are filtered by walkFn. The files are walked in lexical +// order, which makes the output deterministic but means that for very +// large directories Walk can be inefficient. +// Walk does not follow symbolic links. +func startWalk(c *dhCreator, root string, walkFn filepath.WalkFunc) error { + info, err := os.Lstat(root) + if err != nil { + return walkFn(root, nil, err) + } + return walk(c, root, info, walkFn) +} + +// walk recursively descends path, calling w. +func walk(c *dhCreator, path string, info os.FileInfo, walkFn filepath.WalkFunc) error { + err := walkFn(path, info, nil) + if err != nil { + if info.IsDir() && err == filepath.SkipDir { + return nil + } + return err + } + + if !info.IsDir() { + return nil + } + + names, err := readOrderedDirNames(path) + if err != nil { + return walkFn(path, info, err) + } + + for _, name := range names { + filename := filepath.Join(path, name) + fileInfo, err := os.Lstat(filename) + if err != nil { + if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { + return err + } + } else { + err = walk(c, filename, fileInfo, walkFn) + if err != nil { + if !fileInfo.IsDir() || err != filepath.SkipDir { + return err + } + } + } + } + c.DH.Entries = append(c.DH.Entries, Entry{ + Name: "..", + Type: DotDotType, + Pos: len(c.DH.Entries), + }) + return nil +} + +// readOrderedDirNames reads the directory and returns a sorted list of all +// entries with non-directories first, followed by directories. +func readOrderedDirNames(dirname string) ([]string, error) { + f, err := os.Open(dirname) + if err != nil { + return nil, err + } + infos, err := f.Readdir(-1) + f.Close() + if err != nil { + return nil, err + } + + names := []string{} + dirnames := []string{} + for _, info := range infos { + if info.IsDir() { + dirnames = append(dirnames, info.Name()) + continue + } + names = append(names, info.Name()) + } + sort.Strings(names) + sort.Strings(dirnames) + return append(names, dirnames...), nil + } diff --git a/walk_test.go b/walk_test.go index 7d3f59f..258bbd5 100644 --- a/walk_test.go +++ b/walk_test.go @@ -1,16 +1,33 @@ package mtree import ( - "os" + "io/ioutil" "testing" ) func TestWalk(t *testing.T) { - dh, err := Walk(".", nil, append(DefaultKeywords, "xattr")) + dh, err := Walk(".", nil, append(DefaultKeywords, "sha1")) if err != nil { t.Fatal(err) } - if _, err = dh.WriteTo(os.Stdout); err != nil { + + fh, err := ioutil.TempFile("", "walk.") + if err != nil { + t.Fatal(err) + } + + if _, err = dh.WriteTo(fh); err != nil { t.Error(err) } + fh.Close() + t.Fatal(fh.Name()) + //os.Remove(fh.Name()) +} + +func TestReadNames(t *testing.T) { + names, err := readOrderedDirNames(".") + if err != nil { + t.Error(err) + } + t.Errorf("names: %q", names) }