From 455edf6d2137b8e1db96f3f80daf0230c3530c95 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Wed, 16 Mar 2016 15:59:34 -0400 Subject: [PATCH] *: loads of walking features For the most part, all the keywords for a standard mtree spec now have a function to produce the contents for a creator. These are used in the "walk" function, and will be used next in the "check" logic. This is still a WIP, as the DirectoryHierarchy produced from the current Walk() is not all-together a valid document. Signed-off-by: Vincent Batts --- check.go | 8 ++ cksum.go | 48 ++++++++++++ cksum_test.go | 30 ++++++++ parse.go | 1 + walk.go | 176 ++++++++++++++++++++++++++++++++++++++++++++ walk_linux.go | 33 +++++++++ walk_test.go | 16 ++++ walk_unsupported.go | 20 +++++ 8 files changed, 332 insertions(+) create mode 100644 check.go create mode 100644 cksum.go create mode 100644 cksum_test.go create mode 100644 walk.go create mode 100644 walk_linux.go create mode 100644 walk_test.go create mode 100644 walk_unsupported.go diff --git a/check.go b/check.go new file mode 100644 index 0000000..4664191 --- /dev/null +++ b/check.go @@ -0,0 +1,8 @@ +package mtree + +type Result struct { +} + +func Check(root string, dh *DirectoryHierarchy) (*Result, error) { + return nil, nil +} diff --git a/cksum.go b/cksum.go new file mode 100644 index 0000000..bf5c76c --- /dev/null +++ b/cksum.go @@ -0,0 +1,48 @@ +package mtree + +import ( + "bufio" + "io" +) + +const posixPolynomial uint32 = 0x04C11DB7 + +func cksum(r io.Reader) (uint32, int, error) { + in := bufio.NewReader(r) + count := 0 + var sum uint32 = 0 + f := func(b byte) { + for i := 7; i >= 0; i-- { + msb := sum & (1 << 31) + sum = sum << 1 + if msb != 0 { + sum = sum ^ posixPolynomial + } + } + sum ^= uint32(b) + } + + for done := false; !done; { + switch b, err := in.ReadByte(); err { + case io.EOF: + done = true + case nil: + f(b) + count++ + default: + return ^sum, count, err + } + } + for m := count; ; { + f(byte(m) & 0xff) + m = m >> 8 + if m == 0 { + break + } + } + f(0) + f(0) + f(0) + f(0) + return ^sum, count, nil +} diff --git a/cksum_test.go b/cksum_test.go new file mode 100644 index 0000000..5402c38 --- /dev/null +++ b/cksum_test.go @@ -0,0 +1,30 @@ +package mtree + +import ( + "os" + "testing" +) + +var ( + checkFile = "./testdata/source.mtree" + checkSum uint32 = 1048442895 + checkSize = 9110 +) + +func TestCksum(t *testing.T) { + fh, err := os.Open(checkFile) + if err != nil { + t.Fatal(err) + } + defer fh.Close() + sum, i, err := cksum(fh) + if err != nil { + t.Fatal(err) + } + if i != checkSize { + t.Errorf("%q: expected size %d, got %d", checkFile, checkSize, i) + } + if sum != checkSum { + t.Errorf("%q: expected sum %d, got %d", checkFile, checkSum, sum) + } +} diff --git a/parse.go b/parse.go index 9d3b25e..accf204 100644 --- a/parse.go +++ b/parse.go @@ -6,6 +6,7 @@ import ( "strings" ) +// ParseSpec reads a stream of an mtree specification, and returns the DirectoryHierarchy func ParseSpec(r io.Reader) (*DirectoryHierarchy, error) { s := bufio.NewScanner(r) i := int(0) diff --git a/walk.go b/walk.go new file mode 100644 index 0000000..da20b69 --- /dev/null +++ b/walk.go @@ -0,0 +1,176 @@ +package mtree + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "fmt" + "hash" + "io" + "math" + "os" + "path/filepath" + + "golang.org/x/crypto/ripemd160" +) + +// ExcludeFunc is the type of function called on each path walked to determine +// whether to be excluded from the assembled DirectoryHierarchy. If the func +// returns true, then the path is not included in the spec. +type ExcludeFunc func(path string, info os.FileInfo) bool + +// KeywordFunc is the type of a function called on each file to be included in +// a DirectoryHierarchy, that will produce the string output of the keyword to +// be included for the file entry. Otherwise, empty string. +type KeywordFunc func(path string, info os.FileInfo) (string, error) + +var ( + // DefaultKeywords has the several default keyword producers (uid, gid, + // mode, nlink, type, size, mtime) + DefaultKeywords = []string{ + "size", + "type", + "uid", + "gid", + "link", + "nlink", + "time", + } + // KeywordFuncs is the map of all keywords (and the functions to produce them) + KeywordFuncs = map[string]KeywordFunc{ + "size": sizeKeywordFunc, // The size, in bytes, of the file + "type": typeKeywordFunc, // The type of the file + "time": timeKeywordFunc, // The last modification time of the file + "link": linkKeywordFunc, // The target of the symbolic link when type=link + "uid": uidKeywordFunc, // The file owner as a numeric value + "gid": gidKeywordFunc, // The file group as a numeric value + "nlink": nlinkKeywordFunc, // The number of hard links the file is expected to have + "uname": unameKeywordFunc, // The file owner as a symbolic name + "cksum": cksumKeywordFunc, // The checksum of the file using the default algorithm specified by the cksum(1) utility + "md5": hasherKeywordFunc("md5", md5.New), // The MD5 message digest of the file + "md5digest": hasherKeywordFunc("md5digest", md5.New), // A synonym for `md5` + "rmd160": hasherKeywordFunc("rmd160", ripemd160.New), // The RIPEMD160 message digest of the file + "rmd160digest": hasherKeywordFunc("rmd160digest", ripemd160.New), // A synonym for `rmd160` + "ripemd160digest": hasherKeywordFunc("ripemd160digest", ripemd160.New), // A synonym for `rmd160` + "sha1": hasherKeywordFunc("sha1", sha1.New), // The SHA1 message digest of the file + "sha1digest": hasherKeywordFunc("sha1digest", sha1.New), // A synonym for `sha1` + "sha256": hasherKeywordFunc("sha256", sha256.New), // The SHA256 message digest of the file + "sha256digest": hasherKeywordFunc("sha256digest", sha256.New), // A synonym for `sha256` + "sha384": hasherKeywordFunc("sha384", sha512.New384), // The SHA384 message digest of the file + "sha384digest": hasherKeywordFunc("sha384digest", sha512.New384), // A synonym for `sha384` + "sha512": hasherKeywordFunc("sha512", sha512.New), // The SHA512 message digest of the file + "sha512digest": hasherKeywordFunc("sha512digest", sha512.New), // A synonym for `sha512` + } +) + +var ( + sizeKeywordFunc = func(path string, info os.FileInfo) (string, error) { + return fmt.Sprintf("size=%d", info.Size()), nil + } + cksumKeywordFunc = func(path string, info os.FileInfo) (string, error) { + if !info.Mode().IsRegular() { + return "", nil + } + + fh, err := os.Open(path) + if err != nil { + return "", err + } + defer fh.Close() + sum, _, err := cksum(fh) + if err != nil { + return "", err + } + return fmt.Sprintf("cksum=%d", sum), nil + } + hasherKeywordFunc = func(name string, newHash func() hash.Hash) KeywordFunc { + return func(path string, info os.FileInfo) (string, error) { + if !info.Mode().IsRegular() { + return "", nil + } + + fh, err := os.Open(path) + if err != nil { + return "", err + } + defer fh.Close() + + h := newHash() + if _, err := io.Copy(h, fh); err != nil { + return "", err + } + return fmt.Sprintf("%s=%x", name, h.Sum(nil)), nil + } + } + timeKeywordFunc = func(path string, info os.FileInfo) (string, error) { + t := info.ModTime() + n := float64(t.UnixNano()) / math.Pow10(9) + return fmt.Sprintf("time=%0.9f", n), nil + } + linkKeywordFunc = func(path string, info os.FileInfo) (string, error) { + if info.Mode()&os.ModeSymlink != 0 { + str, err := os.Readlink(path) + if err != nil { + return "", err + } + return fmt.Sprintf("link=%s", str), nil + } + return "", nil + } + typeKeywordFunc = func(path string, info os.FileInfo) (string, error) { + if info.Mode().IsDir() { + return "type=dir", nil + } + if info.Mode().IsRegular() { + return "type=file", nil + } + if info.Mode()&os.ModeSocket != 0 { + return "type=socket", nil + } + if info.Mode()&os.ModeSymlink != 0 { + return "type=link", nil + } + if info.Mode()&os.ModeNamedPipe != 0 { + return "type=fifo", nil + } + if info.Mode()&os.ModeDevice != 0 { + if info.Mode()&os.ModeCharDevice != 0 { + return "type=char", nil + } + return "type=device", nil + } + return "", nil + } +) + +// +// To be able to do a "walk" that produces an outcome with `/set ...` would +// need a more linear walk, which this can not ensure. +func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHierarchy, error) { + dh := DirectoryHierarchy{} + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + for _, ex := range exlcudes { + if ex(path, info) { + return nil + } + } + e := Entry{} + //e.Name = filepath.Base(path) + e.Name = path + for _, keyword := range keywords { + if str, err := KeywordFuncs[keyword](path, info); err == nil && str != "" { + e.Keywords = append(e.Keywords, str) + } else if err != nil { + return err + } + } + // XXX + dh.Entries = append(dh.Entries, e) + return nil + }) + return &dh, err +} diff --git a/walk_linux.go b/walk_linux.go new file mode 100644 index 0000000..a52b369 --- /dev/null +++ b/walk_linux.go @@ -0,0 +1,33 @@ +// +build linux + +package mtree + +import ( + "fmt" + "os" + "os/user" + "syscall" +) + +var ( + unameKeywordFunc = func(path string, info os.FileInfo) (string, error) { + stat := info.Sys().(*syscall.Stat_t) + u, err := user.LookupId(fmt.Sprintf("%d", stat.Uid)) + if err != nil { + return "", err + } + return fmt.Sprintf("uname=%s", u.Username), nil + } + uidKeywordFunc = func(path string, info os.FileInfo) (string, error) { + stat := info.Sys().(*syscall.Stat_t) + return fmt.Sprintf("uid=%d", stat.Uid), nil + } + gidKeywordFunc = func(path string, info os.FileInfo) (string, error) { + stat := info.Sys().(*syscall.Stat_t) + return fmt.Sprintf("gid=%d", stat.Gid), nil + } + nlinkKeywordFunc = func(path string, info os.FileInfo) (string, error) { + stat := info.Sys().(*syscall.Stat_t) + return fmt.Sprintf("nlink=%d", stat.Nlink), nil + } +) diff --git a/walk_test.go b/walk_test.go new file mode 100644 index 0000000..9a60e51 --- /dev/null +++ b/walk_test.go @@ -0,0 +1,16 @@ +package mtree + +import ( + "os" + "testing" +) + +func TestWalk(t *testing.T) { + dh, err := Walk(".", nil, append(DefaultKeywords, "cksum", "md5", "rmd160digest", "sha1", "sha256", "sha512")) + if err != nil { + t.Fatal(err) + } + if _, err = dh.WriteTo(os.Stdout); err != nil { + t.Error(err) + } +} diff --git a/walk_unsupported.go b/walk_unsupported.go new file mode 100644 index 0000000..88b92a5 --- /dev/null +++ b/walk_unsupported.go @@ -0,0 +1,20 @@ +// +build !linux + +package mtree + +import "os" + +var ( + unameKeywordFunc = func(path string, info os.FileInfo) (string, error) { + return "", nil + } + uidKeywordFunc = func(path string, info os.FileInfo) (string, error) { + return "", nil + } + gidKeywordFunc = func(path string, info os.FileInfo) (string, error) { + return "", nil + } + nlinkKeywordFunc = func(path string, info os.FileInfo) (string, error) { + return "", nil + } +)