*: loads of walking features

For the most part, all the keywords for a standard mtree spec now have a
function to produce the contents for a creator.

These are used in the "walk" function, and will be used next in the
"check" logic.

This is still a WIP, as the DirectoryHierarchy produced from the current
Walk() is not all-together a valid document.

Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>

check.go

@@ -0,0 +1,8 @@
+package mtree
+
+type Result struct {
+}
+
+func Check(root string, dh *DirectoryHierarchy) (*Result, error) {
+	return nil, nil
+}

cksum.go

@@ -0,0 +1,48 @@
+package mtree
+
+import (
+	"bufio"
+	"io"
+)
+
+const posixPolynomial uint32 = 0x04C11DB7
+
+func cksum(r io.Reader) (uint32, int, error) {
+	in := bufio.NewReader(r)
+	count := 0
+	var sum uint32 = 0
+	f := func(b byte) {
+		for i := 7; i >= 0; i-- {
+			msb := sum & (1 << 31)
+			sum = sum << 1
+			if msb != 0 {
+				sum = sum ^ posixPolynomial
+			}
+		}
+		sum ^= uint32(b)
+	}
+
+	for done := false; !done; {
+		switch b, err := in.ReadByte(); err {
+		case io.EOF:
+			done = true
+		case nil:
+			f(b)
+			count++
+		default:
+			return ^sum, count, err
+		}
+	}
+	for m := count; ; {
+		f(byte(m) & 0xff)
+		m = m >> 8
+		if m == 0 {
+			break
+		}
+	}
+	f(0)
+	f(0)
+	f(0)
+	f(0)
+	return ^sum, count, nil
+}

cksum_test.go

@@ -0,0 +1,30 @@
+package mtree
+
+import (
+	"os"
+	"testing"
+)
+
+var (
+	checkFile        = "./testdata/source.mtree"
+	checkSum  uint32 = 1048442895
+	checkSize        = 9110
+)
+
+func TestCksum(t *testing.T) {
+	fh, err := os.Open(checkFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer fh.Close()
+	sum, i, err := cksum(fh)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if i != checkSize {
+		t.Errorf("%q: expected size %d, got %d", checkFile, checkSize, i)
+	}
+	if sum != checkSum {
+		t.Errorf("%q: expected sum %d, got %d", checkFile, checkSum, sum)
+	}
+}

parse.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 )
 
+// ParseSpec reads a stream of an mtree specification, and returns the DirectoryHierarchy
 func ParseSpec(r io.Reader) (*DirectoryHierarchy, error) {
 	s := bufio.NewScanner(r)
 	i := int(0)

walk.go

@@ -0,0 +1,176 @@
+package mtree
+
+import (
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/sha512"
+	"fmt"
+	"hash"
+	"io"
+	"math"
+	"os"
+	"path/filepath"
+
+	"golang.org/x/crypto/ripemd160"
+)
+
+// ExcludeFunc is the type of function called on each path walked to determine
+// whether to be excluded from the assembled DirectoryHierarchy. If the func
+// returns true, then the path is not included in the spec.
+type ExcludeFunc func(path string, info os.FileInfo) bool
+
+// KeywordFunc is the type of a function called on each file to be included in
+// a DirectoryHierarchy, that will produce the string output of the keyword to
+// be included for the file entry. Otherwise, empty string.
+type KeywordFunc func(path string, info os.FileInfo) (string, error)
+
+var (
+	// DefaultKeywords has the several default keyword producers (uid, gid,
+	// mode, nlink, type, size, mtime)
+	DefaultKeywords = []string{
+		"size",
+		"type",
+		"uid",
+		"gid",
+		"link",
+		"nlink",
+		"time",
+	}
+	// KeywordFuncs is the map of all keywords (and the functions to produce them)
+	KeywordFuncs = map[string]KeywordFunc{
+		"size":            sizeKeywordFunc,                                     // The size, in bytes, of the file
+		"type":            typeKeywordFunc,                                     // The type of the file
+		"time":            timeKeywordFunc,                                     // The last modification time of the file
+		"link":            linkKeywordFunc,                                     // The target of the symbolic link when type=link
+		"uid":             uidKeywordFunc,                                      // The file owner as a numeric value
+		"gid":             gidKeywordFunc,                                      // The file group as a numeric value
+		"nlink":           nlinkKeywordFunc,                                    // The number of hard links the file is expected to have
+		"uname":           unameKeywordFunc,                                    // The file owner as a symbolic name
+		"cksum":           cksumKeywordFunc,                                    // The checksum of the file using the default algorithm specified by the cksum(1) utility
+		"md5":             hasherKeywordFunc("md5", md5.New),                   // The MD5 message digest of the file
+		"md5digest":       hasherKeywordFunc("md5digest", md5.New),             // A synonym for `md5`
+		"rmd160":          hasherKeywordFunc("rmd160", ripemd160.New),          // The RIPEMD160 message digest of the file
+		"rmd160digest":    hasherKeywordFunc("rmd160digest", ripemd160.New),    // A synonym for `rmd160`
+		"ripemd160digest": hasherKeywordFunc("ripemd160digest", ripemd160.New), // A synonym for `rmd160`
+		"sha1":            hasherKeywordFunc("sha1", sha1.New),                 // The SHA1 message digest of the file
+		"sha1digest":      hasherKeywordFunc("sha1digest", sha1.New),           // A synonym for `sha1`
+		"sha256":          hasherKeywordFunc("sha256", sha256.New),             // The SHA256 message digest of the file
+		"sha256digest":    hasherKeywordFunc("sha256digest", sha256.New),       // A synonym for `sha256`
+		"sha384":          hasherKeywordFunc("sha384", sha512.New384),          // The SHA384 message digest of the file
+		"sha384digest":    hasherKeywordFunc("sha384digest", sha512.New384),    // A synonym for `sha384`
+		"sha512":          hasherKeywordFunc("sha512", sha512.New),             // The SHA512 message digest of the file
+		"sha512digest":    hasherKeywordFunc("sha512digest", sha512.New),       // A synonym for `sha512`
+	}
+)
+
+var (
+	sizeKeywordFunc = func(path string, info os.FileInfo) (string, error) {
+		return fmt.Sprintf("size=%d", info.Size()), nil
+	}
+	cksumKeywordFunc = func(path string, info os.FileInfo) (string, error) {
+		if !info.Mode().IsRegular() {
+			return "", nil
+		}
+
+		fh, err := os.Open(path)
+		if err != nil {
+			return "", err
+		}
+		defer fh.Close()
+		sum, _, err := cksum(fh)
+		if err != nil {
+			return "", err
+		}
+		return fmt.Sprintf("cksum=%d", sum), nil
+	}
+	hasherKeywordFunc = func(name string, newHash func() hash.Hash) KeywordFunc {
+		return func(path string, info os.FileInfo) (string, error) {
+			if !info.Mode().IsRegular() {
+				return "", nil
+			}
+
+			fh, err := os.Open(path)
+			if err != nil {
+				return "", err
+			}
+			defer fh.Close()
+
+			h := newHash()
+			if _, err := io.Copy(h, fh); err != nil {
+				return "", err
+			}
+			return fmt.Sprintf("%s=%x", name, h.Sum(nil)), nil
+		}
+	}
+	timeKeywordFunc = func(path string, info os.FileInfo) (string, error) {
+		t := info.ModTime()
+		n := float64(t.UnixNano()) / math.Pow10(9)
+		return fmt.Sprintf("time=%0.9f", n), nil
+	}
+	linkKeywordFunc = func(path string, info os.FileInfo) (string, error) {
+		if info.Mode()&os.ModeSymlink != 0 {
+			str, err := os.Readlink(path)
+			if err != nil {
+				return "", err
+			}
+			return fmt.Sprintf("link=%s", str), nil
+		}
+		return "", nil
+	}
+	typeKeywordFunc = func(path string, info os.FileInfo) (string, error) {
+		if info.Mode().IsDir() {
+			return "type=dir", nil
+		}
+		if info.Mode().IsRegular() {
+			return "type=file", nil
+		}
+		if info.Mode()&os.ModeSocket != 0 {
+			return "type=socket", nil
+		}
+		if info.Mode()&os.ModeSymlink != 0 {
+			return "type=link", nil
+		}
+		if info.Mode()&os.ModeNamedPipe != 0 {
+			return "type=fifo", nil
+		}
+		if info.Mode()&os.ModeDevice != 0 {
+			if info.Mode()&os.ModeCharDevice != 0 {
+				return "type=char", nil
+			}
+			return "type=device", nil
+		}
+		return "", nil
+	}
+)
+
+//
+// To be able to do a "walk" that produces an outcome with `/set ...` would
+// need a more linear walk, which this can not ensure.
+func Walk(root string, exlcudes []ExcludeFunc, keywords []string) (*DirectoryHierarchy, error) {
+	dh := DirectoryHierarchy{}
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		for _, ex := range exlcudes {
+			if ex(path, info) {
+				return nil
+			}
+		}
+		e := Entry{}
+		//e.Name = filepath.Base(path)
+		e.Name = path
+		for _, keyword := range keywords {
+			if str, err := KeywordFuncs[keyword](path, info); err == nil && str != "" {
+				e.Keywords = append(e.Keywords, str)
+			} else if err != nil {
+				return err
+			}
+		}
+		// XXX
+		dh.Entries = append(dh.Entries, e)
+		return nil
+	})
+	return &dh, err
+}

walk_linux.go

@@ -0,0 +1,33 @@
+// +build linux
+
+package mtree
+
+import (
+	"fmt"
+	"os"
+	"os/user"
+	"syscall"
+)
+
+var (
+	unameKeywordFunc = func(path string, info os.FileInfo) (string, error) {
+		stat := info.Sys().(*syscall.Stat_t)
+		u, err := user.LookupId(fmt.Sprintf("%d", stat.Uid))
+		if err != nil {
+			return "", err
+		}
+		return fmt.Sprintf("uname=%s", u.Username), nil
+	}
+	uidKeywordFunc = func(path string, info os.FileInfo) (string, error) {
+		stat := info.Sys().(*syscall.Stat_t)
+		return fmt.Sprintf("uid=%d", stat.Uid), nil
+	}
+	gidKeywordFunc = func(path string, info os.FileInfo) (string, error) {
+		stat := info.Sys().(*syscall.Stat_t)
+		return fmt.Sprintf("gid=%d", stat.Gid), nil
+	}
+	nlinkKeywordFunc = func(path string, info os.FileInfo) (string, error) {
+		stat := info.Sys().(*syscall.Stat_t)
+		return fmt.Sprintf("nlink=%d", stat.Nlink), nil
+	}
+)

walk_test.go

@@ -0,0 +1,16 @@
+package mtree
+
+import (
+	"os"
+	"testing"
+)
+
+func TestWalk(t *testing.T) {
+	dh, err := Walk(".", nil, append(DefaultKeywords, "cksum", "md5", "rmd160digest", "sha1", "sha256", "sha512"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err = dh.WriteTo(os.Stdout); err != nil {
+		t.Error(err)
+	}
+}

walk_unsupported.go

@@ -0,0 +1,20 @@
+// +build !linux
+
+package mtree
+
+import "os"
+
+var (
+	unameKeywordFunc = func(path string, info os.FileInfo) (string, error) {
+		return "", nil
+	}
+	uidKeywordFunc = func(path string, info os.FileInfo) (string, error) {
+		return "", nil
+	}
+	gidKeywordFunc = func(path string, info os.FileInfo) (string, error) {
+		return "", nil
+	}
+	nlinkKeywordFunc = func(path string, info os.FileInfo) (string, error) {
+		return "", nil
+	}
+)