vbatts/go-mtree
1
0
Fork 0
Code Issues Pull requests Releases Activity

Merge pull request #135 from vbatts/xattr-updates

Browse source 
*: xattr can Update()
This commit is contained in:
Vincent Batts 2017-06-26 14:14:00 -04:00 committed by GitHub
commit 0b5038d0bc
19 changed files with 398 additions and 258 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
check_test.go
View file

@ -12,7 +12,7 @@ import (
// simple walk of current directory, and imediately check it.
// may not be parallelizable.
func TestCheck(t *testing.T) {
dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
dh, err := Walk(".", nil, append(DefaultKeywords, []Keyword{"sha1", "xattr"}...), nil)
if err != nil {
t.Fatal(err)
}

5
cmd/gomtree/main.go
View file

@ -7,10 +7,10 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"os"
"strings"
"github.com/Sirupsen/logrus"
"github.com/vbatts/go-mtree"
)
@ -37,7 +37,7 @@ var (
func main() {
// so that defers cleanly exec
if err := app(); err != nil {
log.Fatal(err)
logrus.Fatal(err)
}
}
@ -46,6 +46,7 @@ func app() error {
if *flDebug {
os.Setenv("DEBUG", "1")
logrus.SetLevel(logrus.DebugLevel)
}
if *flVersion {

4
compare.go
View file

@ -197,7 +197,7 @@ func compareEntry(oldEntry, newEntry Entry) ([]KeyDelta, error) {
for _, kv := range oldKeys {
key := kv.Keyword()
// only add this diff if the new keys has this keyword
if key != "tar_time" && key != "time" && key != "xattr" && HasKeyword(newKeys, key) == emptyKV {
if key != "tar_time" && key != "time" && key != "xattr" && len(HasKeyword(newKeys, key)) == 0 {
continue
}
@ -216,7 +216,7 @@ func compareEntry(oldEntry, newEntry Entry) ([]KeyDelta, error) {
for _, kv := range newKeys {
key := kv.Keyword()
// only add this diff if the old keys has this keyword
if key != "tar_time" && key != "time" && key != "xattr" && HasKeyword(oldKeys, key) == emptyKV {
if key != "tar_time" && key != "time" && key != "xattr" && len(HasKeyword(oldKeys, key)) == 0 {
continue
}

26
debug.go
View file

@ -1,26 +0,0 @@
package mtree
import (
"fmt"
"os"
"time"
)
// DebugOutput is the where DEBUG output is written
var DebugOutput = os.Stderr
// Debugln writes output to DebugOutput, only if DEBUG environment variable is set
func Debugln(a ...interface{}) (n int, err error) {
if os.Getenv("DEBUG") != "" {
return fmt.Fprintf(DebugOutput, "[%d] [DEBUG] %s\n", time.Now().UnixNano(), a)
}
return 0, nil
}
// Debugf writes formatted output to DebugOutput, only if DEBUG environment variable is set
func Debugf(format string, a ...interface{}) (n int, err error) {
if os.Getenv("DEBUG") != "" {
return fmt.Fprintf(DebugOutput, "[%d] [DEBUG] %s\n", time.Now().UnixNano(), fmt.Sprintf(format, a...))
}
return 0, nil
}

70
keywordfunc.go
View file

@ -21,7 +21,7 @@ import (
// io.Reader `r` is to the file stream for the file payload. While this
// function takes an io.Reader, the caller needs to reset it to the beginning
// for each new KeywordFunc
type KeywordFunc func(path string, info os.FileInfo, r io.Reader) (KeyVal, error)
type KeywordFunc func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error)
var (
// KeywordFuncs is the map of all keywords (and the functions to produce them)
@ -67,7 +67,7 @@ var (
}
)
var (
modeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
modeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
permissions := info.Mode().Perm()
if os.ModeSetuid&info.Mode() > 0 {
permissions |= (1 << 11)
@ -78,93 +78,93 @@ var (
if os.ModeSticky&info.Mode() > 0 {
permissions |= (1 << 9)
}
return KeyVal(fmt.Sprintf("mode=%#o", permissions)), nil
return []KeyVal{KeyVal(fmt.Sprintf("mode=%#o", permissions))}, nil
}
sizeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
sizeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if sys, ok := info.Sys().(*tar.Header); ok {
if sys.Typeflag == tar.TypeSymlink {
return KeyVal(fmt.Sprintf("size=%d", len(sys.Linkname))), nil
return []KeyVal{KeyVal(fmt.Sprintf("size=%d", len(sys.Linkname)))}, nil
}
}
return KeyVal(fmt.Sprintf("size=%d", info.Size())), nil
return []KeyVal{KeyVal(fmt.Sprintf("size=%d", info.Size()))}, nil
}
cksumKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
cksumKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if !info.Mode().IsRegular() {
return emptyKV, nil
return nil, nil
}
sum, _, err := cksum(r)
if err != nil {
return emptyKV, err
return nil, err
}
return KeyVal(fmt.Sprintf("cksum=%d", sum)), nil
return []KeyVal{KeyVal(fmt.Sprintf("cksum=%d", sum))}, nil
}
hasherKeywordFunc = func(name string, newHash func() hash.Hash) KeywordFunc {
return func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
return func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if !info.Mode().IsRegular() {
return emptyKV, nil
return nil, nil
}
h := newHash()
if _, err := io.Copy(h, r); err != nil {
return emptyKV, err
return nil, err
}
return KeyVal(fmt.Sprintf("%s=%x", KeywordSynonym(name), h.Sum(nil))), nil
return []KeyVal{KeyVal(fmt.Sprintf("%s=%x", KeywordSynonym(name), h.Sum(nil)))}, nil
}
}
tartimeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
return KeyVal(fmt.Sprintf("tar_time=%d.%9.9d", info.ModTime().Unix(), 0)), nil
tartimeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
return []KeyVal{KeyVal(fmt.Sprintf("tar_time=%d.%9.9d", info.ModTime().Unix(), 0))}, nil
}
timeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
timeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
tSec := info.ModTime().Unix()
tNano := info.ModTime().Nanosecond()
return KeyVal(fmt.Sprintf("time=%d.%9.9d", tSec, tNano)), nil
return []KeyVal{KeyVal(fmt.Sprintf("time=%d.%9.9d", tSec, tNano))}, nil
}
linkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
linkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if sys, ok := info.Sys().(*tar.Header); ok {
if sys.Linkname != "" {
linkname, err := govis.Vis(sys.Linkname, DefaultVisFlags)
if err != nil {
return emptyKV, err
return nil, nil
}
return KeyVal(fmt.Sprintf("link=%s", linkname)), nil
return []KeyVal{KeyVal(fmt.Sprintf("link=%s", linkname))}, nil
}
return emptyKV, nil
return nil, nil
}
if info.Mode()&os.ModeSymlink != 0 {
str, err := os.Readlink(path)
if err != nil {
return emptyKV, err
return nil, nil
}
linkname, err := govis.Vis(str, DefaultVisFlags)
if err != nil {
return emptyKV, err
return nil, nil
}
return KeyVal(fmt.Sprintf("link=%s", linkname)), nil
return []KeyVal{KeyVal(fmt.Sprintf("link=%s", linkname))}, nil
}
return emptyKV, nil
return nil, nil
}
typeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
typeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if info.Mode().IsDir() {
return "type=dir", nil
return []KeyVal{"type=dir"}, nil
}
if info.Mode().IsRegular() {
return "type=file", nil
return []KeyVal{"type=file"}, nil
}
if info.Mode()&os.ModeSocket != 0 {
return "type=socket", nil
return []KeyVal{"type=socket"}, nil
}
if info.Mode()&os.ModeSymlink != 0 {
return "type=link", nil
return []KeyVal{"type=link"}, nil
}
if info.Mode()&os.ModeNamedPipe != 0 {
return "type=fifo", nil
return []KeyVal{"type=fifo"}, nil
}
if info.Mode()&os.ModeDevice != 0 {
if info.Mode()&os.ModeCharDevice != 0 {
return "type=char", nil
return []KeyVal{"type=char"}, nil
}
return "type=block", nil
return []KeyVal{"type=block"}, nil
}
return emptyKV, nil
return nil, nil
}
)

44
keywordfuncs_bsd.go
View file

@ -12,58 +12,58 @@ import (
)
var (
flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
// ideally this will pull in from here https://www.freebsd.org/cgi/man.cgi?query=chflags&sektion=2
return emptyKV, nil
return nil, nil
}
unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("uname=%s", hdr.Uname)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uname=%s", hdr.Uname))}, nil
}
stat := info.Sys().(*syscall.Stat_t)
u, err := user.LookupId(fmt.Sprintf("%d", stat.Uid))
if err != nil {
return emptyKV, err
return nil, err
}
return KeyVal(fmt.Sprintf("uname=%s", u.Username)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uname=%s", u.Username))}, nil
}
gnameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
gnameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("gname=%s", hdr.Gname)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gname=%s", hdr.Gname))}, nil
}
stat := info.Sys().(*syscall.Stat_t)
g, err := lookupGroupID(fmt.Sprintf("%d", stat.Gid))
if err != nil {
return emptyKV, err
return nil, err
}
return KeyVal(fmt.Sprintf("gname=%s", g.Name)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gname=%s", g.Name))}, nil
}
uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("uid=%d", hdr.Uid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uid=%d", hdr.Uid))}, nil
}
stat := info.Sys().(*syscall.Stat_t)
return KeyVal(fmt.Sprintf("uid=%d", stat.Uid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uid=%d", stat.Uid))}, nil
}
gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("gid=%d", hdr.Gid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gid=%d", hdr.Gid))}, nil
}
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
return KeyVal(fmt.Sprintf("gid=%d", stat.Gid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gid=%d", stat.Gid))}, nil
}
return emptyKV, nil
return nil, nil
}
nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
return KeyVal(fmt.Sprintf("nlink=%d", stat.Nlink)), nil
return []KeyVal{KeyVal(fmt.Sprintf("nlink=%d", stat.Nlink))}, nil
}
return emptyKV, nil
return nil, nil
}
xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
return emptyKV, nil
xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
return nil, nil
}
)

59
keywordfuncs_linux.go
View file

@ -9,7 +9,6 @@ import (
"io"
"os"
"os/user"
"strings"
"syscall"
"github.com/vbatts/go-mtree/pkg/govis"
@ -18,91 +17,91 @@ import (
var (
// this is bsd specific https://www.freebsd.org/cgi/man.cgi?query=chflags&sektion=2
flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
return emptyKV, nil
flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
return nil, nil
}
unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("uname=%s", hdr.Uname)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uname=%s", hdr.Uname))}, nil
}
stat := info.Sys().(*syscall.Stat_t)
u, err := user.LookupId(fmt.Sprintf("%d", stat.Uid))
if err != nil {
return emptyKV, err
return nil, nil
}
return KeyVal(fmt.Sprintf("uname=%s", u.Username)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uname=%s", u.Username))}, nil
}
gnameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
gnameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("gname=%s", hdr.Gname)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gname=%s", hdr.Gname))}, nil
}
stat := info.Sys().(*syscall.Stat_t)
g, err := lookupGroupID(fmt.Sprintf("%d", stat.Gid))
if err != nil {
return emptyKV, err
return nil, nil
}
return KeyVal(fmt.Sprintf("gname=%s", g.Name)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gname=%s", g.Name))}, nil
}
uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("uid=%d", hdr.Uid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uid=%d", hdr.Uid))}, nil
}
stat := info.Sys().(*syscall.Stat_t)
return KeyVal(fmt.Sprintf("uid=%d", stat.Uid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uid=%d", stat.Uid))}, nil
}
gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("gid=%d", hdr.Gid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gid=%d", hdr.Gid))}, nil
}
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
return KeyVal(fmt.Sprintf("gid=%d", stat.Gid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gid=%d", stat.Gid))}, nil
}
return emptyKV, nil
return nil, nil
}
nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
return KeyVal(fmt.Sprintf("nlink=%d", stat.Nlink)), nil
return []KeyVal{KeyVal(fmt.Sprintf("nlink=%d", stat.Nlink))}, nil
}
return emptyKV, nil
return nil, nil
}
xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
if len(hdr.Xattrs) == 0 {
return emptyKV, nil
return nil, nil
}
klist := []KeyVal{}
for k, v := range hdr.Xattrs {
encKey, err := govis.Vis(k, DefaultVisFlags)
if err != nil {
return emptyKV, err
return nil, nil
}
klist = append(klist, KeyVal(fmt.Sprintf("xattr.%s=%s", encKey, base64.StdEncoding.EncodeToString([]byte(v)))))
}
return KeyVal(strings.Join(KeyValToString(klist), " ")), nil
return klist, nil
}
if !info.Mode().IsRegular() && !info.Mode().IsDir() {
return emptyKV, nil
return nil, nil
}
xlist, err := xattr.List(path)
if err != nil {
return emptyKV, err
return nil, nil
}
klist := make([]KeyVal, len(xlist))
for i := range xlist {
data, err := xattr.Get(path, xlist[i])
if err != nil {
return emptyKV, err
return nil, nil
}
encKey, err := govis.Vis(xlist[i], DefaultVisFlags)
if err != nil {
return emptyKV, err
return nil, nil
}
klist[i] = KeyVal(fmt.Sprintf("xattr.%s=%s", encKey, base64.StdEncoding.EncodeToString(data)))
}
return KeyVal(strings.Join(KeyValToString(klist), " ")), nil
return klist, nil
}
)

36
keywordfuncs_unsupported.go
View file

@ -11,37 +11,37 @@ import (
var (
// this is bsd specific https://www.freebsd.org/cgi/man.cgi?query=chflags&sektion=2
flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
return emptyKV, nil
flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
return nil, nil
}
unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("uname=%s", hdr.Uname)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uname=%s", hdr.Uname))}, nil
}
return emptyKV, nil
return nil, nil
}
gnameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
gnameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("gname=%s", hdr.Gname)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gname=%s", hdr.Gname))}, nil
}
return emptyKV, nil
return nil, nil
}
uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("uid=%d", hdr.Uid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uid=%d", hdr.Uid))}, nil
}
return emptyKV, nil
return nil, nil
}
gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("gid=%d", hdr.Gid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gid=%d", hdr.Gid))}, nil
}
return emptyKV, nil
return nil, nil
}
nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
return emptyKV, nil
nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
return nil, nil
}
xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
return emptyKV, nil
xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
return nil, nil
}
)

80
keywords.go
View file

@ -13,8 +13,30 @@ const DefaultVisFlags govis.VisFlag = govis.VisWhite | govis.VisOctal | govis.Vi
// Keyword is the string name of a keyword, with some convenience functions for
// determining whether it is a default or bsd standard keyword.
// It first portion before the "="
type Keyword string
// Prefix is the portion of the keyword before a first "." (if present).
//
// Primarly for the xattr use-case, where the keyword `xattr.security.selinux` would have a Suffix of `security.selinux`.
func (k Keyword) Prefix() Keyword {
if strings.Contains(string(k), ".") {
return Keyword(strings.SplitN(string(k), ".", 2)[0])
}
return k
}
// Suffix is the portion of the keyword after a first ".".
// This is an option feature.
//
// Primarly for the xattr use-case, where the keyword `xattr.security.selinux` would have a Suffix of `security.selinux`.
func (k Keyword) Suffix() string {
if strings.Contains(string(k), ".") {
return strings.SplitN(string(k), ".", 2)[1]
}
return string(k)
}
// Default returns whether this keyword is in the default set of keywords
func (k Keyword) Default() bool {
return InKeywordSlice(k, DefaultKeywords)
@ -93,24 +115,7 @@ func (kv KeyVal) Keyword() Keyword {
if !strings.Contains(string(kv), "=") {
return Keyword("")
}
chunks := strings.SplitN(strings.TrimSpace(string(kv)), "=", 2)[0]
if !strings.Contains(chunks, ".") {
return Keyword(chunks)
}
return Keyword(strings.SplitN(chunks, ".", 2)[0])
}
// KeywordSuffix is really only used for xattr, as the keyword is a prefix to
// the xattr "namespace.key"
func (kv KeyVal) KeywordSuffix() string {
if !strings.Contains(string(kv), "=") {
return ""
}
chunks := strings.SplitN(strings.TrimSpace(string(kv)), "=", 2)[0]
if !strings.Contains(chunks, ".") {
return ""
}
return strings.SplitN(chunks, ".", 2)[1]
return Keyword(strings.SplitN(strings.TrimSpace(string(kv)), "=", 2)[0])
}
// Value is the data/value portion of "keyword=value"
@ -123,9 +128,6 @@ func (kv KeyVal) Value() string {
// NewValue returns a new KeyVal with the newval
func (kv KeyVal) NewValue(newval string) KeyVal {
if suff := kv.KeywordSuffix(); suff != "" {
return KeyVal(fmt.Sprintf("%s.%s=%s", kv.Keyword(), suff, newval))
}
return KeyVal(fmt.Sprintf("%s=%s", kv.Keyword(), newval))
}
@ -135,14 +137,23 @@ func (kv KeyVal) NewValue(newval string) KeyVal {
// doing.
func (kv KeyVal) Equal(b KeyVal) bool {
// TODO: Implement handling of tar_mtime.
return kv.Keyword() == b.Keyword() && kv.KeywordSuffix() == b.KeywordSuffix() && kv.Value() == b.Value()
return kv.Keyword() == b.Keyword() && kv.Value() == b.Value()
}
// keyvalSelector takes an array of KeyVal ("keyword=value") and filters out that only the set of keywords
func keywordPrefixes(kvset []Keyword) []Keyword {
kvs := []Keyword{}
for _, kv := range kvset {
kvs = append(kvs, kv.Prefix())
}
return kvs
}
// keyvalSelector takes an array of KeyVal ("keyword=value") and filters out
// that only the set of keywords
func keyvalSelector(keyval []KeyVal, keyset []Keyword) []KeyVal {
retList := []KeyVal{}
for _, kv := range keyval {
if InKeywordSlice(kv.Keyword(), keyset) {
if InKeywordSlice(kv.Keyword().Prefix(), keywordPrefixes(keyset)) {
retList = append(retList, kv)
}
}
@ -171,23 +182,23 @@ func keyValCopy(set []KeyVal) []KeyVal {
// Has the "keyword" present in the list of KeyVal, and returns the
// corresponding KeyVal, else an empty string.
func Has(keyvals []KeyVal, keyword string) KeyVal {
func Has(keyvals []KeyVal, keyword string) []KeyVal {
return HasKeyword(keyvals, Keyword(keyword))
}
// HasKeyword the "keyword" present in the list of KeyVal, and returns the
// corresponding KeyVal, else an empty string.
func HasKeyword(keyvals []KeyVal, keyword Keyword) KeyVal {
// This match is done on the Prefix of the keyword only.
func HasKeyword(keyvals []KeyVal, keyword Keyword) []KeyVal {
kvs := []KeyVal{}
for i := range keyvals {
if keyvals[i].Keyword() == keyword {
return keyvals[i]
if keyvals[i].Keyword().Prefix() == keyword.Prefix() {
kvs = append(kvs, keyvals[i])
}
}
return emptyKV
return kvs
}
var emptyKV = KeyVal("")
// MergeSet takes the current setKeyVals, and then applies the entryKeyVals
// such that the entry's values win. The union is returned.
func MergeSet(setKeyVals, entryKeyVals []string) []KeyVal {
@ -203,8 +214,11 @@ func MergeKeyValSet(setKeyVals, entryKeyVals []KeyVal) []KeyVal {
seenKeywords := []Keyword{}
for i := range retList {
word := retList[i].Keyword()
if ekv := HasKeyword(entryKeyVals, word); ekv != emptyKV {
retList[i] = ekv
for _, kv := range HasKeyword(entryKeyVals, word) {
// match on the keyword prefix and suffix here
if kv.Keyword() == word {
retList[i] = kv
}
}
seenKeywords = append(seenKeywords, word)
}

12
keywords_linux_test.go
View file

@ -51,12 +51,12 @@ func TestXattr(t *testing.T) {
t.Fatal(err)
}
// Check the directory
str, err := xattrKeywordFunc(dir, dirstat, nil)
kvs, err := xattrKeywordFunc(dir, dirstat, nil)
if err != nil {
t.Error(err)
}
if str == "" {
t.Errorf("expected a keyval; got %q", str)
if len(kvs) == 0 {
t.Errorf("expected a keyval; got none")
}
filestat, err := fh.Stat()
@ -64,12 +64,12 @@ func TestXattr(t *testing.T) {
t.Fatal(err)
}
// Check the regular file
str, err = xattrKeywordFunc(filepath.Join(dir, "file"), filestat, fh)
kvs, err = xattrKeywordFunc(filepath.Join(dir, "file"), filestat, fh)
if err != nil {
t.Error(err)
}
if str == "" {
t.Errorf("expected a keyval; got %q", str)
if len(kvs) == 0 {
t.Errorf("expected a keyval; got none")
}
linkstat, err := os.Lstat(filepath.Join(dir, "symlink"))

25
keywords_test.go
View file

@ -9,14 +9,20 @@ import (
func TestKeyValRoundtrip(t *testing.T) {
kv := KeyVal("xattr.security.selinux=dW5jb25maW5lZF91Om9iamVjdF9yOnVzZXJfaG9tZV90OnMwAA==")
expected := "xattr"
expected := "xattr.security.selinux"
got := string(kv.Keyword())
if got != expected {
t.Errorf("expected %q; got %q", expected, got)
}
expected = "xattr"
got = string(kv.Keyword().Prefix())
if got != expected {
t.Errorf("expected %q; got %q", expected, got)
}
expected = "security.selinux"
got = kv.KeywordSuffix()
got = kv.Keyword().Suffix()
if got != expected {
t.Errorf("expected %q; got %q", expected, got)
}
@ -27,7 +33,6 @@ func TestKeyValRoundtrip(t *testing.T) {
t.Errorf("expected %q; got %q", expected, got)
}
expected = "xattr.security.selinux=farts"
got = string(kv.NewValue("farts"))
if got != expected {
@ -97,8 +102,11 @@ func TestKeywordsTimeNano(t *testing.T) {
if err != nil {
t.Errorf("unexpected error while parsing '%q': %q", mtime, err)
}
if expected != got {
t.Errorf("keyword didn't match, expected '%s' got '%s'", expected, got)
if len(got) != 1 {
t.Errorf("expected 1 KeyVal, but got %d", len(got))
}
if expected != got[0] {
t.Errorf("keyword didn't match, expected '%s' got '%s'", expected, got[0])
}
}
}
@ -124,8 +132,11 @@ func TestKeywordsTimeTar(t *testing.T) {
if err != nil {
t.Errorf("unexpected error while parsing '%q': %q", mtime, err)
}
if expected != got {
t.Errorf("keyword didn't match, expected '%s' got '%s'", expected, got)
if len(got) != 1 {
t.Errorf("expected 1 KeyVal, but got %d", len(got))
}
if expected != got[0] {
t.Errorf("keyword didn't match, expected '%s' got '%s'", expected, got[0])
}
}
}

38
tar.go
View file

@ -5,11 +5,11 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"os"
"path/filepath"
"strings"
"github.com/Sirupsen/logrus"
"github.com/vbatts/go-mtree/pkg/govis"
)
@ -144,16 +144,16 @@ hdrloop:
// Keep track of which files are hardlinks so we can resolve them later
if hdr.Typeflag == tar.TypeLink {
linkFunc := KeywordFuncs["link"]
kv, err := linkFunc(hdr.Name, hdr.FileInfo(), nil)
keyFunc := KeywordFuncs["link"]
kvs, err := keyFunc(hdr.Name, hdr.FileInfo(), nil)
if err != nil {
log.Println(err)
break
logrus.Warn(err)
break // XXX is breaking an okay thing to do here?
}
linkname, err := govis.Unvis(KeyVal(kv).Value(), DefaultVisFlags)
linkname, err := govis.Unvis(KeyVal(kvs[0]).Value(), DefaultVisFlags)
if err != nil {
log.Println(err)
break
logrus.Warn(err)
break // XXX is breaking an okay thing to do here?
}
if _, ok := ts.hardlinks[linkname]; !ok {
ts.hardlinks[linkname] = []string{hdr.Name}
@ -164,19 +164,19 @@ hdrloop:
// now collect keywords on the file
for _, keyword := range ts.keywords {
if keyFunc, ok := KeywordFuncs[keyword]; ok {
if keyFunc, ok := KeywordFuncs[keyword.Prefix()]; ok {
// We can't extract directories on to disk, so "size" keyword
// is irrelevant for now
if hdr.FileInfo().IsDir() && keyword == "size" {
continue
}
val, err := keyFunc(hdr.Name, hdr.FileInfo(), tmpFile)
kvs, err := keyFunc(hdr.Name, hdr.FileInfo(), tmpFile)
if err != nil {
ts.setErr(err)
}
// for good measure, check that we actually get a value for a keyword
if val != "" {
e.Keywords = append(e.Keywords, val)
if len(kvs) > 0 && kvs[0] != "" {
e.Keywords = append(e.Keywords, kvs[0])
}
// don't forget to reset the reader
@ -196,13 +196,15 @@ hdrloop:
Type: SpecialType,
}
for _, setKW := range SetKeywords {
if keyFunc, ok := KeywordFuncs[setKW]; ok {
val, err := keyFunc(hdr.Name, hdr.FileInfo(), tmpFile)
if keyFunc, ok := KeywordFuncs[setKW.Prefix()]; ok {
kvs, err := keyFunc(hdr.Name, hdr.FileInfo(), tmpFile)
if err != nil {
ts.setErr(err)
}
if val != "" {
s.Keywords = append(s.Keywords, val)
for _, kv := range kvs {
if kv != "" {
s.Keywords = append(s.Keywords, kv)
}
}
if _, err := tmpFile.Seek(0, 0); err != nil {
tmpFile.Close()
@ -383,7 +385,7 @@ func resolveHardlinks(root *Entry, hardlinks map[string][]string, countlinks boo
if seen, ok := originals[base]; !ok {
basefile = root.Find(base)
if basefile == nil {
log.Printf("%s does not exist in this tree\n", base)
logrus.Printf("%s does not exist in this tree\n", base)
continue
}
originals[base] = basefile
@ -393,7 +395,7 @@ func resolveHardlinks(root *Entry, hardlinks map[string][]string, countlinks boo
for _, link := range links {
linkfile := root.Find(link)
if linkfile == nil {
log.Printf("%s does not exist in this tree\n", link)
logrus.Printf("%s does not exist in this tree\n", link)
continue
}
linkfile.Keywords = basefile.Keywords

23
update.go
View file

@ -3,6 +3,8 @@ package mtree
import (
"os"
"sort"
"github.com/Sirupsen/logrus"
)
// DefaultUpdateKeywords is the default set of keywords that can take updates to the files on disk
@ -11,7 +13,7 @@ var DefaultUpdateKeywords = []Keyword{
"gid",
"mode",
"time",
// TODO xattr
"xattr",
}
// Update attempts to set the attributes of root directory path, given the values of `keywords` in dh DirectoryHierarchy.
@ -36,7 +38,7 @@ func Update(root string, dh *DirectoryHierarchy, keywords []Keyword, fs FsEval)
} else if e.Name == "/unset" {
creator.curSet = nil
}
Debugf("%#v", e)
logrus.Debugf("%#v", e)
continue
case RelativeType, FullType:
e.Set = creator.curSet
@ -46,20 +48,21 @@ func Update(root string, dh *DirectoryHierarchy, keywords []Keyword, fs FsEval)
}
// filter the keywords to update on the file, from the keywords available for this entry:
var toCheck []KeyVal
toCheck = keyvalSelector(e.AllKeys(), keywords)
Debugf("toCheck(%q): %v", pathname, toCheck)
var kvToUpdate []KeyVal
kvToUpdate = keyvalSelector(e.AllKeys(), keywords)
logrus.Debugf("kvToUpdate(%q): %#v", pathname, kvToUpdate)
for _, kv := range toCheck {
if !InKeywordSlice(kv.Keyword(), keywords) {
for _, kv := range kvToUpdate {
if !InKeywordSlice(kv.Keyword().Prefix(), keywordPrefixes(keywords)) {
continue
}
ukFunc, ok := UpdateKeywordFuncs[kv.Keyword()]
logrus.Debugf("finding function for %q (%q)", kv.Keyword(), kv.Keyword().Prefix())
ukFunc, ok := UpdateKeywordFuncs[kv.Keyword().Prefix()]
if !ok {
Debugf("no UpdateKeywordFunc for %s; skipping", kv.Keyword())
logrus.Debugf("no UpdateKeywordFunc for %s; skipping", kv.Keyword())
continue
}
if _, err := ukFunc(pathname, kv.Value()); err != nil {
if _, err := ukFunc(kv.Keyword(), pathname, kv.Value()); err != nil {
results = append(results, InodeDelta{
diff: ErrorDifference,
path: pathname,

94
update_linux_test.go Normal file
View file

@ -0,0 +1,94 @@
package mtree
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/vbatts/go-mtree/xattr"
)
func init() {
//logrus.SetLevel(logrus.DebugLevel)
}
func TestXattrUpdate(t *testing.T) {
content := []byte("I know half of you half as well as I ought to")
// a bit dirty to create/destory a directory in cwd, but often /tmp is
// mounted tmpfs and doesn't support xattrs
dir, err := ioutil.TempDir(".", "test.xattr.restore.")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir) // clean up
tmpfn := filepath.Join(dir, "tmpfile")
if err := ioutil.WriteFile(tmpfn, content, 0666); err != nil {
t.Fatal(err)
}
if err := xattr.Set(dir, "user.test", []byte("directory")); err != nil {
t.Skip(fmt.Sprintf("skipping: %q does not support xattrs", dir))
}
if err := xattr.Set(tmpfn, "user.test", []byte("regular file")); err != nil {
t.Fatal(err)
}
// Walk this tempdir
dh, err := Walk(dir, nil, append(DefaultKeywords, []Keyword{"xattr", "sha1"}...), nil)
if err != nil {
t.Fatal(err)
}
// Now check that we're sane
res, err := Check(dir, dh, nil, nil)
if err != nil {
t.Fatal(err)
}
if len(res) != 0 {
t.Errorf("expecting no failures, but got %q", res)
}
if err := xattr.Set(tmpfn, "user.test", []byte("let it fly")); err != nil {
t.Fatal(err)
}
// Now check that we fail the check
res, err = Check(dir, dh, nil, nil)
if err != nil {
t.Fatal(err)
}
if len(res) == 0 {
t.Error("expected failures (like xattrs), but got none")
}
// restore the xattrs to original
res, err = Update(dir, dh, append(DefaultUpdateKeywords, "xattr"), nil)
if err != nil {
t.Error(err)
}
if len(res) != 0 {
t.Errorf("expecting no failures, but got %q", res)
}
// Now check that we're sane again
res, err = Check(dir, dh, nil, nil)
if err != nil {
t.Fatal(err)
}
if len(res) != 0 {
// pretty this shit up
buf, err := json.MarshalIndent(res, "", " ")
if err != nil {
t.Errorf("expecting no failures, but got %q", res)
} else {
t.Errorf("expecting no failures, but got %s", string(buf))
}
}
// TODO make a test for xattr here. Likely in the user space for privileges. Even still this may be prone to error for some tmpfs don't act right with xattrs. :-\
// I'd hate to have to t.Skip() a test rather than fail alltogether.
}

3
update_test.go
View file

@ -97,8 +97,9 @@ func TestUpdate(t *testing.T) {
buf, err := json.MarshalIndent(res, "", " ")
if err != nil {
t.Errorf("%#v", res)
}
} else {
t.Error(string(buf))
}
}
}

19
updatefuncs.go
View file

@ -6,12 +6,14 @@ import (
"strconv"
"strings"
"time"
"github.com/Sirupsen/logrus"
)
// UpdateKeywordFunc is the signature for a function that will restore a file's
// attributes. Where path is relative path to the file, and value to be
// restored to.
type UpdateKeywordFunc func(path string, value string) (os.FileInfo, error)
type UpdateKeywordFunc func(keyword Keyword, path string, value string) (os.FileInfo, error)
// UpdateKeywordFuncs is the registered list of functions to update file attributes.
// Keyed by the keyword as it would show up in the manifest
@ -21,9 +23,10 @@ var UpdateKeywordFuncs = map[Keyword]UpdateKeywordFunc{
"tar_time": tartimeUpdateKeywordFunc,
"uid": uidUpdateKeywordFunc,
"gid": gidUpdateKeywordFunc,
"xattr": xattrUpdateKeywordFunc,
}
func uidUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
func uidUpdateKeywordFunc(keyword Keyword, path, value string) (os.FileInfo, error) {
uid, err := strconv.Atoi(value)
if err != nil {
return nil, err
@ -34,7 +37,7 @@ func uidUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
return os.Lstat(path)
}
func gidUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
func gidUpdateKeywordFunc(keyword Keyword, path, value string) (os.FileInfo, error) {
gid, err := strconv.Atoi(value)
if err != nil {
return nil, err
@ -45,12 +48,12 @@ func gidUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
return os.Lstat(path)
}
func modeUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
func modeUpdateKeywordFunc(keyword Keyword, path, value string) (os.FileInfo, error) {
vmode, err := strconv.ParseInt(value, 8, 32)
if err != nil {
return nil, err
}
Debugf("path: %q, value: %q, vmode: %o", path, value, vmode)
logrus.Debugf("path: %q, value: %q, vmode: %o", path, value, vmode)
if err := os.Chmod(path, os.FileMode(vmode)); err != nil {
return nil, err
}
@ -60,7 +63,7 @@ func modeUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
// since tar_time will only be second level precision, then when restoring the
// filepath from a tar_time, then compare the seconds first and only Chtimes if
// the seconds value is different.
func tartimeUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
func tartimeUpdateKeywordFunc(keyword Keyword, path, value string) (os.FileInfo, error) {
info, err := os.Lstat(path)
if err != nil {
return nil, err
@ -89,7 +92,7 @@ func tartimeUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
}
// this is nano second precision
func timeUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
func timeUpdateKeywordFunc(keyword Keyword, path, value string) (os.FileInfo, error) {
v := strings.SplitN(value, ".", 2)
if len(v) != 2 {
return nil, fmt.Errorf("expected a number like 1469104727.871937272")
@ -98,7 +101,7 @@ func timeUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
if err != nil {
return nil, fmt.Errorf("expected nano seconds, but got %q", v[0]+v[1])
}
Debugf("arg: %q; nsec: %q", v[0]+v[1], nsec)
logrus.Debugf("arg: %q; nsec: %q", v[0]+v[1], nsec)
vtime := time.Unix(0, nsec)
if err := os.Chtimes(path, vtime, vtime); err != nil {

21
updatefuncs_linux.go Normal file
View file

@ -0,0 +1,21 @@
// +build linux
package mtree
import (
"encoding/base64"
"os"
"github.com/vbatts/go-mtree/xattr"
)
func xattrUpdateKeywordFunc(keyword Keyword, path, value string) (os.FileInfo, error) {
buf, err := base64.StdEncoding.DecodeString(value)
if err != nil {
return nil, err
}
if err := xattr.Set(path, keyword.Suffix(), buf); err != nil {
return nil, err
}
return os.Lstat(path)
}

9
updatefuncs_unsupported.go Normal file
View file

@ -0,0 +1,9 @@
// +build !linux
package mtree
import "os"
func xattrUpdateKeywordFunc(keyword Keyword, path, value string) (os.FileInfo, error) {
return os.Lstat(path)
}

38
walk.go
View file

@ -101,15 +101,19 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval
defer fh.Close()
r = fh
}
keywordFunc, ok := KeywordFuncs[keyword]
keyFunc, ok := KeywordFuncs[keyword.Prefix()]
if !ok {
return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
return fmt.Errorf("Unknown keyword %q for file %q", keyword.Prefix(), path)
}
if str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r); err == nil && str != "" {
e.Keywords = append(e.Keywords, str)
} else if err != nil {
kvs, err := creator.fs.KeywordFunc(keyFunc)(path, info, r)
if err != nil {
return err
}
for _, kv := range kvs {
if kv != "" {
e.Keywords = append(e.Keywords, kv)
}
}
return nil
}()
if err != nil {
@ -132,16 +136,18 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval
defer fh.Close()
r = fh
}
keywordFunc, ok := KeywordFuncs[keyword]
keyFunc, ok := KeywordFuncs[keyword.Prefix()]
if !ok {
return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
return fmt.Errorf("Unknown keyword %q for file %q", keyword.Prefix(), path)
}
str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r)
kvs, err := creator.fs.KeywordFunc(keyFunc)(path, info, r)
if err != nil {
return err
}
if str != "" {
klist = append(klist, str)
for _, kv := range kvs {
if kv != "" {
klist = append(klist, kv)
}
}
return nil
}()
@ -190,16 +196,18 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval
defer fh.Close()
r = fh
}
keywordFunc, ok := KeywordFuncs[keyword]
keyFunc, ok := KeywordFuncs[keyword.Prefix()]
if !ok {
return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
return fmt.Errorf("Unknown keyword %q for file %q", keyword.Prefix(), path)
}
str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r)
kvs, err := creator.fs.KeywordFunc(keyFunc)(path, info, r)
if err != nil {
return err
}
if str != "" && !inKeyValSlice(str, creator.curSet.Keywords) {
e.Keywords = append(e.Keywords, str)
for _, kv := range kvs {
if kv != "" && !inKeyValSlice(kv, creator.curSet.Keywords) {
e.Keywords = append(e.Keywords, kv)
}
}
return nil
}()