mirrors/go-mtree
1
0
Fork 0
mirror of https://github.com/vbatts/go-mtree.git synced 2024-11-24 17:15:40 +00:00
Code Releases Activity

*: xattr can Update()

Browse source 
This is a gnarly patchset that has been mashed together.
It uncovered that some aspects of Check were never really working
correctly for `xattr` keywords, but also the `Update()` had been left
undone for a while.

This includes some API changes around the `Keyword` and `KeyVal` types.

Also I would like to update the signature for the `UpdateKeywordFunc` to
just accept a `KeyVal` as an argugment, rather than a keyword AND the
value. with this context there would be no need to guess on the value of
what's passed to the xattr update function of whether it needs or
already is base64 encoded.

Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
This commit is contained in:
Vincent Batts 2017-06-15 14:54:58 -04:00
parent fb4ec19981
commit ed464af779
Signed by: vbatts
GPG key ID: 10937E57733F1362
19 changed files with 398 additions and 258 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
check_test.go
View file

@ -12,7 +12,7 @@ import (
// simple walk of current directory, and imediately check it.
// may not be parallelizable.
func TestCheck(t *testing.T) {
dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
dh, err := Walk(".", nil, append(DefaultKeywords, []Keyword{"sha1", "xattr"}...), nil)
if err != nil {
t.Fatal(err)
}

5
cmd/gomtree/main.go
View file

@ -7,10 +7,10 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"os"
"strings"
"github.com/Sirupsen/logrus"
"github.com/vbatts/go-mtree"
)
@ -37,7 +37,7 @@ var (
func main() {
// so that defers cleanly exec
if err := app(); err != nil {
log.Fatal(err)
logrus.Fatal(err)
}
}
@ -46,6 +46,7 @@ func app() error {
if *flDebug {
os.Setenv("DEBUG", "1")
logrus.SetLevel(logrus.DebugLevel)
}
if *flVersion {

4
compare.go
View file

@ -197,7 +197,7 @@ func compareEntry(oldEntry, newEntry Entry) ([]KeyDelta, error) {
for _, kv := range oldKeys {
key := kv.Keyword()
// only add this diff if the new keys has this keyword
if key != "tar_time" && key != "time" && key != "xattr" && HasKeyword(newKeys, key) == emptyKV {
if key != "tar_time" && key != "time" && key != "xattr" && len(HasKeyword(newKeys, key)) == 0 {
continue
}
@ -216,7 +216,7 @@ func compareEntry(oldEntry, newEntry Entry) ([]KeyDelta, error) {
for _, kv := range newKeys {
key := kv.Keyword()
// only add this diff if the old keys has this keyword
if key != "tar_time" && key != "time" && key != "xattr" && HasKeyword(oldKeys, key) == emptyKV {
if key != "tar_time" && key != "time" && key != "xattr" && len(HasKeyword(oldKeys, key)) == 0 {
continue
}

26
debug.go
View file

@ -1,26 +0,0 @@
package mtree
import (
"fmt"
"os"
"time"
)
// DebugOutput is the where DEBUG output is written
var DebugOutput = os.Stderr
// Debugln writes output to DebugOutput, only if DEBUG environment variable is set
func Debugln(a ...interface{}) (n int, err error) {
if os.Getenv("DEBUG") != "" {
return fmt.Fprintf(DebugOutput, "[%d] [DEBUG] %s\n", time.Now().UnixNano(), a)
}
return 0, nil
}
// Debugf writes formatted output to DebugOutput, only if DEBUG environment variable is set
func Debugf(format string, a ...interface{}) (n int, err error) {
if os.Getenv("DEBUG") != "" {
return fmt.Fprintf(DebugOutput, "[%d] [DEBUG] %s\n", time.Now().UnixNano(), fmt.Sprintf(format, a...))
}
return 0, nil
}

70
keywordfunc.go
View file

@ -21,7 +21,7 @@ import (
// io.Reader `r` is to the file stream for the file payload. While this
// function takes an io.Reader, the caller needs to reset it to the beginning
// for each new KeywordFunc
type KeywordFunc func(path string, info os.FileInfo, r io.Reader) (KeyVal, error)
type KeywordFunc func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error)
var (
// KeywordFuncs is the map of all keywords (and the functions to produce them)
@ -67,7 +67,7 @@ var (
}
)
var (
modeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
modeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
permissions := info.Mode().Perm()
if os.ModeSetuid&info.Mode() > 0 {
permissions |= (1 << 11)
@ -78,93 +78,93 @@ var (
if os.ModeSticky&info.Mode() > 0 {
permissions |= (1 << 9)
}
return KeyVal(fmt.Sprintf("mode=%#o", permissions)), nil
return []KeyVal{KeyVal(fmt.Sprintf("mode=%#o", permissions))}, nil
}
sizeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
sizeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if sys, ok := info.Sys().(*tar.Header); ok {
if sys.Typeflag == tar.TypeSymlink {
return KeyVal(fmt.Sprintf("size=%d", len(sys.Linkname))), nil
return []KeyVal{KeyVal(fmt.Sprintf("size=%d", len(sys.Linkname)))}, nil
}
}
return KeyVal(fmt.Sprintf("size=%d", info.Size())), nil
return []KeyVal{KeyVal(fmt.Sprintf("size=%d", info.Size()))}, nil
}
cksumKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
cksumKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if !info.Mode().IsRegular() {
return emptyKV, nil
return nil, nil
}
sum, _, err := cksum(r)
if err != nil {
return emptyKV, err
return nil, err
}
return KeyVal(fmt.Sprintf("cksum=%d", sum)), nil
return []KeyVal{KeyVal(fmt.Sprintf("cksum=%d", sum))}, nil
}
hasherKeywordFunc = func(name string, newHash func() hash.Hash) KeywordFunc {
return func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
return func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if !info.Mode().IsRegular() {
return emptyKV, nil
return nil, nil
}
h := newHash()
if _, err := io.Copy(h, r); err != nil {
return emptyKV, err
return nil, err
}
return KeyVal(fmt.Sprintf("%s=%x", KeywordSynonym(name), h.Sum(nil))), nil
return []KeyVal{KeyVal(fmt.Sprintf("%s=%x", KeywordSynonym(name), h.Sum(nil)))}, nil
}
}
tartimeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
return KeyVal(fmt.Sprintf("tar_time=%d.%9.9d", info.ModTime().Unix(), 0)), nil
tartimeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
return []KeyVal{KeyVal(fmt.Sprintf("tar_time=%d.%9.9d", info.ModTime().Unix(), 0))}, nil
}
timeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
timeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
tSec := info.ModTime().Unix()
tNano := info.ModTime().Nanosecond()
return KeyVal(fmt.Sprintf("time=%d.%9.9d", tSec, tNano)), nil
return []KeyVal{KeyVal(fmt.Sprintf("time=%d.%9.9d", tSec, tNano))}, nil
}
linkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
linkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if sys, ok := info.Sys().(*tar.Header); ok {
if sys.Linkname != "" {
linkname, err := govis.Vis(sys.Linkname, DefaultVisFlags)
if err != nil {
return emptyKV, err
return nil, nil
}
return KeyVal(fmt.Sprintf("link=%s", linkname)), nil
return []KeyVal{KeyVal(fmt.Sprintf("link=%s", linkname))}, nil
}
return emptyKV, nil
return nil, nil
}
if info.Mode()&os.ModeSymlink != 0 {
str, err := os.Readlink(path)
if err != nil {
return emptyKV, err
return nil, nil
}
linkname, err := govis.Vis(str, DefaultVisFlags)
if err != nil {
return emptyKV, err
return nil, nil
}
return KeyVal(fmt.Sprintf("link=%s", linkname)), nil
return []KeyVal{KeyVal(fmt.Sprintf("link=%s", linkname))}, nil
}
return emptyKV, nil
return nil, nil
}
typeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
typeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if info.Mode().IsDir() {
return "type=dir", nil
return []KeyVal{"type=dir"}, nil
}
if info.Mode().IsRegular() {
return "type=file", nil
return []KeyVal{"type=file"}, nil
}
if info.Mode()&os.ModeSocket != 0 {
return "type=socket", nil
return []KeyVal{"type=socket"}, nil
}
if info.Mode()&os.ModeSymlink != 0 {
return "type=link", nil
return []KeyVal{"type=link"}, nil
}
if info.Mode()&os.ModeNamedPipe != 0 {
return "type=fifo", nil
return []KeyVal{"type=fifo"}, nil
}
if info.Mode()&os.ModeDevice != 0 {
if info.Mode()&os.ModeCharDevice != 0 {
return "type=char", nil
return []KeyVal{"type=char"}, nil
}
return "type=block", nil
return []KeyVal{"type=block"}, nil
}
return emptyKV, nil
return nil, nil
}
)

44
keywordfuncs_bsd.go
View file

@ -12,58 +12,58 @@ import (
)
var (
flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
// ideally this will pull in from here https://www.freebsd.org/cgi/man.cgi?query=chflags&sektion=2
return emptyKV, nil
return nil, nil
}
unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("uname=%s", hdr.Uname)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uname=%s", hdr.Uname))}, nil
}
stat := info.Sys().(*syscall.Stat_t)
u, err := user.LookupId(fmt.Sprintf("%d", stat.Uid))
if err != nil {
return emptyKV, err
return nil, err
}
return KeyVal(fmt.Sprintf("uname=%s", u.Username)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uname=%s", u.Username))}, nil
}
gnameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
gnameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("gname=%s", hdr.Gname)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gname=%s", hdr.Gname))}, nil
}
stat := info.Sys().(*syscall.Stat_t)
g, err := lookupGroupID(fmt.Sprintf("%d", stat.Gid))
if err != nil {
return emptyKV, err
return nil, err
}
return KeyVal(fmt.Sprintf("gname=%s", g.Name)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gname=%s", g.Name))}, nil
}
uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("uid=%d", hdr.Uid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uid=%d", hdr.Uid))}, nil
}
stat := info.Sys().(*syscall.Stat_t)
return KeyVal(fmt.Sprintf("uid=%d", stat.Uid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uid=%d", stat.Uid))}, nil
}
gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("gid=%d", hdr.Gid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gid=%d", hdr.Gid))}, nil
}
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
return KeyVal(fmt.Sprintf("gid=%d", stat.Gid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gid=%d", stat.Gid))}, nil
}
return emptyKV, nil
return nil, nil
}
nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
return KeyVal(fmt.Sprintf("nlink=%d", stat.Nlink)), nil
return []KeyVal{KeyVal(fmt.Sprintf("nlink=%d", stat.Nlink))}, nil
}
return emptyKV, nil
return nil, nil
}
xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
return emptyKV, nil
xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
return nil, nil
}
)

59
keywordfuncs_linux.go
View file

@ -9,7 +9,6 @@ import (
"io"
"os"
"os/user"
"strings"
"syscall"
"github.com/vbatts/go-mtree/pkg/govis"
@ -18,91 +17,91 @@ import (
var (
// this is bsd specific https://www.freebsd.org/cgi/man.cgi?query=chflags&sektion=2
flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
return emptyKV, nil
flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
return nil, nil
}
unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("uname=%s", hdr.Uname)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uname=%s", hdr.Uname))}, nil
}
stat := info.Sys().(*syscall.Stat_t)
u, err := user.LookupId(fmt.Sprintf("%d", stat.Uid))
if err != nil {
return emptyKV, err
return nil, nil
}
return KeyVal(fmt.Sprintf("uname=%s", u.Username)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uname=%s", u.Username))}, nil
}
gnameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
gnameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("gname=%s", hdr.Gname)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gname=%s", hdr.Gname))}, nil
}
stat := info.Sys().(*syscall.Stat_t)
g, err := lookupGroupID(fmt.Sprintf("%d", stat.Gid))
if err != nil {
return emptyKV, err
return nil, nil
}
return KeyVal(fmt.Sprintf("gname=%s", g.Name)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gname=%s", g.Name))}, nil
}
uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("uid=%d", hdr.Uid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uid=%d", hdr.Uid))}, nil
}
stat := info.Sys().(*syscall.Stat_t)
return KeyVal(fmt.Sprintf("uid=%d", stat.Uid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uid=%d", stat.Uid))}, nil
}
gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("gid=%d", hdr.Gid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gid=%d", hdr.Gid))}, nil
}
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
return KeyVal(fmt.Sprintf("gid=%d", stat.Gid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gid=%d", stat.Gid))}, nil
}
return emptyKV, nil
return nil, nil
}
nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
return KeyVal(fmt.Sprintf("nlink=%d", stat.Nlink)), nil
return []KeyVal{KeyVal(fmt.Sprintf("nlink=%d", stat.Nlink))}, nil
}
return emptyKV, nil
return nil, nil
}
xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
if len(hdr.Xattrs) == 0 {
return emptyKV, nil
return nil, nil
}
klist := []KeyVal{}
for k, v := range hdr.Xattrs {
encKey, err := govis.Vis(k, DefaultVisFlags)
if err != nil {
return emptyKV, err
return nil, nil
}
klist = append(klist, KeyVal(fmt.Sprintf("xattr.%s=%s", encKey, base64.StdEncoding.EncodeToString([]byte(v)))))
}
return KeyVal(strings.Join(KeyValToString(klist), " ")), nil
return klist, nil
}
if !info.Mode().IsRegular() && !info.Mode().IsDir() {
return emptyKV, nil
return nil, nil
}
xlist, err := xattr.List(path)
if err != nil {
return emptyKV, err
return nil, nil
}
klist := make([]KeyVal, len(xlist))
for i := range xlist {
data, err := xattr.Get(path, xlist[i])
if err != nil {
return emptyKV, err
return nil, nil
}
encKey, err := govis.Vis(xlist[i], DefaultVisFlags)
if err != nil {
return emptyKV, err
return nil, nil
}
klist[i] = KeyVal(fmt.Sprintf("xattr.%s=%s", encKey, base64.StdEncoding.EncodeToString(data)))
}
return KeyVal(strings.Join(KeyValToString(klist), " ")), nil
return klist, nil
}
)

36
keywordfuncs_unsupported.go
View file

@ -11,37 +11,37 @@ import (
var (
// this is bsd specific https://www.freebsd.org/cgi/man.cgi?query=chflags&sektion=2
flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
return emptyKV, nil
flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
return nil, nil
}
unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("uname=%s", hdr.Uname)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uname=%s", hdr.Uname))}, nil
}
return emptyKV, nil
return nil, nil
}
gnameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
gnameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("gname=%s", hdr.Gname)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gname=%s", hdr.Gname))}, nil
}
return emptyKV, nil
return nil, nil
}
uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("uid=%d", hdr.Uid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("uid=%d", hdr.Uid))}, nil
}
return emptyKV, nil
return nil, nil
}
gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
if hdr, ok := info.Sys().(*tar.Header); ok {
return KeyVal(fmt.Sprintf("gid=%d", hdr.Gid)), nil
return []KeyVal{KeyVal(fmt.Sprintf("gid=%d", hdr.Gid))}, nil
}
return emptyKV, nil
return nil, nil
}
nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
return emptyKV, nil
nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
return nil, nil
}
xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
return emptyKV, nil
xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
return nil, nil
}
)

80
keywords.go
View file

@ -13,8 +13,30 @@ const DefaultVisFlags govis.VisFlag = govis.VisWhite | govis.VisOctal | govis.Vi
// Keyword is the string name of a keyword, with some convenience functions for
// determining whether it is a default or bsd standard keyword.
// It first portion before the "="
type Keyword string
// Prefix is the portion of the keyword before a first "." (if present).
//
// Primarly for the xattr use-case, where the keyword `xattr.security.selinux` would have a Suffix of `security.selinux`.
func (k Keyword) Prefix() Keyword {
if strings.Contains(string(k), ".") {
return Keyword(strings.SplitN(string(k), ".", 2)[0])
}
return k
}
// Suffix is the portion of the keyword after a first ".".
// This is an option feature.
//
// Primarly for the xattr use-case, where the keyword `xattr.security.selinux` would have a Suffix of `security.selinux`.
func (k Keyword) Suffix() string {
if strings.Contains(string(k), ".") {
return strings.SplitN(string(k), ".", 2)[1]
}
return string(k)
}
// Default returns whether this keyword is in the default set of keywords
func (k Keyword) Default() bool {
return InKeywordSlice(k, DefaultKeywords)
@ -93,24 +115,7 @@ func (kv KeyVal) Keyword() Keyword {
if !strings.Contains(string(kv), "=") {
return Keyword("")
}
chunks := strings.SplitN(strings.TrimSpace(string(kv)), "=", 2)[0]
if !strings.Contains(chunks, ".") {
return Keyword(chunks)
}
return Keyword(strings.SplitN(chunks, ".", 2)[0])
}
// KeywordSuffix is really only used for xattr, as the keyword is a prefix to
// the xattr "namespace.key"
func (kv KeyVal) KeywordSuffix() string {
if !strings.Contains(string(kv), "=") {
return ""
}
chunks := strings.SplitN(strings.TrimSpace(string(kv)), "=", 2)[0]
if !strings.Contains(chunks, ".") {
return ""
}
return strings.SplitN(chunks, ".", 2)[1]
return Keyword(strings.SplitN(strings.TrimSpace(string(kv)), "=", 2)[0])
}
// Value is the data/value portion of "keyword=value"
@ -123,9 +128,6 @@ func (kv KeyVal) Value() string {
// NewValue returns a new KeyVal with the newval
func (kv KeyVal) NewValue(newval string) KeyVal {
if suff := kv.KeywordSuffix(); suff != "" {
return KeyVal(fmt.Sprintf("%s.%s=%s", kv.Keyword(), suff, newval))
}
return KeyVal(fmt.Sprintf("%s=%s", kv.Keyword(), newval))
}
@ -135,14 +137,23 @@ func (kv KeyVal) NewValue(newval string) KeyVal {
// doing.
func (kv KeyVal) Equal(b KeyVal) bool {
// TODO: Implement handling of tar_mtime.
return kv.Keyword() == b.Keyword() && kv.KeywordSuffix() == b.KeywordSuffix() && kv.Value() == b.Value()
return kv.Keyword() == b.Keyword() && kv.Value() == b.Value()
}
// keyvalSelector takes an array of KeyVal ("keyword=value") and filters out that only the set of keywords
func keywordPrefixes(kvset []Keyword) []Keyword {
kvs := []Keyword{}
for _, kv := range kvset {
kvs = append(kvs, kv.Prefix())
}
return kvs
}
// keyvalSelector takes an array of KeyVal ("keyword=value") and filters out
// that only the set of keywords
func keyvalSelector(keyval []KeyVal, keyset []Keyword) []KeyVal {
retList := []KeyVal{}
for _, kv := range keyval {
if InKeywordSlice(kv.Keyword(), keyset) {
if InKeywordSlice(kv.Keyword().Prefix(), keywordPrefixes(keyset)) {
retList = append(retList, kv)
}
}
@ -171,23 +182,23 @@ func keyValCopy(set []KeyVal) []KeyVal {
// Has the "keyword" present in the list of KeyVal, and returns the
// corresponding KeyVal, else an empty string.
func Has(keyvals []KeyVal, keyword string) KeyVal {
func Has(keyvals []KeyVal, keyword string) []KeyVal {
return HasKeyword(keyvals, Keyword(keyword))
}
// HasKeyword the "keyword" present in the list of KeyVal, and returns the
// corresponding KeyVal, else an empty string.
func HasKeyword(keyvals []KeyVal, keyword Keyword) KeyVal {
// This match is done on the Prefix of the keyword only.
func HasKeyword(keyvals []KeyVal, keyword Keyword) []KeyVal {
kvs := []KeyVal{}
for i := range keyvals {
if keyvals[i].Keyword() == keyword {
return keyvals[i]
if keyvals[i].Keyword().Prefix() == keyword.Prefix() {
kvs = append(kvs, keyvals[i])
}
}
return emptyKV
return kvs
}
var emptyKV = KeyVal("")
// MergeSet takes the current setKeyVals, and then applies the entryKeyVals
// such that the entry's values win. The union is returned.
func MergeSet(setKeyVals, entryKeyVals []string) []KeyVal {
@ -203,8 +214,11 @@ func MergeKeyValSet(setKeyVals, entryKeyVals []KeyVal) []KeyVal {
seenKeywords := []Keyword{}
for i := range retList {
word := retList[i].Keyword()
if ekv := HasKeyword(entryKeyVals, word); ekv != emptyKV {
retList[i] = ekv
for _, kv := range HasKeyword(entryKeyVals, word) {
// match on the keyword prefix and suffix here
if kv.Keyword() == word {
retList[i] = kv
}
}
seenKeywords = append(seenKeywords, word)
}

12
keywords_linux_test.go
View file

@ -51,12 +51,12 @@ func TestXattr(t *testing.T) {
t.Fatal(err)
}
// Check the directory
str, err := xattrKeywordFunc(dir, dirstat, nil)
kvs, err := xattrKeywordFunc(dir, dirstat, nil)
if err != nil {
t.Error(err)
}
if str == "" {
t.Errorf("expected a keyval; got %q", str)
if len(kvs) == 0 {
t.Errorf("expected a keyval; got none")
}
filestat, err := fh.Stat()
@ -64,12 +64,12 @@ func TestXattr(t *testing.T) {
t.Fatal(err)
}
// Check the regular file
str, err = xattrKeywordFunc(filepath.Join(dir, "file"), filestat, fh)
kvs, err = xattrKeywordFunc(filepath.Join(dir, "file"), filestat, fh)
if err != nil {
t.Error(err)
}
if str == "" {
t.Errorf("expected a keyval; got %q", str)
if len(kvs) == 0 {
t.Errorf("expected a keyval; got none")
}
linkstat, err := os.Lstat(filepath.Join(dir, "symlink"))

25
keywords_test.go
View file

@ -9,14 +9,20 @@ import (
func TestKeyValRoundtrip(t *testing.T) {
kv := KeyVal("xattr.security.selinux=dW5jb25maW5lZF91Om9iamVjdF9yOnVzZXJfaG9tZV90OnMwAA==")
expected := "xattr"
expected := "xattr.security.selinux"
got := string(kv.Keyword())
if got != expected {
t.Errorf("expected %q; got %q", expected, got)
}
expected = "xattr"
got = string(kv.Keyword().Prefix())
if got != expected {
t.Errorf("expected %q; got %q", expected, got)
}
expected = "security.selinux"
got = kv.KeywordSuffix()
got = kv.Keyword().Suffix()
if got != expected {
t.Errorf("expected %q; got %q", expected, got)
}
@ -27,7 +33,6 @@ func TestKeyValRoundtrip(t *testing.T) {
t.Errorf("expected %q; got %q", expected, got)
}
expected = "xattr.security.selinux=farts"
got = string(kv.NewValue("farts"))
if got != expected {
@ -97,8 +102,11 @@ func TestKeywordsTimeNano(t *testing.T) {
if err != nil {
t.Errorf("unexpected error while parsing '%q': %q", mtime, err)
}
if expected != got {
t.Errorf("keyword didn't match, expected '%s' got '%s'", expected, got)
if len(got) != 1 {
t.Errorf("expected 1 KeyVal, but got %d", len(got))
}
if expected != got[0] {
t.Errorf("keyword didn't match, expected '%s' got '%s'", expected, got[0])
}
}
}
@ -124,8 +132,11 @@ func TestKeywordsTimeTar(t *testing.T) {
if err != nil {
t.Errorf("unexpected error while parsing '%q': %q", mtime, err)
}
if expected != got {
t.Errorf("keyword didn't match, expected '%s' got '%s'", expected, got)
if len(got) != 1 {
t.Errorf("expected 1 KeyVal, but got %d", len(got))
}
if expected != got[0] {
t.Errorf("keyword didn't match, expected '%s' got '%s'", expected, got[0])
}
}
}

38
tar.go
View file

@ -5,11 +5,11 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"os"
"path/filepath"
"strings"
"github.com/Sirupsen/logrus"
"github.com/vbatts/go-mtree/pkg/govis"
)
@ -144,16 +144,16 @@ hdrloop:
// Keep track of which files are hardlinks so we can resolve them later
if hdr.Typeflag == tar.TypeLink {
linkFunc := KeywordFuncs["link"]
kv, err := linkFunc(hdr.Name, hdr.FileInfo(), nil)
keyFunc := KeywordFuncs["link"]
kvs, err := keyFunc(hdr.Name, hdr.FileInfo(), nil)
if err != nil {
log.Println(err)
break
logrus.Warn(err)
break // XXX is breaking an okay thing to do here?
}
linkname, err := govis.Unvis(KeyVal(kv).Value(), DefaultVisFlags)
linkname, err := govis.Unvis(KeyVal(kvs[0]).Value(), DefaultVisFlags)
if err != nil {
log.Println(err)
break
logrus.Warn(err)
break // XXX is breaking an okay thing to do here?
}
if _, ok := ts.hardlinks[linkname]; !ok {
ts.hardlinks[linkname] = []string{hdr.Name}
@ -164,19 +164,19 @@ hdrloop:
// now collect keywords on the file
for _, keyword := range ts.keywords {
if keyFunc, ok := KeywordFuncs[keyword]; ok {
if keyFunc, ok := KeywordFuncs[keyword.Prefix()]; ok {
// We can't extract directories on to disk, so "size" keyword
// is irrelevant for now
if hdr.FileInfo().IsDir() && keyword == "size" {
continue
}
val, err := keyFunc(hdr.Name, hdr.FileInfo(), tmpFile)
kvs, err := keyFunc(hdr.Name, hdr.FileInfo(), tmpFile)
if err != nil {
ts.setErr(err)
}
// for good measure, check that we actually get a value for a keyword
if val != "" {
e.Keywords = append(e.Keywords, val)
if len(kvs) > 0 && kvs[0] != "" {
e.Keywords = append(e.Keywords, kvs[0])
}
// don't forget to reset the reader
@ -196,13 +196,15 @@ hdrloop:
Type: SpecialType,
}
for _, setKW := range SetKeywords {
if keyFunc, ok := KeywordFuncs[setKW]; ok {
val, err := keyFunc(hdr.Name, hdr.FileInfo(), tmpFile)
if keyFunc, ok := KeywordFuncs[setKW.Prefix()]; ok {
kvs, err := keyFunc(hdr.Name, hdr.FileInfo(), tmpFile)
if err != nil {
ts.setErr(err)
}
if val != "" {
s.Keywords = append(s.Keywords, val)
for _, kv := range kvs {
if kv != "" {
s.Keywords = append(s.Keywords, kv)
}
}
if _, err := tmpFile.Seek(0, 0); err != nil {
tmpFile.Close()
@ -383,7 +385,7 @@ func resolveHardlinks(root *Entry, hardlinks map[string][]string, countlinks boo
if seen, ok := originals[base]; !ok {
basefile = root.Find(base)
if basefile == nil {
log.Printf("%s does not exist in this tree\n", base)
logrus.Printf("%s does not exist in this tree\n", base)
continue
}
originals[base] = basefile
@ -393,7 +395,7 @@ func resolveHardlinks(root *Entry, hardlinks map[string][]string, countlinks boo
for _, link := range links {
linkfile := root.Find(link)
if linkfile == nil {
log.Printf("%s does not exist in this tree\n", link)
logrus.Printf("%s does not exist in this tree\n", link)
continue
}
linkfile.Keywords = basefile.Keywords

23
update.go
View file

@ -3,6 +3,8 @@ package mtree
import (
"os"
"sort"
"github.com/Sirupsen/logrus"
)
// DefaultUpdateKeywords is the default set of keywords that can take updates to the files on disk
@ -11,7 +13,7 @@ var DefaultUpdateKeywords = []Keyword{
"gid",
"mode",
"time",
// TODO xattr
"xattr",
}
// Update attempts to set the attributes of root directory path, given the values of `keywords` in dh DirectoryHierarchy.
@ -36,7 +38,7 @@ func Update(root string, dh *DirectoryHierarchy, keywords []Keyword, fs FsEval)
} else if e.Name == "/unset" {
creator.curSet = nil
}
Debugf("%#v", e)
logrus.Debugf("%#v", e)
continue
case RelativeType, FullType:
e.Set = creator.curSet
@ -46,20 +48,21 @@ func Update(root string, dh *DirectoryHierarchy, keywords []Keyword, fs FsEval)
}
// filter the keywords to update on the file, from the keywords available for this entry:
var toCheck []KeyVal
toCheck = keyvalSelector(e.AllKeys(), keywords)
Debugf("toCheck(%q): %v", pathname, toCheck)
var kvToUpdate []KeyVal
kvToUpdate = keyvalSelector(e.AllKeys(), keywords)
logrus.Debugf("kvToUpdate(%q): %#v", pathname, kvToUpdate)
for _, kv := range toCheck {
if !InKeywordSlice(kv.Keyword(), keywords) {
for _, kv := range kvToUpdate {
if !InKeywordSlice(kv.Keyword().Prefix(), keywordPrefixes(keywords)) {
continue
}
ukFunc, ok := UpdateKeywordFuncs[kv.Keyword()]
logrus.Debugf("finding function for %q (%q)", kv.Keyword(), kv.Keyword().Prefix())
ukFunc, ok := UpdateKeywordFuncs[kv.Keyword().Prefix()]
if !ok {
Debugf("no UpdateKeywordFunc for %s; skipping", kv.Keyword())
logrus.Debugf("no UpdateKeywordFunc for %s; skipping", kv.Keyword())
continue
}
if _, err := ukFunc(pathname, kv.Value()); err != nil {
if _, err := ukFunc(kv.Keyword(), pathname, kv.Value()); err != nil {
results = append(results, InodeDelta{
diff: ErrorDifference,
path: pathname,

94
update_linux_test.go Normal file
View file

@ -0,0 +1,94 @@
package mtree
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/vbatts/go-mtree/xattr"
)
func init() {
//logrus.SetLevel(logrus.DebugLevel)
}
func TestXattrUpdate(t *testing.T) {
content := []byte("I know half of you half as well as I ought to")
// a bit dirty to create/destory a directory in cwd, but often /tmp is
// mounted tmpfs and doesn't support xattrs
dir, err := ioutil.TempDir(".", "test.xattr.restore.")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir) // clean up
tmpfn := filepath.Join(dir, "tmpfile")
if err := ioutil.WriteFile(tmpfn, content, 0666); err != nil {
t.Fatal(err)
}
if err := xattr.Set(dir, "user.test", []byte("directory")); err != nil {
t.Skip(fmt.Sprintf("skipping: %q does not support xattrs", dir))
}
if err := xattr.Set(tmpfn, "user.test", []byte("regular file")); err != nil {
t.Fatal(err)
}
// Walk this tempdir
dh, err := Walk(dir, nil, append(DefaultKeywords, []Keyword{"xattr", "sha1"}...), nil)
if err != nil {
t.Fatal(err)
}
// Now check that we're sane
res, err := Check(dir, dh, nil, nil)
if err != nil {
t.Fatal(err)
}
if len(res) != 0 {
t.Errorf("expecting no failures, but got %q", res)
}
if err := xattr.Set(tmpfn, "user.test", []byte("let it fly")); err != nil {
t.Fatal(err)
}
// Now check that we fail the check
res, err = Check(dir, dh, nil, nil)
if err != nil {
t.Fatal(err)
}
if len(res) == 0 {
t.Error("expected failures (like xattrs), but got none")
}
// restore the xattrs to original
res, err = Update(dir, dh, append(DefaultUpdateKeywords, "xattr"), nil)
if err != nil {
t.Error(err)
}
if len(res) != 0 {
t.Errorf("expecting no failures, but got %q", res)
}
// Now check that we're sane again
res, err = Check(dir, dh, nil, nil)
if err != nil {
t.Fatal(err)
}
if len(res) != 0 {
// pretty this shit up
buf, err := json.MarshalIndent(res, "", " ")
if err != nil {
t.Errorf("expecting no failures, but got %q", res)
} else {
t.Errorf("expecting no failures, but got %s", string(buf))
}
}
// TODO make a test for xattr here. Likely in the user space for privileges. Even still this may be prone to error for some tmpfs don't act right with xattrs. :-\
// I'd hate to have to t.Skip() a test rather than fail alltogether.
}

3
update_test.go
View file

@ -97,8 +97,9 @@ func TestUpdate(t *testing.T) {
buf, err := json.MarshalIndent(res, "", " ")
if err != nil {
t.Errorf("%#v", res)
}
} else {
t.Error(string(buf))
}
}
}

19
updatefuncs.go
View file

@ -6,12 +6,14 @@ import (
"strconv"
"strings"
"time"
"github.com/Sirupsen/logrus"
)
// UpdateKeywordFunc is the signature for a function that will restore a file's
// attributes. Where path is relative path to the file, and value to be
// restored to.
type UpdateKeywordFunc func(path string, value string) (os.FileInfo, error)
type UpdateKeywordFunc func(keyword Keyword, path string, value string) (os.FileInfo, error)
// UpdateKeywordFuncs is the registered list of functions to update file attributes.
// Keyed by the keyword as it would show up in the manifest
@ -21,9 +23,10 @@ var UpdateKeywordFuncs = map[Keyword]UpdateKeywordFunc{
"tar_time": tartimeUpdateKeywordFunc,
"uid": uidUpdateKeywordFunc,
"gid": gidUpdateKeywordFunc,
"xattr": xattrUpdateKeywordFunc,
}
func uidUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
func uidUpdateKeywordFunc(keyword Keyword, path, value string) (os.FileInfo, error) {
uid, err := strconv.Atoi(value)
if err != nil {
return nil, err
@ -34,7 +37,7 @@ func uidUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
return os.Lstat(path)
}
func gidUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
func gidUpdateKeywordFunc(keyword Keyword, path, value string) (os.FileInfo, error) {
gid, err := strconv.Atoi(value)
if err != nil {
return nil, err
@ -45,12 +48,12 @@ func gidUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
return os.Lstat(path)
}
func modeUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
func modeUpdateKeywordFunc(keyword Keyword, path, value string) (os.FileInfo, error) {
vmode, err := strconv.ParseInt(value, 8, 32)
if err != nil {
return nil, err
}
Debugf("path: %q, value: %q, vmode: %o", path, value, vmode)
logrus.Debugf("path: %q, value: %q, vmode: %o", path, value, vmode)
if err := os.Chmod(path, os.FileMode(vmode)); err != nil {
return nil, err
}
@ -60,7 +63,7 @@ func modeUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
// since tar_time will only be second level precision, then when restoring the
// filepath from a tar_time, then compare the seconds first and only Chtimes if
// the seconds value is different.
func tartimeUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
func tartimeUpdateKeywordFunc(keyword Keyword, path, value string) (os.FileInfo, error) {
info, err := os.Lstat(path)
if err != nil {
return nil, err
@ -89,7 +92,7 @@ func tartimeUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
}
// this is nano second precision
func timeUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
func timeUpdateKeywordFunc(keyword Keyword, path, value string) (os.FileInfo, error) {
v := strings.SplitN(value, ".", 2)
if len(v) != 2 {
return nil, fmt.Errorf("expected a number like 1469104727.871937272")
@ -98,7 +101,7 @@ func timeUpdateKeywordFunc(path, value string) (os.FileInfo, error) {
if err != nil {
return nil, fmt.Errorf("expected nano seconds, but got %q", v[0]+v[1])
}
Debugf("arg: %q; nsec: %q", v[0]+v[1], nsec)
logrus.Debugf("arg: %q; nsec: %q", v[0]+v[1], nsec)
vtime := time.Unix(0, nsec)
if err := os.Chtimes(path, vtime, vtime); err != nil {

21
updatefuncs_linux.go Normal file
View file

@ -0,0 +1,21 @@
// +build linux
package mtree
import (
"encoding/base64"
"os"
"github.com/vbatts/go-mtree/xattr"
)
func xattrUpdateKeywordFunc(keyword Keyword, path, value string) (os.FileInfo, error) {
buf, err := base64.StdEncoding.DecodeString(value)
if err != nil {
return nil, err
}
if err := xattr.Set(path, keyword.Suffix(), buf); err != nil {
return nil, err
}
return os.Lstat(path)
}

9
updatefuncs_unsupported.go Normal file
View file

@ -0,0 +1,9 @@
// +build !linux
package mtree
import "os"
func xattrUpdateKeywordFunc(keyword Keyword, path, value string) (os.FileInfo, error) {
return os.Lstat(path)
}

38
walk.go
View file

@ -101,15 +101,19 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval
defer fh.Close()
r = fh
}
keywordFunc, ok := KeywordFuncs[keyword]
keyFunc, ok := KeywordFuncs[keyword.Prefix()]
if !ok {
return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
return fmt.Errorf("Unknown keyword %q for file %q", keyword.Prefix(), path)
}
if str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r); err == nil && str != "" {
e.Keywords = append(e.Keywords, str)
} else if err != nil {
kvs, err := creator.fs.KeywordFunc(keyFunc)(path, info, r)
if err != nil {
return err
}
for _, kv := range kvs {
if kv != "" {
e.Keywords = append(e.Keywords, kv)
}
}
return nil
}()
if err != nil {
@ -132,16 +136,18 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval
defer fh.Close()
r = fh
}
keywordFunc, ok := KeywordFuncs[keyword]
keyFunc, ok := KeywordFuncs[keyword.Prefix()]
if !ok {
return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
return fmt.Errorf("Unknown keyword %q for file %q", keyword.Prefix(), path)
}
str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r)
kvs, err := creator.fs.KeywordFunc(keyFunc)(path, info, r)
if err != nil {
return err
}
if str != "" {
klist = append(klist, str)
for _, kv := range kvs {
if kv != "" {
klist = append(klist, kv)
}
}
return nil
}()
@ -190,16 +196,18 @@ func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval
defer fh.Close()
r = fh
}
keywordFunc, ok := KeywordFuncs[keyword]
keyFunc, ok := KeywordFuncs[keyword.Prefix()]
if !ok {
return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
return fmt.Errorf("Unknown keyword %q for file %q", keyword.Prefix(), path)
}
str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r)
kvs, err := creator.fs.KeywordFunc(keyFunc)(path, info, r)
if err != nil {
return err
}
if str != "" && !inKeyValSlice(str, creator.curSet.Keywords) {
e.Keywords = append(e.Keywords, str)
for _, kv := range kvs {
if kv != "" && !inKeyValSlice(kv, creator.curSet.Keywords) {
e.Keywords = append(e.Keywords, kv)
}
}
return nil
}()