vbatts/tar-split
1
0
Fork 0
forked from mirrors/tar-split
Code Releases Activity

tar: mv the Getter to tar/storage

Browse source
This commit is contained in:
Vincent Batts 2015-03-09 13:20:26 -04:00
parent e045daf0b0
commit d8ebf3c0a7
5 changed files with 18 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

6
tar/asm/assemble.go
View file

@ -12,11 +12,11 @@ import (
// NewOutputTarStream returns an io.ReadCloser that is an assemble tar archive
// stream.
//
// It takes a FileGetter, for mapping the file payloads that are to be read in,
// It takes a storage.FileGetter, for mapping the file payloads that are to be read in,
// and a storage.Unpacker, which has access to the rawbytes and file order
// metadata. With the combination of these two items, a precise assembled Tar
// archive is possible.
func NewOutputTarStream(fg FileGetter, up storage.Unpacker) io.ReadCloser {
func NewOutputTarStream(fg storage.FileGetter, up storage.Unpacker) io.ReadCloser {
// ... Since these are interfaces, this is possible, so let's not have a nil pointer
if fg == nil || up == nil {
return nil
@ -45,7 +45,7 @@ func NewOutputTarStream(fg FileGetter, up storage.Unpacker) io.ReadCloser {
break
}
defer fh.Close()
c := crc64.New(crcTable)
c := crc64.New(storage.CRCTable)
tRdr := io.TeeReader(fh, c)
if _, err := io.Copy(pw, tRdr); err != nil {
pw.CloseWithError(err)

4
tar/asm/assemble_test.go
View file

@ -39,7 +39,7 @@ var entries = []struct {
}
func TestTarStreamOld(t *testing.T) {
fgp := NewBufferFileGetPutter()
fgp := storage.NewBufferFileGetPutter()
// first lets prep a GetPutter and Packer
for i := range entries {
@ -88,7 +88,7 @@ func TestTarStream(t *testing.T) {
// Setup where we'll store the metadata
w := bytes.NewBuffer([]byte{})
sp := storage.NewJsonPacker(w)
fgp := NewBufferFileGetPutter()
fgp := storage.NewBufferFileGetPutter()
// wrap the disassembly stream
tarStream, err := NewInputTarStream(gzRdr, sp, fgp)

14
tar/asm/disassemble.go
View file

@ -14,19 +14,19 @@ import (
// In the middle it will pack the segments and file metadata to storage.Packer
// `p`.
//
// The the FilePutter is where payload of files in the stream are stashed. If
// this stashing is not needed, fp can be nil or use NewDiscardFilePutter.
func NewInputTarStream(r io.Reader, p storage.Packer, fp FilePutter) (io.Reader, error) {
// The the storage.FilePutter is where payload of files in the stream are stashed. If
// this stashing is not needed, fp can be nil or use storage.NewDiscardFilePutter.
func NewInputTarStream(r io.Reader, p storage.Packer, fp storage.FilePutter) (io.Reader, error) {
// What to do here... folks will want their own access to the Reader that is
// their tar archive stream, but we'll need that same stream to use our
// forked 'archive/tar'.
// Perhaps do an io.TeeReader that hand back an io.Reader for them to read
// from, and we'll mitm the stream to store metadata.
// We'll need a FilePutter too ...
// We'll need a storage.FilePutter too ...
// Another concern, whether to do any FilePutter operations, such that we
// Another concern, whether to do any storage.FilePutter operations, such that we
// don't extract any amount of the archive. But then again, we're not making
// files/directories, hardlinks, etc. Just writing the io to the FilePutter.
// files/directories, hardlinks, etc. Just writing the io to the storage.FilePutter.
// Perhaps we have a DiscardFilePutter that is a bit bucket.
// we'll return the pipe reader, since TeeReader does not buffer and will
@ -38,7 +38,7 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp FilePutter) (io.Reader,
// we need a putter that will generate the crc64 sums of file payloads
if fp == nil {
fp = NewDiscardFilePutter()
fp = storage.NewDiscardFilePutter()
}
go func() {

97
tar/asm/getter.go
View file

@ -1,97 +0,0 @@
package asm
import (
"bytes"
"errors"
"hash/crc64"
"io"
"io/ioutil"
"os"
"path"
)
type FileGetter interface {
// Get returns a stream for the provided file path
Get(string) (io.ReadCloser, error)
}
type FilePutter interface {
// Put returns the crc64 checksum for the provided file
Put(string, io.Reader) (int64, []byte, error)
}
type FileGetPutter interface {
FileGetter
FilePutter
}
// NewPathFileGetter returns a FileGetter that is for files relative to path relpath.
func NewPathFileGetter(relpath string) FileGetter {
return &pathFileGetter{root: relpath}
}
type pathFileGetter struct {
root string
}
func (pfg pathFileGetter) Get(filename string) (io.ReadCloser, error) {
// FIXME might should have a check for '../../../../etc/passwd' attempts?
return os.Open(path.Join(pfg.root, filename))
}
type bufferFileGetPutter struct {
files map[string][]byte
}
func (bfgp bufferFileGetPutter) Get(name string) (io.ReadCloser, error) {
if _, ok := bfgp.files[name]; !ok {
return nil, errors.New("no such file")
}
b := bytes.NewBuffer(bfgp.files[name])
return &readCloserWrapper{b}, nil
}
func (bfgp *bufferFileGetPutter) Put(name string, r io.Reader) (int64, []byte, error) {
c := crc64.New(crcTable)
tRdr := io.TeeReader(r, c)
b := bytes.NewBuffer([]byte{})
i, err := io.Copy(b, tRdr)
if err != nil {
return 0, nil, err
}
bfgp.files[name] = b.Bytes()
return i, c.Sum(nil), nil
}
type readCloserWrapper struct {
io.Reader
}
func (w *readCloserWrapper) Close() error { return nil }
// NewBufferFileGetPutter is simple in memory FileGetPutter
//
// Implication is this is memory intensive...
// Probably best for testing or light weight cases.
func NewBufferFileGetPutter() FileGetPutter {
return &bufferFileGetPutter{
files: map[string][]byte{},
}
}
// NewDiscardFilePutter is a bit bucket FilePutter
func NewDiscardFilePutter() FilePutter {
return &bitBucketFilePutter{}
}
type bitBucketFilePutter struct {
}
func (bbfp *bitBucketFilePutter) Put(name string, r io.Reader) (int64, []byte, error) {
c := crc64.New(crcTable)
tRdr := io.TeeReader(r, c)
i, err := io.Copy(ioutil.Discard, tRdr)
return i, c.Sum(nil), err
}
var crcTable = crc64.MakeTable(crc64.ISO)

62
tar/asm/getter_test.go
View file

@ -1,62 +0,0 @@
package asm
import (
"bytes"
"io/ioutil"
"testing"
)
func TestGetter(t *testing.T) {
fgp := NewBufferFileGetPutter()
files := map[string]map[string][]byte{
"file1.txt": {"foo": []byte{60, 60, 48, 48, 0, 0, 0, 0}},
"file2.txt": {"bar": []byte{45, 196, 22, 240, 0, 0, 0, 0}},
}
for n, b := range files {
for body, sum := range b {
_, csum, err := fgp.Put(n, bytes.NewBufferString(body))
if err != nil {
t.Error(err)
}
if !bytes.Equal(csum, sum) {
t.Errorf("checksum: expected 0x%x; got 0x%x", sum, csum)
}
}
}
for n, b := range files {
for body := range b {
r, err := fgp.Get(n)
if err != nil {
t.Error(err)
}
buf, err := ioutil.ReadAll(r)
if err != nil {
t.Error(err)
}
if body != string(buf) {
t.Errorf("expected %q, got %q", body, string(buf))
}
}
}
}
func TestPutter(t *testing.T) {
fp := NewDiscardFilePutter()
// map[filename]map[body]crc64sum
files := map[string]map[string][]byte{
"file1.txt": {"foo": []byte{60, 60, 48, 48, 0, 0, 0, 0}},
"file2.txt": {"bar": []byte{45, 196, 22, 240, 0, 0, 0, 0}},
"file3.txt": {"baz": []byte{32, 68, 22, 240, 0, 0, 0, 0}},
"file4.txt": {"bif": []byte{48, 9, 150, 240, 0, 0, 0, 0}},
}
for n, b := range files {
for body, sum := range b {
_, csum, err := fp.Put(n, bytes.NewBufferString(body))
if err != nil {
t.Error(err)
}
if !bytes.Equal(csum, sum) {
t.Errorf("checksum on %q: expected %v; got %v", n, sum, csum)
}
}
}
}