vbatts
/
tar-split
forked from mirrors/tar-split
1
0
Fork 0
Code Releases Activity

tar/asm: FileType entry with crc64 checksum

This commit is contained in:
Vincent Batts 2015-03-03 14:23:04 -05:00
parent 962589aca7
commit 4f1bde4d13
5 changed files with 121 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
tar/asm/assemble.go
View File

@ -1,6 +1,9 @@
package asm
import (
"bytes"
"fmt"
"hash/crc64"
"io"
"github.com/vbatts/tar-split/tar/storage"
@ -42,10 +45,16 @@ func NewOutputTarStream(fg FileGetter, up storage.Unpacker) io.ReadCloser {
break
}
defer fh.Close()
if _, err := io.Copy(pw, fh); err != nil {
c := crc64.New(crcTable)
tRdr := io.TeeReader(fh, c)
if _, err := io.Copy(pw, tRdr); err != nil {
pw.CloseWithError(err)
break
}
if !bytes.Equal(c.Sum(nil), entry.Payload) {
pw.CloseWithError(fmt.Errorf("file integrity checksum failed for %q", entry.Name))
break
}
}
}
}()

41
tar/asm/assemble_test.go
View File

@ -1,9 +1,48 @@
package asm
import "testing"
import (
"testing"
"github.com/vbatts/tar-split/tar/storage"
)
var entries = storage.Entries{
storage.Entry{
Type: storage.SegmentType,
Payload: []byte("how"),
Position: 0,
},
storage.Entry{
Type: storage.SegmentType,
Payload: []byte("y'all"),
Position: 1,
},
storage.Entry{
Type: storage.FileType,
Name: "./hurr.txt",
Payload: []byte("deadbeef"),
Size: 8,
Position: 2,
},
storage.Entry{
Type: storage.SegmentType,
Payload: []byte("doin"),
Position: 3,
},
storage.Entry{
Type: storage.FileType,
Name: "./ermahgerd.txt",
Payload: []byte("cafebabe"),
Size: 8,
Position: 4,
},
}
func TestNewOutputTarStream(t *testing.T) {
// TODO disassembly
fgp := NewBufferFileGetPutter()
_ = NewOutputTarStream(fgp, nil)
}
func TestNewInputTarStream(t *testing.T) {
}

37
tar/asm/disassemble.go
View File

@ -36,6 +36,10 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp FilePutter) (io.Reader,
pR, pW := io.Pipe()
outputRdr := io.TeeReader(r, pW)
if fp == nil {
fp = NewDiscardFilePutter()
}
go func() {
tr := tar.NewReader(outputRdr)
tr.RawAccounting = true
@ -67,31 +71,28 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp FilePutter) (io.Reader,
pW.CloseWithError(err)
}
var csum []byte
if hdr.Size > 0 {
if fp != nil {
// if there is a file payload to write, then write the file to the FilePutter
fileRdr, fileWrtr := io.Pipe()
go func() {
if err := fp.Put(hdr.Name, fileRdr); err != nil {
pW.CloseWithError(err)
}
}()
if _, err = io.Copy(fileWrtr, tr); err != nil {
// if there is a file payload to write, then write the file to the FilePutter
fileRdr, fileWrtr := io.Pipe()
go func() {
var err error
csum, err = fp.Put(hdr.Name, fileRdr)
if err != nil {
pW.CloseWithError(err)
return
}
} else {
if _, err := io.Copy(ioutil.Discard, tr); err != nil {
pW.CloseWithError(err)
return
}
}()
if _, err = io.Copy(fileWrtr, tr); err != nil {
pW.CloseWithError(err)
return
}
}
// File entries added, regardless of size
if _, err := p.AddEntry(storage.Entry{
Type: storage.FileType,
Name: hdr.Name,
Size: hdr.Size,
Type: storage.FileType,
Name: hdr.Name,
Size: hdr.Size,
Payload: csum,
}); err != nil {
pW.CloseWithError(err)
}

23
tar/asm/getter.go
View File

@ -3,6 +3,7 @@ package asm
import (
"bytes"
"errors"
"hash/crc64"
"io"
"io/ioutil"
"os"
@ -16,7 +17,7 @@ type FileGetter interface {
type FilePutter interface {
// Put returns a stream for the provided file path
Put(string, io.Reader) error
Put(string, io.Reader) ([]byte, error)
}
type FileGetPutter interface {
@ -50,13 +51,15 @@ func (bfgp bufferFileGetPutter) Get(name string) (io.ReadCloser, error) {
return &readCloserWrapper{b}, nil
}
func (bfgp *bufferFileGetPutter) Put(name string, r io.Reader) error {
func (bfgp *bufferFileGetPutter) Put(name string, r io.Reader) ([]byte, error) {
c := crc64.New(crcTable)
tRdr := io.TeeReader(r, c)
b := bytes.NewBuffer([]byte{})
if _, err := io.Copy(b, r); err != nil {
return err
if _, err := io.Copy(b, tRdr); err != nil {
return nil, err
}
bfgp.files[name] = b.Bytes()
return nil
return c.Sum(nil), nil
}
type readCloserWrapper struct {
@ -83,7 +86,11 @@ func NewDiscardFilePutter() FilePutter {
type bitBucketFilePutter struct {
}
func (bbfp *bitBucketFilePutter) Put(name string, r io.Reader) error {
_, err := io.Copy(ioutil.Discard, r)
return err
func (bbfp *bitBucketFilePutter) Put(name string, r io.Reader) ([]byte, error) {
c := crc64.New(crcTable)
tRdr := io.TeeReader(r, c)
_, err := io.Copy(ioutil.Discard, tRdr)
return c.Sum(nil), err
}
var crcTable = crc64.MakeTable(crc64.ISO)

59
tar/asm/getter_test.go
View File

@ -8,40 +8,55 @@ import (
func TestGetter(t *testing.T) {
fgp := NewBufferFileGetPutter()
files := map[string][]byte{
"file1.txt": []byte("foo"),
"file2.txt": []byte("bar"),
files := map[string]map[string][]byte{
"file1.txt": {"foo": []byte{60, 60, 48, 48, 0, 0, 0, 0}},
"file2.txt": {"bar": []byte{45, 196, 22, 240, 0, 0, 0, 0}},
}
for n, b := range files {
if err := fgp.Put(n, bytes.NewBuffer(b)); err != nil {
t.Error(err)
for body, sum := range b {
csum, err := fgp.Put(n, bytes.NewBufferString(body))
if err != nil {
t.Error(err)
}
if !bytes.Equal(csum, sum) {
t.Errorf("checksum: expected 0x%x; got 0x%x", sum, csum)
}
}
}
for n, b := range files {
r, err := fgp.Get(n)
if err != nil {
t.Error(err)
}
buf, err := ioutil.ReadAll(r)
if err != nil {
t.Error(err)
}
if string(b) != string(buf) {
t.Errorf("expected %q, got %q", string(b), string(buf))
for body := range b {
r, err := fgp.Get(n)
if err != nil {
t.Error(err)
}
buf, err := ioutil.ReadAll(r)
if err != nil {
t.Error(err)
}
if body != string(buf) {
t.Errorf("expected %q, got %q", body, string(buf))
}
}
}
}
func TestPutter(t *testing.T) {
fp := NewDiscardFilePutter()
files := map[string][]byte{
"file1.txt": []byte("foo"),
"file2.txt": []byte("bar"),
"file3.txt": []byte("baz"),
"file4.txt": []byte("bif"),
// map[filename]map[body]crc64sum
files := map[string]map[string][]byte{
"file1.txt": {"foo": []byte{60, 60, 48, 48, 0, 0, 0, 0}},
"file2.txt": {"bar": []byte{45, 196, 22, 240, 0, 0, 0, 0}},
"file3.txt": {"baz": []byte{32, 68, 22, 240, 0, 0, 0, 0}},
"file4.txt": {"bif": []byte{48, 9, 150, 240, 0, 0, 0, 0}},
}
for n, b := range files {
if err := fp.Put(n, bytes.NewBuffer(b)); err != nil {
t.Error(err)
for body, sum := range b {
csum, err := fp.Put(n, bytes.NewBufferString(body))
if err != nil {
t.Error(err)
}
if !bytes.Equal(csum, sum) {
t.Errorf("checksum on %q: expected %v; got %v", n, sum, csum)
}
}
}
}