From 4f1bde4d13785f6ff58a4773cedfbaac600dc6e0 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 3 Mar 2015 14:23:04 -0500 Subject: [PATCH] tar/asm: FileType entry with crc64 checksum --- tar/asm/assemble.go | 11 +++++++- tar/asm/assemble_test.go | 41 +++++++++++++++++++++++++++- tar/asm/disassemble.go | 37 +++++++++++++------------ tar/asm/getter.go | 23 ++++++++++------ tar/asm/getter_test.go | 59 +++++++++++++++++++++++++--------------- 5 files changed, 121 insertions(+), 50 deletions(-) diff --git a/tar/asm/assemble.go b/tar/asm/assemble.go index 7f20a16..b0804da 100644 --- a/tar/asm/assemble.go +++ b/tar/asm/assemble.go @@ -1,6 +1,9 @@ package asm import ( + "bytes" + "fmt" + "hash/crc64" "io" "github.com/vbatts/tar-split/tar/storage" @@ -42,10 +45,16 @@ func NewOutputTarStream(fg FileGetter, up storage.Unpacker) io.ReadCloser { break } defer fh.Close() - if _, err := io.Copy(pw, fh); err != nil { + c := crc64.New(crcTable) + tRdr := io.TeeReader(fh, c) + if _, err := io.Copy(pw, tRdr); err != nil { pw.CloseWithError(err) break } + if !bytes.Equal(c.Sum(nil), entry.Payload) { + pw.CloseWithError(fmt.Errorf("file integrity checksum failed for %q", entry.Name)) + break + } } } }() diff --git a/tar/asm/assemble_test.go b/tar/asm/assemble_test.go index 81dff9f..04afcfe 100644 --- a/tar/asm/assemble_test.go +++ b/tar/asm/assemble_test.go @@ -1,9 +1,48 @@ package asm -import "testing" +import ( + "testing" + + "github.com/vbatts/tar-split/tar/storage" +) + +var entries = storage.Entries{ + storage.Entry{ + Type: storage.SegmentType, + Payload: []byte("how"), + Position: 0, + }, + storage.Entry{ + Type: storage.SegmentType, + Payload: []byte("y'all"), + Position: 1, + }, + storage.Entry{ + Type: storage.FileType, + Name: "./hurr.txt", + Payload: []byte("deadbeef"), + Size: 8, + Position: 2, + }, + storage.Entry{ + Type: storage.SegmentType, + Payload: []byte("doin"), + Position: 3, + }, + storage.Entry{ + Type: storage.FileType, + Name: "./ermahgerd.txt", + Payload: []byte("cafebabe"), + Size: 8, + Position: 4, + }, +} func TestNewOutputTarStream(t *testing.T) { // TODO disassembly fgp := NewBufferFileGetPutter() _ = NewOutputTarStream(fgp, nil) } + +func TestNewInputTarStream(t *testing.T) { +} diff --git a/tar/asm/disassemble.go b/tar/asm/disassemble.go index acab3a9..1219c89 100644 --- a/tar/asm/disassemble.go +++ b/tar/asm/disassemble.go @@ -36,6 +36,10 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp FilePutter) (io.Reader, pR, pW := io.Pipe() outputRdr := io.TeeReader(r, pW) + if fp == nil { + fp = NewDiscardFilePutter() + } + go func() { tr := tar.NewReader(outputRdr) tr.RawAccounting = true @@ -67,31 +71,28 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp FilePutter) (io.Reader, pW.CloseWithError(err) } + var csum []byte if hdr.Size > 0 { - if fp != nil { - // if there is a file payload to write, then write the file to the FilePutter - fileRdr, fileWrtr := io.Pipe() - go func() { - if err := fp.Put(hdr.Name, fileRdr); err != nil { - pW.CloseWithError(err) - } - }() - if _, err = io.Copy(fileWrtr, tr); err != nil { + // if there is a file payload to write, then write the file to the FilePutter + fileRdr, fileWrtr := io.Pipe() + go func() { + var err error + csum, err = fp.Put(hdr.Name, fileRdr) + if err != nil { pW.CloseWithError(err) - return - } - } else { - if _, err := io.Copy(ioutil.Discard, tr); err != nil { - pW.CloseWithError(err) - return } + }() + if _, err = io.Copy(fileWrtr, tr); err != nil { + pW.CloseWithError(err) + return } } // File entries added, regardless of size if _, err := p.AddEntry(storage.Entry{ - Type: storage.FileType, - Name: hdr.Name, - Size: hdr.Size, + Type: storage.FileType, + Name: hdr.Name, + Size: hdr.Size, + Payload: csum, }); err != nil { pW.CloseWithError(err) } diff --git a/tar/asm/getter.go b/tar/asm/getter.go index 2b94c89..b11ff7b 100644 --- a/tar/asm/getter.go +++ b/tar/asm/getter.go @@ -3,6 +3,7 @@ package asm import ( "bytes" "errors" + "hash/crc64" "io" "io/ioutil" "os" @@ -16,7 +17,7 @@ type FileGetter interface { type FilePutter interface { // Put returns a stream for the provided file path - Put(string, io.Reader) error + Put(string, io.Reader) ([]byte, error) } type FileGetPutter interface { @@ -50,13 +51,15 @@ func (bfgp bufferFileGetPutter) Get(name string) (io.ReadCloser, error) { return &readCloserWrapper{b}, nil } -func (bfgp *bufferFileGetPutter) Put(name string, r io.Reader) error { +func (bfgp *bufferFileGetPutter) Put(name string, r io.Reader) ([]byte, error) { + c := crc64.New(crcTable) + tRdr := io.TeeReader(r, c) b := bytes.NewBuffer([]byte{}) - if _, err := io.Copy(b, r); err != nil { - return err + if _, err := io.Copy(b, tRdr); err != nil { + return nil, err } bfgp.files[name] = b.Bytes() - return nil + return c.Sum(nil), nil } type readCloserWrapper struct { @@ -83,7 +86,11 @@ func NewDiscardFilePutter() FilePutter { type bitBucketFilePutter struct { } -func (bbfp *bitBucketFilePutter) Put(name string, r io.Reader) error { - _, err := io.Copy(ioutil.Discard, r) - return err +func (bbfp *bitBucketFilePutter) Put(name string, r io.Reader) ([]byte, error) { + c := crc64.New(crcTable) + tRdr := io.TeeReader(r, c) + _, err := io.Copy(ioutil.Discard, tRdr) + return c.Sum(nil), err } + +var crcTable = crc64.MakeTable(crc64.ISO) diff --git a/tar/asm/getter_test.go b/tar/asm/getter_test.go index 9adb171..f8d25be 100644 --- a/tar/asm/getter_test.go +++ b/tar/asm/getter_test.go @@ -8,40 +8,55 @@ import ( func TestGetter(t *testing.T) { fgp := NewBufferFileGetPutter() - files := map[string][]byte{ - "file1.txt": []byte("foo"), - "file2.txt": []byte("bar"), + files := map[string]map[string][]byte{ + "file1.txt": {"foo": []byte{60, 60, 48, 48, 0, 0, 0, 0}}, + "file2.txt": {"bar": []byte{45, 196, 22, 240, 0, 0, 0, 0}}, } for n, b := range files { - if err := fgp.Put(n, bytes.NewBuffer(b)); err != nil { - t.Error(err) + for body, sum := range b { + csum, err := fgp.Put(n, bytes.NewBufferString(body)) + if err != nil { + t.Error(err) + } + if !bytes.Equal(csum, sum) { + t.Errorf("checksum: expected 0x%x; got 0x%x", sum, csum) + } } } for n, b := range files { - r, err := fgp.Get(n) - if err != nil { - t.Error(err) - } - buf, err := ioutil.ReadAll(r) - if err != nil { - t.Error(err) - } - if string(b) != string(buf) { - t.Errorf("expected %q, got %q", string(b), string(buf)) + for body := range b { + r, err := fgp.Get(n) + if err != nil { + t.Error(err) + } + buf, err := ioutil.ReadAll(r) + if err != nil { + t.Error(err) + } + if body != string(buf) { + t.Errorf("expected %q, got %q", body, string(buf)) + } } } } func TestPutter(t *testing.T) { fp := NewDiscardFilePutter() - files := map[string][]byte{ - "file1.txt": []byte("foo"), - "file2.txt": []byte("bar"), - "file3.txt": []byte("baz"), - "file4.txt": []byte("bif"), + // map[filename]map[body]crc64sum + files := map[string]map[string][]byte{ + "file1.txt": {"foo": []byte{60, 60, 48, 48, 0, 0, 0, 0}}, + "file2.txt": {"bar": []byte{45, 196, 22, 240, 0, 0, 0, 0}}, + "file3.txt": {"baz": []byte{32, 68, 22, 240, 0, 0, 0, 0}}, + "file4.txt": {"bif": []byte{48, 9, 150, 240, 0, 0, 0, 0}}, } for n, b := range files { - if err := fp.Put(n, bytes.NewBuffer(b)); err != nil { - t.Error(err) + for body, sum := range b { + csum, err := fp.Put(n, bytes.NewBufferString(body)) + if err != nil { + t.Error(err) + } + if !bytes.Equal(csum, sum) { + t.Errorf("checksum on %q: expected %v; got %v", n, sum, csum) + } } } }