tar/asm: FileType entry with crc64 checksum

tar/asm/assemble.go

@@ -1,6 +1,9 @@
 package asm
 
 import (
+	"bytes"
+	"fmt"
+	"hash/crc64"
 	"io"
 
 	"github.com/vbatts/tar-split/tar/storage"
@@ -42,10 +45,16 @@ func NewOutputTarStream(fg FileGetter, up storage.Unpacker) io.ReadCloser {
 					break
 				}
 				defer fh.Close()
-				if _, err := io.Copy(pw, fh); err != nil {
+				c := crc64.New(crcTable)
+				tRdr := io.TeeReader(fh, c)
+				if _, err := io.Copy(pw, tRdr); err != nil {
 					pw.CloseWithError(err)
 					break
 				}
+				if !bytes.Equal(c.Sum(nil), entry.Payload) {
+					pw.CloseWithError(fmt.Errorf("file integrity checksum failed for %q", entry.Name))
+					break
+				}
 			}
 		}
 	}()

tar/asm/assemble_test.go

@@ -1,9 +1,48 @@
 package asm
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/vbatts/tar-split/tar/storage"
+)
+
+var entries = storage.Entries{
+	storage.Entry{
+		Type:     storage.SegmentType,
+		Payload:  []byte("how"),
+		Position: 0,
+	},
+	storage.Entry{
+		Type:     storage.SegmentType,
+		Payload:  []byte("y'all"),
+		Position: 1,
+	},
+	storage.Entry{
+		Type:     storage.FileType,
+		Name:     "./hurr.txt",
+		Payload:  []byte("deadbeef"),
+		Size:     8,
+		Position: 2,
+	},
+	storage.Entry{
+		Type:     storage.SegmentType,
+		Payload:  []byte("doin"),
+		Position: 3,
+	},
+	storage.Entry{
+		Type:     storage.FileType,
+		Name:     "./ermahgerd.txt",
+		Payload:  []byte("cafebabe"),
+		Size:     8,
+		Position: 4,
+	},
+}
 
 func TestNewOutputTarStream(t *testing.T) {
 	// TODO disassembly
 	fgp := NewBufferFileGetPutter()
 	_ = NewOutputTarStream(fgp, nil)
 }
+
+func TestNewInputTarStream(t *testing.T) {
+}

tar/asm/disassemble.go

@@ -36,6 +36,10 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp FilePutter) (io.Reader,
 	pR, pW := io.Pipe()
 	outputRdr := io.TeeReader(r, pW)
 
+	if fp == nil {
+		fp = NewDiscardFilePutter()
+	}
+
 	go func() {
 		tr := tar.NewReader(outputRdr)
 		tr.RawAccounting = true
@@ -67,31 +71,28 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp FilePutter) (io.Reader,
 				pW.CloseWithError(err)
 			}
 
+			var csum []byte
 			if hdr.Size > 0 {
-				if fp != nil {
+				// if there is a file payload to write, then write the file to the FilePutter
-					// if there is a file payload to write, then write the file to the FilePutter
+				fileRdr, fileWrtr := io.Pipe()
-					fileRdr, fileWrtr := io.Pipe()
+				go func() {
-					go func() {
+					var err error
-						if err := fp.Put(hdr.Name, fileRdr); err != nil {
+					csum, err = fp.Put(hdr.Name, fileRdr)
-							pW.CloseWithError(err)
+					if err != nil {
-						}
-					}()
-					if _, err = io.Copy(fileWrtr, tr); err != nil {
 						pW.CloseWithError(err)
-						return
-					}
-				} else {
-					if _, err := io.Copy(ioutil.Discard, tr); err != nil {
-						pW.CloseWithError(err)
-						return
 					}
+				}()
+				if _, err = io.Copy(fileWrtr, tr); err != nil {
+					pW.CloseWithError(err)
+					return
 				}
 			}
 			// File entries added, regardless of size
 			if _, err := p.AddEntry(storage.Entry{
-				Type: storage.FileType,
+				Type:    storage.FileType,
-				Name: hdr.Name,
+				Name:    hdr.Name,
-				Size: hdr.Size,
+				Size:    hdr.Size,
+				Payload: csum,
 			}); err != nil {
 				pW.CloseWithError(err)
 			}

tar/asm/getter.go

@@ -3,6 +3,7 @@ package asm
 import (
 	"bytes"
 	"errors"
+	"hash/crc64"
 	"io"
 	"io/ioutil"
 	"os"
@@ -16,7 +17,7 @@ type FileGetter interface {
 
 type FilePutter interface {
 	// Put returns a stream for the provided file path
-	Put(string, io.Reader) error
+	Put(string, io.Reader) ([]byte, error)
 }
 
 type FileGetPutter interface {
@@ -50,13 +51,15 @@ func (bfgp bufferFileGetPutter) Get(name string) (io.ReadCloser, error) {
 	return &readCloserWrapper{b}, nil
 }
 
-func (bfgp *bufferFileGetPutter) Put(name string, r io.Reader) error {
+func (bfgp *bufferFileGetPutter) Put(name string, r io.Reader) ([]byte, error) {
+	c := crc64.New(crcTable)
+	tRdr := io.TeeReader(r, c)
 	b := bytes.NewBuffer([]byte{})
-	if _, err := io.Copy(b, r); err != nil {
+	if _, err := io.Copy(b, tRdr); err != nil {
-		return err
+		return nil, err
 	}
 	bfgp.files[name] = b.Bytes()
-	return nil
+	return c.Sum(nil), nil
 }
 
 type readCloserWrapper struct {
@@ -83,7 +86,11 @@ func NewDiscardFilePutter() FilePutter {
 type bitBucketFilePutter struct {
 }
 
-func (bbfp *bitBucketFilePutter) Put(name string, r io.Reader) error {
+func (bbfp *bitBucketFilePutter) Put(name string, r io.Reader) ([]byte, error) {
-	_, err := io.Copy(ioutil.Discard, r)
+	c := crc64.New(crcTable)
-	return err
+	tRdr := io.TeeReader(r, c)
+	_, err := io.Copy(ioutil.Discard, tRdr)
+	return c.Sum(nil), err
 }
+
+var crcTable = crc64.MakeTable(crc64.ISO)

tar/asm/getter_test.go

@@ -8,40 +8,55 @@ import (
 
 func TestGetter(t *testing.T) {
 	fgp := NewBufferFileGetPutter()
-	files := map[string][]byte{
+	files := map[string]map[string][]byte{
-		"file1.txt": []byte("foo"),
+		"file1.txt": {"foo": []byte{60, 60, 48, 48, 0, 0, 0, 0}},
-		"file2.txt": []byte("bar"),
+		"file2.txt": {"bar": []byte{45, 196, 22, 240, 0, 0, 0, 0}},
 	}
 	for n, b := range files {
-		if err := fgp.Put(n, bytes.NewBuffer(b)); err != nil {
+		for body, sum := range b {
-			t.Error(err)
+			csum, err := fgp.Put(n, bytes.NewBufferString(body))
+			if err != nil {
+				t.Error(err)
+			}
+			if !bytes.Equal(csum, sum) {
+				t.Errorf("checksum: expected 0x%x; got 0x%x", sum, csum)
+			}
 		}
 	}
 	for n, b := range files {
-		r, err := fgp.Get(n)
+		for body := range b {
-		if err != nil {
+			r, err := fgp.Get(n)
-			t.Error(err)
+			if err != nil {
-		}
+				t.Error(err)
-		buf, err := ioutil.ReadAll(r)
+			}
-		if err != nil {
+			buf, err := ioutil.ReadAll(r)
-			t.Error(err)
+			if err != nil {
-		}
+				t.Error(err)
-		if string(b) != string(buf) {
+			}
-			t.Errorf("expected %q, got %q", string(b), string(buf))
+			if body != string(buf) {
+				t.Errorf("expected %q, got %q", body, string(buf))
+			}
 		}
 	}
 }
 func TestPutter(t *testing.T) {
 	fp := NewDiscardFilePutter()
-	files := map[string][]byte{
+	// map[filename]map[body]crc64sum
-		"file1.txt": []byte("foo"),
+	files := map[string]map[string][]byte{
-		"file2.txt": []byte("bar"),
+		"file1.txt": {"foo": []byte{60, 60, 48, 48, 0, 0, 0, 0}},
-		"file3.txt": []byte("baz"),
+		"file2.txt": {"bar": []byte{45, 196, 22, 240, 0, 0, 0, 0}},
-		"file4.txt": []byte("bif"),
+		"file3.txt": {"baz": []byte{32, 68, 22, 240, 0, 0, 0, 0}},
+		"file4.txt": {"bif": []byte{48, 9, 150, 240, 0, 0, 0, 0}},
 	}
 	for n, b := range files {
-		if err := fp.Put(n, bytes.NewBuffer(b)); err != nil {
+		for body, sum := range b {
-			t.Error(err)
+			csum, err := fp.Put(n, bytes.NewBufferString(body))
+			if err != nil {
+				t.Error(err)
+			}
+			if !bytes.Equal(csum, sum) {
+				t.Errorf("checksum on %q: expected %v; got %v", n, sum, csum)
+			}
 		}
 	}
 }