forked from mirrors/tar-split
tar/asm: FileType entry with crc64 checksum
This commit is contained in:
parent
962589aca7
commit
4f1bde4d13
5 changed files with 121 additions and 50 deletions
|
@ -1,6 +1,9 @@
|
|||
package asm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"hash/crc64"
|
||||
"io"
|
||||
|
||||
"github.com/vbatts/tar-split/tar/storage"
|
||||
|
@ -42,10 +45,16 @@ func NewOutputTarStream(fg FileGetter, up storage.Unpacker) io.ReadCloser {
|
|||
break
|
||||
}
|
||||
defer fh.Close()
|
||||
if _, err := io.Copy(pw, fh); err != nil {
|
||||
c := crc64.New(crcTable)
|
||||
tRdr := io.TeeReader(fh, c)
|
||||
if _, err := io.Copy(pw, tRdr); err != nil {
|
||||
pw.CloseWithError(err)
|
||||
break
|
||||
}
|
||||
if !bytes.Equal(c.Sum(nil), entry.Payload) {
|
||||
pw.CloseWithError(fmt.Errorf("file integrity checksum failed for %q", entry.Name))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
|
|
@ -1,9 +1,48 @@
|
|||
package asm
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/vbatts/tar-split/tar/storage"
|
||||
)
|
||||
|
||||
var entries = storage.Entries{
|
||||
storage.Entry{
|
||||
Type: storage.SegmentType,
|
||||
Payload: []byte("how"),
|
||||
Position: 0,
|
||||
},
|
||||
storage.Entry{
|
||||
Type: storage.SegmentType,
|
||||
Payload: []byte("y'all"),
|
||||
Position: 1,
|
||||
},
|
||||
storage.Entry{
|
||||
Type: storage.FileType,
|
||||
Name: "./hurr.txt",
|
||||
Payload: []byte("deadbeef"),
|
||||
Size: 8,
|
||||
Position: 2,
|
||||
},
|
||||
storage.Entry{
|
||||
Type: storage.SegmentType,
|
||||
Payload: []byte("doin"),
|
||||
Position: 3,
|
||||
},
|
||||
storage.Entry{
|
||||
Type: storage.FileType,
|
||||
Name: "./ermahgerd.txt",
|
||||
Payload: []byte("cafebabe"),
|
||||
Size: 8,
|
||||
Position: 4,
|
||||
},
|
||||
}
|
||||
|
||||
func TestNewOutputTarStream(t *testing.T) {
|
||||
// TODO disassembly
|
||||
fgp := NewBufferFileGetPutter()
|
||||
_ = NewOutputTarStream(fgp, nil)
|
||||
}
|
||||
|
||||
func TestNewInputTarStream(t *testing.T) {
|
||||
}
|
||||
|
|
|
@ -36,6 +36,10 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp FilePutter) (io.Reader,
|
|||
pR, pW := io.Pipe()
|
||||
outputRdr := io.TeeReader(r, pW)
|
||||
|
||||
if fp == nil {
|
||||
fp = NewDiscardFilePutter()
|
||||
}
|
||||
|
||||
go func() {
|
||||
tr := tar.NewReader(outputRdr)
|
||||
tr.RawAccounting = true
|
||||
|
@ -67,12 +71,14 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp FilePutter) (io.Reader,
|
|||
pW.CloseWithError(err)
|
||||
}
|
||||
|
||||
var csum []byte
|
||||
if hdr.Size > 0 {
|
||||
if fp != nil {
|
||||
// if there is a file payload to write, then write the file to the FilePutter
|
||||
fileRdr, fileWrtr := io.Pipe()
|
||||
go func() {
|
||||
if err := fp.Put(hdr.Name, fileRdr); err != nil {
|
||||
var err error
|
||||
csum, err = fp.Put(hdr.Name, fileRdr)
|
||||
if err != nil {
|
||||
pW.CloseWithError(err)
|
||||
}
|
||||
}()
|
||||
|
@ -80,18 +86,13 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp FilePutter) (io.Reader,
|
|||
pW.CloseWithError(err)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
if _, err := io.Copy(ioutil.Discard, tr); err != nil {
|
||||
pW.CloseWithError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
// File entries added, regardless of size
|
||||
if _, err := p.AddEntry(storage.Entry{
|
||||
Type: storage.FileType,
|
||||
Name: hdr.Name,
|
||||
Size: hdr.Size,
|
||||
Payload: csum,
|
||||
}); err != nil {
|
||||
pW.CloseWithError(err)
|
||||
}
|
||||
|
|
|
@ -3,6 +3,7 @@ package asm
|
|||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"hash/crc64"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
@ -16,7 +17,7 @@ type FileGetter interface {
|
|||
|
||||
type FilePutter interface {
|
||||
// Put returns a stream for the provided file path
|
||||
Put(string, io.Reader) error
|
||||
Put(string, io.Reader) ([]byte, error)
|
||||
}
|
||||
|
||||
type FileGetPutter interface {
|
||||
|
@ -50,13 +51,15 @@ func (bfgp bufferFileGetPutter) Get(name string) (io.ReadCloser, error) {
|
|||
return &readCloserWrapper{b}, nil
|
||||
}
|
||||
|
||||
func (bfgp *bufferFileGetPutter) Put(name string, r io.Reader) error {
|
||||
func (bfgp *bufferFileGetPutter) Put(name string, r io.Reader) ([]byte, error) {
|
||||
c := crc64.New(crcTable)
|
||||
tRdr := io.TeeReader(r, c)
|
||||
b := bytes.NewBuffer([]byte{})
|
||||
if _, err := io.Copy(b, r); err != nil {
|
||||
return err
|
||||
if _, err := io.Copy(b, tRdr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bfgp.files[name] = b.Bytes()
|
||||
return nil
|
||||
return c.Sum(nil), nil
|
||||
}
|
||||
|
||||
type readCloserWrapper struct {
|
||||
|
@ -83,7 +86,11 @@ func NewDiscardFilePutter() FilePutter {
|
|||
type bitBucketFilePutter struct {
|
||||
}
|
||||
|
||||
func (bbfp *bitBucketFilePutter) Put(name string, r io.Reader) error {
|
||||
_, err := io.Copy(ioutil.Discard, r)
|
||||
return err
|
||||
func (bbfp *bitBucketFilePutter) Put(name string, r io.Reader) ([]byte, error) {
|
||||
c := crc64.New(crcTable)
|
||||
tRdr := io.TeeReader(r, c)
|
||||
_, err := io.Copy(ioutil.Discard, tRdr)
|
||||
return c.Sum(nil), err
|
||||
}
|
||||
|
||||
var crcTable = crc64.MakeTable(crc64.ISO)
|
||||
|
|
|
@ -8,16 +8,23 @@ import (
|
|||
|
||||
func TestGetter(t *testing.T) {
|
||||
fgp := NewBufferFileGetPutter()
|
||||
files := map[string][]byte{
|
||||
"file1.txt": []byte("foo"),
|
||||
"file2.txt": []byte("bar"),
|
||||
files := map[string]map[string][]byte{
|
||||
"file1.txt": {"foo": []byte{60, 60, 48, 48, 0, 0, 0, 0}},
|
||||
"file2.txt": {"bar": []byte{45, 196, 22, 240, 0, 0, 0, 0}},
|
||||
}
|
||||
for n, b := range files {
|
||||
if err := fgp.Put(n, bytes.NewBuffer(b)); err != nil {
|
||||
for body, sum := range b {
|
||||
csum, err := fgp.Put(n, bytes.NewBufferString(body))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if !bytes.Equal(csum, sum) {
|
||||
t.Errorf("checksum: expected 0x%x; got 0x%x", sum, csum)
|
||||
}
|
||||
}
|
||||
}
|
||||
for n, b := range files {
|
||||
for body := range b {
|
||||
r, err := fgp.Get(n)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
|
@ -26,22 +33,30 @@ func TestGetter(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if string(b) != string(buf) {
|
||||
t.Errorf("expected %q, got %q", string(b), string(buf))
|
||||
if body != string(buf) {
|
||||
t.Errorf("expected %q, got %q", body, string(buf))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestPutter(t *testing.T) {
|
||||
fp := NewDiscardFilePutter()
|
||||
files := map[string][]byte{
|
||||
"file1.txt": []byte("foo"),
|
||||
"file2.txt": []byte("bar"),
|
||||
"file3.txt": []byte("baz"),
|
||||
"file4.txt": []byte("bif"),
|
||||
// map[filename]map[body]crc64sum
|
||||
files := map[string]map[string][]byte{
|
||||
"file1.txt": {"foo": []byte{60, 60, 48, 48, 0, 0, 0, 0}},
|
||||
"file2.txt": {"bar": []byte{45, 196, 22, 240, 0, 0, 0, 0}},
|
||||
"file3.txt": {"baz": []byte{32, 68, 22, 240, 0, 0, 0, 0}},
|
||||
"file4.txt": {"bif": []byte{48, 9, 150, 240, 0, 0, 0, 0}},
|
||||
}
|
||||
for n, b := range files {
|
||||
if err := fp.Put(n, bytes.NewBuffer(b)); err != nil {
|
||||
for body, sum := range b {
|
||||
csum, err := fp.Put(n, bytes.NewBufferString(body))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if !bytes.Equal(csum, sum) {
|
||||
t.Errorf("checksum on %q: expected %v; got %v", n, sum, csum)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue