From ab2fc5ec40ae7604a1dbf48f726a9fc8c987ea1b Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 5 Mar 2015 14:09:17 -0500 Subject: [PATCH] tar/asm: now testing assemble and disassemble passing a tar archive through disassembly, then reassembling a tar stream from it's metadata. Checking size and sha1 of the whole stream. --- tar/asm/assemble.go | 1 + tar/asm/assemble_test.go | 121 +++++++++++---------------------------- tar/asm/disassemble.go | 22 +++---- 3 files changed, 45 insertions(+), 99 deletions(-) diff --git a/tar/asm/assemble.go b/tar/asm/assemble.go index fed65fc..610eebd 100644 --- a/tar/asm/assemble.go +++ b/tar/asm/assemble.go @@ -60,6 +60,7 @@ func NewOutputTarStream(fg FileGetter, up storage.Unpacker) io.ReadCloser { } } } + pw.Close() }() return pr } diff --git a/tar/asm/assemble_test.go b/tar/asm/assemble_test.go index dc5c1db..4ae3f69 100644 --- a/tar/asm/assemble_test.go +++ b/tar/asm/assemble_test.go @@ -17,12 +17,6 @@ var entries = []struct { Entry storage.Entry Body []byte }{ - { - Entry: storage.Entry{ - Type: storage.SegmentType, - Payload: []byte("y'all"), // FIXME need real header here - }, - }, { Entry: storage.Entry{ Type: storage.FileType, @@ -32,12 +26,6 @@ var entries = []struct { }, Body: []byte("imma hurr til I derp"), }, - { - Entry: storage.Entry{ - Type: storage.SegmentType, - Payload: []byte("doin"), // FIXME need real header here - }, - }, { Entry: storage.Entry{ Type: storage.FileType, @@ -48,72 +36,9 @@ var entries = []struct { Body: []byte("café con leche, por favor"), }, - { - Entry: storage.Entry{ - Type: storage.SegmentType, - Payload: []byte{00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, - 00, 00}, - }, - }, } -func TestTarStream(t *testing.T) { +func TestTarStreamOld(t *testing.T) { fgp := NewBufferFileGetPutter() // first lets prep a GetPutter and Packer @@ -143,7 +68,7 @@ func TestTarStream(t *testing.T) { // TODO finish this } -func TestInputTarStream(t *testing.T) { +func TestTarStream(t *testing.T) { var ( expectedSum = "1eb237ff69bca6e22789ecb05b45d35ca307adbd" expectedSize int64 = 10240 @@ -165,22 +90,18 @@ func TestInputTarStream(t *testing.T) { sp := storage.NewJsonPacker(w) fgp := NewBufferFileGetPutter() - // check the tar on the front end too - h0 := sha1.New() - tRdr0 := io.TeeReader(gzRdr, h0) - // wrap the disassembly stream - tarStream, err := NewInputTarStream(tRdr0, sp, fgp) + tarStream, err := NewInputTarStream(gzRdr, sp, fgp) if err != nil { t.Fatal(err) } // get a sum of the stream after it has passed through to ensure it's the same. - h1 := sha1.New() - tRdr1 := io.TeeReader(tarStream, h1) + h0 := sha1.New() + tRdr0 := io.TeeReader(tarStream, h0) // read it all to the bit bucket - i, err := io.Copy(ioutil.Discard, tRdr1) + i, err := io.Copy(ioutil.Discard, tRdr0) if err != nil { t.Fatal(err) } @@ -188,10 +109,32 @@ func TestInputTarStream(t *testing.T) { if i != expectedSize { t.Errorf("size of tar: expected %d; got %d", expectedSize, i) } - - if fmt.Sprintf("%x", h1.Sum(nil)) != expectedSum { - t.Logf("h0 was %x", h0.Sum(nil)) - t.Errorf("checksum of tar: expected %q; got %x", expectedSum, h1.Sum(nil)) + if fmt.Sprintf("%x", h0.Sum(nil)) != expectedSum { + t.Fatalf("checksum of tar: expected %s; got %x", expectedSum, h0.Sum(nil)) } + t.Logf("%s", w.String()) // if we fail, then show the packed info + + // If we've made it this far, then we'll turn it around and create a tar + // stream from the packed metadata and buffered file contents. + r := bytes.NewBuffer(w.Bytes()) + sup := storage.NewJsonUnpacker(r) + // and reuse the fgp that we Put the payloads to. + + rc := NewOutputTarStream(fgp, sup) + h1 := sha1.New() + tRdr1 := io.TeeReader(rc, h1) + + // read it all to the bit bucket + i, err = io.Copy(ioutil.Discard, tRdr1) + if err != nil { + t.Fatal(err) + } + + if i != expectedSize { + t.Errorf("size of output tar: expected %d; got %d", expectedSize, i) + } + if fmt.Sprintf("%x", h1.Sum(nil)) != expectedSum { + t.Fatalf("checksum of output tar: expected %s; got %x", expectedSum, h1.Sum(nil)) + } } diff --git a/tar/asm/disassemble.go b/tar/asm/disassemble.go index 91176e6..58ba5ca 100644 --- a/tar/asm/disassemble.go +++ b/tar/asm/disassemble.go @@ -70,16 +70,16 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp FilePutter) (io.Reader, pW.CloseWithError(err) } - var csum []byte + sumChan := make(chan []byte) if hdr.Size > 0 { // if there is a file payload to write, then write the file to the FilePutter fileRdr, fileWrtr := io.Pipe() go func() { - var err error - _, csum, err = fp.Put(hdr.Name, fileRdr) + _, csum, err := fp.Put(hdr.Name, fileRdr) if err != nil { pW.CloseWithError(err) } + sumChan <- csum }() if _, err = io.Copy(fileWrtr, tr); err != nil { pW.CloseWithError(err) @@ -92,18 +92,20 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp FilePutter) (io.Reader, Type: storage.FileType, Name: hdr.Name, Size: hdr.Size, - Payload: csum, + Payload: <-sumChan, }) if err != nil { pW.CloseWithError(err) } - _, err = p.AddEntry(storage.Entry{ - Type: storage.SegmentType, - Payload: tr.RawBytes(), - }) - if err != nil { - pW.CloseWithError(err) + if b := tr.RawBytes(); len(b) > 0 { + _, err = p.AddEntry(storage.Entry{ + Type: storage.SegmentType, + Payload: b, + }) + if err != nil { + pW.CloseWithError(err) + } } }