forked from mirrors/tar-split
tar/asm: now testing assemble and disassemble
passing a tar archive through disassembly, then reassembling a tar stream from it's metadata. Checking size and sha1 of the whole stream.
This commit is contained in:
parent
feaa049730
commit
ab2fc5ec40
3 changed files with 45 additions and 99 deletions
|
|
@ -60,6 +60,7 @@ func NewOutputTarStream(fg FileGetter, up storage.Unpacker) io.ReadCloser {
|
|||
}
|
||||
}
|
||||
}
|
||||
pw.Close()
|
||||
}()
|
||||
return pr
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,12 +17,6 @@ var entries = []struct {
|
|||
Entry storage.Entry
|
||||
Body []byte
|
||||
}{
|
||||
{
|
||||
Entry: storage.Entry{
|
||||
Type: storage.SegmentType,
|
||||
Payload: []byte("y'all"), // FIXME need real header here
|
||||
},
|
||||
},
|
||||
{
|
||||
Entry: storage.Entry{
|
||||
Type: storage.FileType,
|
||||
|
|
@ -32,12 +26,6 @@ var entries = []struct {
|
|||
},
|
||||
Body: []byte("imma hurr til I derp"),
|
||||
},
|
||||
{
|
||||
Entry: storage.Entry{
|
||||
Type: storage.SegmentType,
|
||||
Payload: []byte("doin"), // FIXME need real header here
|
||||
},
|
||||
},
|
||||
{
|
||||
Entry: storage.Entry{
|
||||
Type: storage.FileType,
|
||||
|
|
@ -48,72 +36,9 @@ var entries = []struct {
|
|||
|
||||
Body: []byte("café con leche, por favor"),
|
||||
},
|
||||
{
|
||||
Entry: storage.Entry{
|
||||
Type: storage.SegmentType,
|
||||
Payload: []byte{00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00,
|
||||
00, 00},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestTarStream(t *testing.T) {
|
||||
func TestTarStreamOld(t *testing.T) {
|
||||
fgp := NewBufferFileGetPutter()
|
||||
|
||||
// first lets prep a GetPutter and Packer
|
||||
|
|
@ -143,7 +68,7 @@ func TestTarStream(t *testing.T) {
|
|||
// TODO finish this
|
||||
}
|
||||
|
||||
func TestInputTarStream(t *testing.T) {
|
||||
func TestTarStream(t *testing.T) {
|
||||
var (
|
||||
expectedSum = "1eb237ff69bca6e22789ecb05b45d35ca307adbd"
|
||||
expectedSize int64 = 10240
|
||||
|
|
@ -165,22 +90,18 @@ func TestInputTarStream(t *testing.T) {
|
|||
sp := storage.NewJsonPacker(w)
|
||||
fgp := NewBufferFileGetPutter()
|
||||
|
||||
// check the tar on the front end too
|
||||
h0 := sha1.New()
|
||||
tRdr0 := io.TeeReader(gzRdr, h0)
|
||||
|
||||
// wrap the disassembly stream
|
||||
tarStream, err := NewInputTarStream(tRdr0, sp, fgp)
|
||||
tarStream, err := NewInputTarStream(gzRdr, sp, fgp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// get a sum of the stream after it has passed through to ensure it's the same.
|
||||
h1 := sha1.New()
|
||||
tRdr1 := io.TeeReader(tarStream, h1)
|
||||
h0 := sha1.New()
|
||||
tRdr0 := io.TeeReader(tarStream, h0)
|
||||
|
||||
// read it all to the bit bucket
|
||||
i, err := io.Copy(ioutil.Discard, tRdr1)
|
||||
i, err := io.Copy(ioutil.Discard, tRdr0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -188,10 +109,32 @@ func TestInputTarStream(t *testing.T) {
|
|||
if i != expectedSize {
|
||||
t.Errorf("size of tar: expected %d; got %d", expectedSize, i)
|
||||
}
|
||||
|
||||
if fmt.Sprintf("%x", h1.Sum(nil)) != expectedSum {
|
||||
t.Logf("h0 was %x", h0.Sum(nil))
|
||||
t.Errorf("checksum of tar: expected %q; got %x", expectedSum, h1.Sum(nil))
|
||||
if fmt.Sprintf("%x", h0.Sum(nil)) != expectedSum {
|
||||
t.Fatalf("checksum of tar: expected %s; got %x", expectedSum, h0.Sum(nil))
|
||||
}
|
||||
|
||||
t.Logf("%s", w.String()) // if we fail, then show the packed info
|
||||
|
||||
// If we've made it this far, then we'll turn it around and create a tar
|
||||
// stream from the packed metadata and buffered file contents.
|
||||
r := bytes.NewBuffer(w.Bytes())
|
||||
sup := storage.NewJsonUnpacker(r)
|
||||
// and reuse the fgp that we Put the payloads to.
|
||||
|
||||
rc := NewOutputTarStream(fgp, sup)
|
||||
h1 := sha1.New()
|
||||
tRdr1 := io.TeeReader(rc, h1)
|
||||
|
||||
// read it all to the bit bucket
|
||||
i, err = io.Copy(ioutil.Discard, tRdr1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if i != expectedSize {
|
||||
t.Errorf("size of output tar: expected %d; got %d", expectedSize, i)
|
||||
}
|
||||
if fmt.Sprintf("%x", h1.Sum(nil)) != expectedSum {
|
||||
t.Fatalf("checksum of output tar: expected %s; got %x", expectedSum, h1.Sum(nil))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -70,16 +70,16 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp FilePutter) (io.Reader,
|
|||
pW.CloseWithError(err)
|
||||
}
|
||||
|
||||
var csum []byte
|
||||
sumChan := make(chan []byte)
|
||||
if hdr.Size > 0 {
|
||||
// if there is a file payload to write, then write the file to the FilePutter
|
||||
fileRdr, fileWrtr := io.Pipe()
|
||||
go func() {
|
||||
var err error
|
||||
_, csum, err = fp.Put(hdr.Name, fileRdr)
|
||||
_, csum, err := fp.Put(hdr.Name, fileRdr)
|
||||
if err != nil {
|
||||
pW.CloseWithError(err)
|
||||
}
|
||||
sumChan <- csum
|
||||
}()
|
||||
if _, err = io.Copy(fileWrtr, tr); err != nil {
|
||||
pW.CloseWithError(err)
|
||||
|
|
@ -92,18 +92,20 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp FilePutter) (io.Reader,
|
|||
Type: storage.FileType,
|
||||
Name: hdr.Name,
|
||||
Size: hdr.Size,
|
||||
Payload: csum,
|
||||
Payload: <-sumChan,
|
||||
})
|
||||
if err != nil {
|
||||
pW.CloseWithError(err)
|
||||
}
|
||||
|
||||
_, err = p.AddEntry(storage.Entry{
|
||||
Type: storage.SegmentType,
|
||||
Payload: tr.RawBytes(),
|
||||
})
|
||||
if err != nil {
|
||||
pW.CloseWithError(err)
|
||||
if b := tr.RawBytes(); len(b) > 0 {
|
||||
_, err = p.AddEntry(storage.Entry{
|
||||
Type: storage.SegmentType,
|
||||
Payload: b,
|
||||
})
|
||||
if err != nil {
|
||||
pW.CloseWithError(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue