1
0
Fork 0
forked from mirrors/tar-split

Compare commits

..

No commits in common. "master" and "v0.9.13" have entirely different histories.

13 changed files with 25 additions and 125 deletions

View file

@ -1,17 +1,16 @@
language: go
go:
- tip
- 1.x
- 1.8.x
- 1.7.x
- 1.6.x
- 1.5.x
- 1.5.1
- 1.4.3
- 1.3.3
# let us have pretty, fast Docker-based Travis workers!
sudo: false
install:
- go get -d ./...
- go get golang.org/x/tools/cmd/vet
script:
- go test -v ./...

View file

@ -1,7 +1,6 @@
# tar-split
[![Build Status](https://travis-ci.org/vbatts/tar-split.svg?branch=master)](https://travis-ci.org/vbatts/tar-split)
[![Go Report Card](https://goreportcard.com/badge/github.com/vbatts/tar-split)](https://goreportcard.com/report/github.com/vbatts/tar-split)
Pristinely disassembling a tar archive, and stashing needed raw bytes and offsets to reassemble a validating original archive.
@ -40,7 +39,7 @@ This demonstrates the `tar-split` command and how to assemble a tar archive from
This demonstrates the tar-split integration for docker-1.8. Providing consistent tar archives for the image layer content.
![docker tar-split demo](https://i.ytimg.com/vi_webp/vh5wyjIOBtc/default.webp)
![docker tar-split demo](https://www.youtube.com/upload_thumbnail?v=tV_Dia8E8xw&t=2&ts=1445028436275)
[youtube vide of docker layer checksums](https://youtu.be/tV_Dia8E8xw)
## Caveat
@ -67,7 +66,7 @@ Do not break the API of stdlib `archive/tar` in our fork (ideally find an upstre
## Std Version
The version of golang stdlib `archive/tar` is from go1.6
The version of golang stdlib `archive/tar` is from go1.4.1, and their master branch around [a9dddb53f](https://github.com/golang/go/tree/a9dddb53f).
It is minimally extended to expose the raw bytes of the TAR, rather than just the marshalled headers and file stream.

View file

@ -608,12 +608,11 @@ func (tr *Reader) readHeader() *Header {
header := tr.hdrBuff[:]
copy(header, zeroBlock)
if n, err := io.ReadFull(tr.r, header); err != nil {
tr.err = err
if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil {
// because it could read some of the block, but reach EOF first
if tr.err == io.EOF && tr.RawAccounting {
if _, err := tr.rawBytes.Write(header[:n]); err != nil {
tr.err = err
if _, tr.err = tr.rawBytes.Write(header); tr.err != nil {
return nil
}
}
return nil // io.EOF is okay here
@ -626,12 +625,11 @@ func (tr *Reader) readHeader() *Header {
// Two blocks of zero bytes marks the end of the archive.
if bytes.Equal(header, zeroBlock[0:blockSize]) {
if n, err := io.ReadFull(tr.r, header); err != nil {
tr.err = err
if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil {
// because it could read some of the block, but reach EOF first
if tr.err == io.EOF && tr.RawAccounting {
if _, err := tr.rawBytes.Write(header[:n]); err != nil {
tr.err = err
if _, tr.err = tr.rawBytes.Write(header); tr.err != nil {
return nil
}
}
return nil // io.EOF is okay here

View file

@ -94,12 +94,13 @@ func TestRoundTrip(t *testing.T) {
var b bytes.Buffer
tw := NewWriter(&b)
hdr := &Header{
Name: "file.txt",
Uid: 1 << 21, // too big for 8 octal digits
Size: int64(len(data)),
// https://github.com/golang/go/commit/0e3355903d2ebcf5ee9e76096f51ac9a116a9dbb#diff-d7bf2a98d7b57b6ff754ca406f1b7581R105
ModTime: time.Now().AddDate(0, 0, 0).Round(1 * time.Second),
Name: "file.txt",
Uid: 1 << 21, // too big for 8 octal digits
Size: int64(len(data)),
ModTime: time.Now(),
}
// tar only supports second precision.
hdr.ModTime = hdr.ModTime.Add(-time.Duration(hdr.ModTime.Nanosecond()) * time.Nanosecond)
if err := tw.WriteHeader(hdr); err != nil {
t.Fatalf("tw.WriteHeader: %v", err)
}

View file

@ -6,7 +6,7 @@ import (
"os"
"github.com/Sirupsen/logrus"
"github.com/urfave/cli"
"github.com/codegangsta/cli"
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
)

View file

@ -10,7 +10,7 @@ import (
"os"
"github.com/Sirupsen/logrus"
"github.com/urfave/cli"
"github.com/codegangsta/cli"
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
)

View file

@ -3,11 +3,10 @@ package main
import (
"compress/gzip"
"io"
"io/ioutil"
"os"
"github.com/Sirupsen/logrus"
"github.com/urfave/cli"
"github.com/codegangsta/cli"
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
)
@ -49,13 +48,7 @@ func CommandDisasm(c *cli.Context) {
if err != nil {
logrus.Fatal(err)
}
var out io.Writer
if c.Bool("no-stdout") {
out = ioutil.Discard
} else {
out = os.Stdout
}
i, err := io.Copy(out, its)
i, err := io.Copy(os.Stdout, its)
if err != nil {
logrus.Fatal(err)
}

View file

@ -4,7 +4,7 @@ import (
"os"
"github.com/Sirupsen/logrus"
"github.com/urfave/cli"
"github.com/codegangsta/cli"
"github.com/vbatts/tar-split/version"
)
@ -42,10 +42,6 @@ func main() {
Value: "tar-data.json.gz",
Usage: "output of disassembled tar stream",
},
cli.BoolFlag{
Name: "no-stdout",
Usage: "do not throughput the stream to STDOUT",
},
},
},
{

View file

@ -139,8 +139,6 @@ var testCases = []struct {
{"./testdata/longlink.tar.gz", "d9f6babe107b7247953dff6b5b5ae31a3a880add", 20480},
{"./testdata/fatlonglink.tar.gz", "8537f03f89aeef537382f8b0bb065d93e03b0be8", 26234880},
{"./testdata/iso-8859.tar.gz", "ddafa51cb03c74ec117ab366ee2240d13bba1ec3", 10240},
{"./testdata/extranils.tar.gz", "e187b4b3e739deaccc257342f4940f34403dc588", 10648},
{"./testdata/notenoughnils.tar.gz", "72f93f41efd95290baa5c174c234f5d4c22ce601", 512},
}
func TestTarStream(t *testing.T) {

Binary file not shown.

Binary file not shown.

View file

@ -1,84 +0,0 @@
package tartest
import (
"io"
"io/ioutil"
"os"
"testing"
upTar "archive/tar"
ourTar "github.com/vbatts/tar-split/archive/tar"
)
var testfile = "./archive/tar/testdata/sparse-formats.tar"
func BenchmarkUpstreamTar(b *testing.B) {
for n := 0; n < b.N; n++ {
fh, err := os.Open(testfile)
if err != nil {
b.Fatal(err)
}
tr := upTar.NewReader(fh)
for {
_, err := tr.Next()
if err != nil {
if err == io.EOF {
break
}
fh.Close()
b.Fatal(err)
}
io.Copy(ioutil.Discard, tr)
}
fh.Close()
}
}
func BenchmarkOurTarNoAccounting(b *testing.B) {
for n := 0; n < b.N; n++ {
fh, err := os.Open(testfile)
if err != nil {
b.Fatal(err)
}
tr := ourTar.NewReader(fh)
tr.RawAccounting = false // this is default, but explicit here
for {
_, err := tr.Next()
if err != nil {
if err == io.EOF {
break
}
fh.Close()
b.Fatal(err)
}
io.Copy(ioutil.Discard, tr)
}
fh.Close()
}
}
func BenchmarkOurTarYesAccounting(b *testing.B) {
for n := 0; n < b.N; n++ {
fh, err := os.Open(testfile)
if err != nil {
b.Fatal(err)
}
tr := ourTar.NewReader(fh)
tr.RawAccounting = true // This enables mechanics for collecting raw bytes
for {
_ = tr.RawBytes()
_, err := tr.Next()
_ = tr.RawBytes()
if err != nil {
if err == io.EOF {
break
}
fh.Close()
b.Fatal(err)
}
io.Copy(ioutil.Discard, tr)
_ = tr.RawBytes()
}
fh.Close()
}
}

View file

@ -1,7 +1,7 @@
package version
// AUTO-GENEREATED. DO NOT EDIT
// 2016-09-26 19:53:30.825879 -0400 EDT
// 2016-02-15 09:43:15.919197079 -0500 EST
// VERSION is the generated version from /home/vbatts/src/vb/tar-split/version
var VERSION = "v0.10.1-4-gf280282"
var VERSION = "v0.9.13-19-862ccd05b"