diff --git a/.travis.yml b/.travis.yml index dcce57a..c0a17c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,16 @@ language: go go: - tip - - 1.x - - 1.8.x - - 1.7.x - - 1.6.x - - 1.5.x + - 1.5.1 + - 1.4.3 + - 1.3.3 # let us have pretty, fast Docker-based Travis workers! sudo: false install: - go get -d ./... + - go get golang.org/x/tools/cmd/vet script: - go test -v ./... diff --git a/README.md b/README.md index c2e7f48..90a8edf 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # tar-split [![Build Status](https://travis-ci.org/vbatts/tar-split.svg?branch=master)](https://travis-ci.org/vbatts/tar-split) -[![Go Report Card](https://goreportcard.com/badge/github.com/vbatts/tar-split)](https://goreportcard.com/report/github.com/vbatts/tar-split) Pristinely disassembling a tar archive, and stashing needed raw bytes and offsets to reassemble a validating original archive. @@ -40,7 +39,7 @@ This demonstrates the `tar-split` command and how to assemble a tar archive from This demonstrates the tar-split integration for docker-1.8. Providing consistent tar archives for the image layer content. -![docker tar-split demo](https://i.ytimg.com/vi_webp/vh5wyjIOBtc/default.webp) +![docker tar-split demo](https://www.youtube.com/upload_thumbnail?v=tV_Dia8E8xw&t=2&ts=1445028436275) [youtube vide of docker layer checksums](https://youtu.be/tV_Dia8E8xw) ## Caveat @@ -67,7 +66,7 @@ Do not break the API of stdlib `archive/tar` in our fork (ideally find an upstre ## Std Version -The version of golang stdlib `archive/tar` is from go1.6 +The version of golang stdlib `archive/tar` is from go1.4.1, and their master branch around [a9dddb53f](https://github.com/golang/go/tree/a9dddb53f). It is minimally extended to expose the raw bytes of the TAR, rather than just the marshalled headers and file stream. diff --git a/archive/tar/reader.go b/archive/tar/reader.go index adf3212..a8b63a2 100644 --- a/archive/tar/reader.go +++ b/archive/tar/reader.go @@ -608,12 +608,11 @@ func (tr *Reader) readHeader() *Header { header := tr.hdrBuff[:] copy(header, zeroBlock) - if n, err := io.ReadFull(tr.r, header); err != nil { - tr.err = err + if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { // because it could read some of the block, but reach EOF first if tr.err == io.EOF && tr.RawAccounting { - if _, err := tr.rawBytes.Write(header[:n]); err != nil { - tr.err = err + if _, tr.err = tr.rawBytes.Write(header); tr.err != nil { + return nil } } return nil // io.EOF is okay here @@ -626,12 +625,11 @@ func (tr *Reader) readHeader() *Header { // Two blocks of zero bytes marks the end of the archive. if bytes.Equal(header, zeroBlock[0:blockSize]) { - if n, err := io.ReadFull(tr.r, header); err != nil { - tr.err = err + if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { // because it could read some of the block, but reach EOF first if tr.err == io.EOF && tr.RawAccounting { - if _, err := tr.rawBytes.Write(header[:n]); err != nil { - tr.err = err + if _, tr.err = tr.rawBytes.Write(header); tr.err != nil { + return nil } } return nil // io.EOF is okay here diff --git a/archive/tar/tar_test.go b/archive/tar/tar_test.go index 9ef319a..d63c072 100644 --- a/archive/tar/tar_test.go +++ b/archive/tar/tar_test.go @@ -94,12 +94,13 @@ func TestRoundTrip(t *testing.T) { var b bytes.Buffer tw := NewWriter(&b) hdr := &Header{ - Name: "file.txt", - Uid: 1 << 21, // too big for 8 octal digits - Size: int64(len(data)), - // https://github.com/golang/go/commit/0e3355903d2ebcf5ee9e76096f51ac9a116a9dbb#diff-d7bf2a98d7b57b6ff754ca406f1b7581R105 - ModTime: time.Now().AddDate(0, 0, 0).Round(1 * time.Second), + Name: "file.txt", + Uid: 1 << 21, // too big for 8 octal digits + Size: int64(len(data)), + ModTime: time.Now(), } + // tar only supports second precision. + hdr.ModTime = hdr.ModTime.Add(-time.Duration(hdr.ModTime.Nanosecond()) * time.Nanosecond) if err := tw.WriteHeader(hdr); err != nil { t.Fatalf("tw.WriteHeader: %v", err) } diff --git a/cmd/tar-split/asm.go b/cmd/tar-split/asm.go index e188ce1..312e54b 100644 --- a/cmd/tar-split/asm.go +++ b/cmd/tar-split/asm.go @@ -6,7 +6,7 @@ import ( "os" "github.com/Sirupsen/logrus" - "github.com/urfave/cli" + "github.com/codegangsta/cli" "github.com/vbatts/tar-split/tar/asm" "github.com/vbatts/tar-split/tar/storage" ) diff --git a/cmd/tar-split/checksize.go b/cmd/tar-split/checksize.go index 1e5eed7..38f830e 100644 --- a/cmd/tar-split/checksize.go +++ b/cmd/tar-split/checksize.go @@ -10,7 +10,7 @@ import ( "os" "github.com/Sirupsen/logrus" - "github.com/urfave/cli" + "github.com/codegangsta/cli" "github.com/vbatts/tar-split/tar/asm" "github.com/vbatts/tar-split/tar/storage" ) diff --git a/cmd/tar-split/disasm.go b/cmd/tar-split/disasm.go index 5472894..b7b0dfe 100644 --- a/cmd/tar-split/disasm.go +++ b/cmd/tar-split/disasm.go @@ -3,11 +3,10 @@ package main import ( "compress/gzip" "io" - "io/ioutil" "os" "github.com/Sirupsen/logrus" - "github.com/urfave/cli" + "github.com/codegangsta/cli" "github.com/vbatts/tar-split/tar/asm" "github.com/vbatts/tar-split/tar/storage" ) @@ -49,13 +48,7 @@ func CommandDisasm(c *cli.Context) { if err != nil { logrus.Fatal(err) } - var out io.Writer - if c.Bool("no-stdout") { - out = ioutil.Discard - } else { - out = os.Stdout - } - i, err := io.Copy(out, its) + i, err := io.Copy(os.Stdout, its) if err != nil { logrus.Fatal(err) } diff --git a/cmd/tar-split/main.go b/cmd/tar-split/main.go index 8b4035f..b417120 100644 --- a/cmd/tar-split/main.go +++ b/cmd/tar-split/main.go @@ -4,7 +4,7 @@ import ( "os" "github.com/Sirupsen/logrus" - "github.com/urfave/cli" + "github.com/codegangsta/cli" "github.com/vbatts/tar-split/version" ) @@ -42,10 +42,6 @@ func main() { Value: "tar-data.json.gz", Usage: "output of disassembled tar stream", }, - cli.BoolFlag{ - Name: "no-stdout", - Usage: "do not throughput the stream to STDOUT", - }, }, }, { diff --git a/tar/asm/assemble_test.go b/tar/asm/assemble_test.go index afdce9d..c0c7f17 100644 --- a/tar/asm/assemble_test.go +++ b/tar/asm/assemble_test.go @@ -139,8 +139,6 @@ var testCases = []struct { {"./testdata/longlink.tar.gz", "d9f6babe107b7247953dff6b5b5ae31a3a880add", 20480}, {"./testdata/fatlonglink.tar.gz", "8537f03f89aeef537382f8b0bb065d93e03b0be8", 26234880}, {"./testdata/iso-8859.tar.gz", "ddafa51cb03c74ec117ab366ee2240d13bba1ec3", 10240}, - {"./testdata/extranils.tar.gz", "e187b4b3e739deaccc257342f4940f34403dc588", 10648}, - {"./testdata/notenoughnils.tar.gz", "72f93f41efd95290baa5c174c234f5d4c22ce601", 512}, } func TestTarStream(t *testing.T) { diff --git a/tar/asm/testdata/extranils.tar.gz b/tar/asm/testdata/extranils.tar.gz deleted file mode 100644 index 70caf6e..0000000 Binary files a/tar/asm/testdata/extranils.tar.gz and /dev/null differ diff --git a/tar/asm/testdata/notenoughnils.tar.gz b/tar/asm/testdata/notenoughnils.tar.gz deleted file mode 100644 index 146bb00..0000000 Binary files a/tar/asm/testdata/notenoughnils.tar.gz and /dev/null differ diff --git a/tar_benchmark_test.go b/tar_benchmark_test.go deleted file mode 100644 index d946f2a..0000000 --- a/tar_benchmark_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package tartest - -import ( - "io" - "io/ioutil" - "os" - "testing" - - upTar "archive/tar" - - ourTar "github.com/vbatts/tar-split/archive/tar" -) - -var testfile = "./archive/tar/testdata/sparse-formats.tar" - -func BenchmarkUpstreamTar(b *testing.B) { - for n := 0; n < b.N; n++ { - fh, err := os.Open(testfile) - if err != nil { - b.Fatal(err) - } - tr := upTar.NewReader(fh) - for { - _, err := tr.Next() - if err != nil { - if err == io.EOF { - break - } - fh.Close() - b.Fatal(err) - } - io.Copy(ioutil.Discard, tr) - } - fh.Close() - } -} - -func BenchmarkOurTarNoAccounting(b *testing.B) { - for n := 0; n < b.N; n++ { - fh, err := os.Open(testfile) - if err != nil { - b.Fatal(err) - } - tr := ourTar.NewReader(fh) - tr.RawAccounting = false // this is default, but explicit here - for { - _, err := tr.Next() - if err != nil { - if err == io.EOF { - break - } - fh.Close() - b.Fatal(err) - } - io.Copy(ioutil.Discard, tr) - } - fh.Close() - } -} -func BenchmarkOurTarYesAccounting(b *testing.B) { - for n := 0; n < b.N; n++ { - fh, err := os.Open(testfile) - if err != nil { - b.Fatal(err) - } - tr := ourTar.NewReader(fh) - tr.RawAccounting = true // This enables mechanics for collecting raw bytes - for { - _ = tr.RawBytes() - _, err := tr.Next() - _ = tr.RawBytes() - if err != nil { - if err == io.EOF { - break - } - fh.Close() - b.Fatal(err) - } - io.Copy(ioutil.Discard, tr) - _ = tr.RawBytes() - } - fh.Close() - } -} diff --git a/version/version.go b/version/version.go index f317010..345274d 100644 --- a/version/version.go +++ b/version/version.go @@ -1,7 +1,7 @@ package version // AUTO-GENEREATED. DO NOT EDIT -// 2016-09-26 19:53:30.825879 -0400 EDT +// 2016-02-15 09:43:15.919197079 -0500 EST // VERSION is the generated version from /home/vbatts/src/vb/tar-split/version -var VERSION = "v0.10.1-4-gf280282" +var VERSION = "v0.9.13-19-862ccd05b"