From e2a62d6b0d98fd7f1a57646812c74564fda999b4 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Mon, 29 Feb 2016 11:40:38 -0500 Subject: [PATCH 01/14] README.md: fix thumbnail Signed-off-by: Vincent Batts --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 90a8edf..02bbcd0 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ This demonstrates the `tar-split` command and how to assemble a tar archive from This demonstrates the tar-split integration for docker-1.8. Providing consistent tar archives for the image layer content. -![docker tar-split demo](https://www.youtube.com/upload_thumbnail?v=tV_Dia8E8xw&t=2&ts=1445028436275) +![docker tar-split demo](https://i.ytimg.com/vi_webp/vh5wyjIOBtc/default.webp) [youtube vide of docker layer checksums](https://youtu.be/tV_Dia8E8xw) ## Caveat From 226f7c74905f1fcc08ac128b517a1d65a1948eb9 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Wed, 30 Mar 2016 16:38:51 -0400 Subject: [PATCH 02/14] README: update `archive/tar` version reference Signed-off-by: Vincent Batts --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 02bbcd0..4c544d8 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ Do not break the API of stdlib `archive/tar` in our fork (ideally find an upstre ## Std Version -The version of golang stdlib `archive/tar` is from go1.4.1, and their master branch around [a9dddb53f](https://github.com/golang/go/tree/a9dddb53f). +The version of golang stdlib `archive/tar` is from go1.6 It is minimally extended to expose the raw bytes of the TAR, rather than just the marshalled headers and file stream. From 354fd6cf34bfa908f2d9796de5b1ff955b060ca6 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Sun, 26 Jun 2016 10:15:12 -0400 Subject: [PATCH 03/14] cmd: add a `disasm --no-stdout` flag Since sometimes you just need to > /dev/null Signed-off-by: Vincent Batts --- cmd/tar-split/disasm.go | 9 ++++++++- cmd/tar-split/main.go | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cmd/tar-split/disasm.go b/cmd/tar-split/disasm.go index b7b0dfe..7f8a056 100644 --- a/cmd/tar-split/disasm.go +++ b/cmd/tar-split/disasm.go @@ -3,6 +3,7 @@ package main import ( "compress/gzip" "io" + "io/ioutil" "os" "github.com/Sirupsen/logrus" @@ -48,7 +49,13 @@ func CommandDisasm(c *cli.Context) { if err != nil { logrus.Fatal(err) } - i, err := io.Copy(os.Stdout, its) + var out io.Writer + if c.Bool("no-stdout") { + out = ioutil.Discard + } else { + out = os.Stdout + } + i, err := io.Copy(out, its) if err != nil { logrus.Fatal(err) } diff --git a/cmd/tar-split/main.go b/cmd/tar-split/main.go index b417120..c584352 100644 --- a/cmd/tar-split/main.go +++ b/cmd/tar-split/main.go @@ -42,6 +42,10 @@ func main() { Value: "tar-data.json.gz", Usage: "output of disassembled tar stream", }, + cli.BoolFlag{ + Name: "no-stdout", + Usage: "do not throughput the stream to STDOUT", + }, }, }, { From beaeceb06f0a81f66e6687a2d830ed3a1e6d0875 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Sun, 26 Jun 2016 14:56:04 -0400 Subject: [PATCH 04/14] travis: update golang versions This is not saying that tar-split no longer works on go1.3 or go1.4, but rather that the headache of `go vet` having a version dependent ability to install it, makes it a headache in travis. Signed-off-by: Vincent Batts --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index c0a17c7..2d9a842 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,14 @@ language: go go: - tip - - 1.5.1 - - 1.4.3 - - 1.3.3 + - 1.6.2 + - 1.5.4 # let us have pretty, fast Docker-based Travis workers! sudo: false install: - go get -d ./... - - go get golang.org/x/tools/cmd/vet script: - go test -v ./... From 6810cedb21b2c3d0b9bb8f9af12ff2dc7a2f14df Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 26 Jul 2016 09:50:08 -0400 Subject: [PATCH 05/14] benchmark: add a comparison of 'archive/tar' Since this project has forked logic of upstream 'archive/tar', this does a brief comparison including the RawBytes usage. ```bash $ go test -run="XXX" -bench=. testing: warning: no tests to run BenchmarkUpstreamTar-4 2000 700809 ns/op BenchmarkOurTarNoAccounting-4 2000 692055 ns/op BenchmarkOurTarYesAccounting-4 2000 723184 ns/op PASS ok vb/tar-split 4.461s ``` From this, the difference is negligible. Signed-off-by: Vincent Batts --- tar_benchmark_test.go | 84 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 tar_benchmark_test.go diff --git a/tar_benchmark_test.go b/tar_benchmark_test.go new file mode 100644 index 0000000..d946f2a --- /dev/null +++ b/tar_benchmark_test.go @@ -0,0 +1,84 @@ +package tartest + +import ( + "io" + "io/ioutil" + "os" + "testing" + + upTar "archive/tar" + + ourTar "github.com/vbatts/tar-split/archive/tar" +) + +var testfile = "./archive/tar/testdata/sparse-formats.tar" + +func BenchmarkUpstreamTar(b *testing.B) { + for n := 0; n < b.N; n++ { + fh, err := os.Open(testfile) + if err != nil { + b.Fatal(err) + } + tr := upTar.NewReader(fh) + for { + _, err := tr.Next() + if err != nil { + if err == io.EOF { + break + } + fh.Close() + b.Fatal(err) + } + io.Copy(ioutil.Discard, tr) + } + fh.Close() + } +} + +func BenchmarkOurTarNoAccounting(b *testing.B) { + for n := 0; n < b.N; n++ { + fh, err := os.Open(testfile) + if err != nil { + b.Fatal(err) + } + tr := ourTar.NewReader(fh) + tr.RawAccounting = false // this is default, but explicit here + for { + _, err := tr.Next() + if err != nil { + if err == io.EOF { + break + } + fh.Close() + b.Fatal(err) + } + io.Copy(ioutil.Discard, tr) + } + fh.Close() + } +} +func BenchmarkOurTarYesAccounting(b *testing.B) { + for n := 0; n < b.N; n++ { + fh, err := os.Open(testfile) + if err != nil { + b.Fatal(err) + } + tr := ourTar.NewReader(fh) + tr.RawAccounting = true // This enables mechanics for collecting raw bytes + for { + _ = tr.RawBytes() + _, err := tr.Next() + _ = tr.RawBytes() + if err != nil { + if err == io.EOF { + break + } + fh.Close() + b.Fatal(err) + } + io.Copy(ioutil.Discard, tr) + _ = tr.RawBytes() + } + fh.Close() + } +} From e527e70d2599f4294397c46bffe9806f404ce1eb Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 22 Sep 2016 17:38:18 -0700 Subject: [PATCH 06/14] Fix panic in Next readHeader should never return nil with a tr.err also nil. To correct this, ensure tr.err never gets reset to nil followed by a nil return. --- archive/tar/reader.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/archive/tar/reader.go b/archive/tar/reader.go index a8b63a2..fdc8ae3 100644 --- a/archive/tar/reader.go +++ b/archive/tar/reader.go @@ -611,7 +611,8 @@ func (tr *Reader) readHeader() *Header { if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { // because it could read some of the block, but reach EOF first if tr.err == io.EOF && tr.RawAccounting { - if _, tr.err = tr.rawBytes.Write(header); tr.err != nil { + if _, err := tr.rawBytes.Write(header); err != nil { + tr.err = err return nil } } From eb3808673d60f519722fc86f095f90e4edfbb32a Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Fri, 23 Sep 2016 11:01:58 -0400 Subject: [PATCH 07/14] version: bump to v0.10.0 Signed-off-by: Vincent Batts --- version/version.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version/version.go b/version/version.go index 345274d..b39b671 100644 --- a/version/version.go +++ b/version/version.go @@ -1,7 +1,7 @@ package version // AUTO-GENEREATED. DO NOT EDIT -// 2016-02-15 09:43:15.919197079 -0500 EST +// 2016-09-23 11:00:18.92191222 -0400 EDT // VERSION is the generated version from /home/vbatts/src/vb/tar-split/version -var VERSION = "v0.9.13-19-862ccd05b" +var VERSION = "v0.10.0-9-gae8540d" From 7410961e758c155389fab3f0391e65320854acbc Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Mon, 26 Sep 2016 14:51:07 -0400 Subject: [PATCH 08/14] tar/asm: failing test for lack of EOF nils Reported-by: Derek McGowan Signed-off-by: Vincent Batts --- tar/asm/assemble_test.go | 2 ++ tar/asm/testdata/extranils.tar.gz | Bin 0 -> 127 bytes tar/asm/testdata/notenoughnils.tar.gz | Bin 0 -> 91 bytes 3 files changed, 2 insertions(+) create mode 100644 tar/asm/testdata/extranils.tar.gz create mode 100644 tar/asm/testdata/notenoughnils.tar.gz diff --git a/tar/asm/assemble_test.go b/tar/asm/assemble_test.go index c0c7f17..afdce9d 100644 --- a/tar/asm/assemble_test.go +++ b/tar/asm/assemble_test.go @@ -139,6 +139,8 @@ var testCases = []struct { {"./testdata/longlink.tar.gz", "d9f6babe107b7247953dff6b5b5ae31a3a880add", 20480}, {"./testdata/fatlonglink.tar.gz", "8537f03f89aeef537382f8b0bb065d93e03b0be8", 26234880}, {"./testdata/iso-8859.tar.gz", "ddafa51cb03c74ec117ab366ee2240d13bba1ec3", 10240}, + {"./testdata/extranils.tar.gz", "e187b4b3e739deaccc257342f4940f34403dc588", 10648}, + {"./testdata/notenoughnils.tar.gz", "72f93f41efd95290baa5c174c234f5d4c22ce601", 512}, } func TestTarStream(t *testing.T) { diff --git a/tar/asm/testdata/extranils.tar.gz b/tar/asm/testdata/extranils.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..70caf6e6e7641340e6bbe7fa1c36bcf7b19b9533 GIT binary patch literal 127 zcmb2|=HTE?e;LlioLW&*l$e*9Q><5#Sj6!5lr7gG2a(o`+owbx+GCh@O<1uhVXDjZ z_wv(jWtRFC|9l|l|J<@}&Ar~kA8x$){p(H5*XWejd$$%|FY1{!)44b3F~8nxGu@>} cz8d7-{`J;eKbrv#YD$=2`88gep~=7i0Ka=WQvd(} literal 0 HcmV?d00001 diff --git a/tar/asm/testdata/notenoughnils.tar.gz b/tar/asm/testdata/notenoughnils.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..146bb008bbeb2ff6e5fe1232dfc29894f85ed57b GIT binary patch literal 91 zcmV-h0HpsPiwFShZ0T15153-yNoAlhFaQEG6B7_^Xl`l%<{LuzFu-7FWNvO`WM*t+ xXvSb*XlQ6+z@T8jKnGA-T#{I%pukX;lvq+yj89!^Q8B}?0{|>t!}0(E001l6BE0|r literal 0 HcmV?d00001 From 6b59e6942e2c9a03fd4506102adfa082a96b5ace Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 26 Sep 2016 14:01:48 -0700 Subject: [PATCH 09/14] archive/tar: fix writing too many raw bytes When an EOF is read, only the part of the header buffer which was read should be accounted for. Signed-off-by: Derek McGowan --- archive/tar/reader.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/archive/tar/reader.go b/archive/tar/reader.go index fdc8ae3..adf3212 100644 --- a/archive/tar/reader.go +++ b/archive/tar/reader.go @@ -608,12 +608,12 @@ func (tr *Reader) readHeader() *Header { header := tr.hdrBuff[:] copy(header, zeroBlock) - if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { + if n, err := io.ReadFull(tr.r, header); err != nil { + tr.err = err // because it could read some of the block, but reach EOF first if tr.err == io.EOF && tr.RawAccounting { - if _, err := tr.rawBytes.Write(header); err != nil { + if _, err := tr.rawBytes.Write(header[:n]); err != nil { tr.err = err - return nil } } return nil // io.EOF is okay here @@ -626,11 +626,12 @@ func (tr *Reader) readHeader() *Header { // Two blocks of zero bytes marks the end of the archive. if bytes.Equal(header, zeroBlock[0:blockSize]) { - if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { + if n, err := io.ReadFull(tr.r, header); err != nil { + tr.err = err // because it could read some of the block, but reach EOF first if tr.err == io.EOF && tr.RawAccounting { - if _, tr.err = tr.rawBytes.Write(header); tr.err != nil { - return nil + if _, err := tr.rawBytes.Write(header[:n]); err != nil { + tr.err = err } } return nil // io.EOF is okay here From d3f1b54304d656376e58f9406a9cb4775799a357 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Mon, 26 Sep 2016 19:53:52 -0400 Subject: [PATCH 10/14] version: bump to v0.10.1 Signed-off-by: Vincent Batts --- version/version.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version/version.go b/version/version.go index b39b671..f317010 100644 --- a/version/version.go +++ b/version/version.go @@ -1,7 +1,7 @@ package version // AUTO-GENEREATED. DO NOT EDIT -// 2016-09-23 11:00:18.92191222 -0400 EDT +// 2016-09-26 19:53:30.825879 -0400 EDT // VERSION is the generated version from /home/vbatts/src/vb/tar-split/version -var VERSION = "v0.10.0-9-gae8540d" +var VERSION = "v0.10.1-4-gf280282" From bd4c5d64c3e9297f410025a3b1bd0c58f659e721 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 27 Sep 2016 02:54:18 +0000 Subject: [PATCH 11/14] main: switch import paths to urfave Signed-off-by: Vincent Batts --- cmd/tar-split/asm.go | 2 +- cmd/tar-split/checksize.go | 2 +- cmd/tar-split/disasm.go | 2 +- cmd/tar-split/main.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/tar-split/asm.go b/cmd/tar-split/asm.go index 312e54b..e188ce1 100644 --- a/cmd/tar-split/asm.go +++ b/cmd/tar-split/asm.go @@ -6,7 +6,7 @@ import ( "os" "github.com/Sirupsen/logrus" - "github.com/codegangsta/cli" + "github.com/urfave/cli" "github.com/vbatts/tar-split/tar/asm" "github.com/vbatts/tar-split/tar/storage" ) diff --git a/cmd/tar-split/checksize.go b/cmd/tar-split/checksize.go index 38f830e..1e5eed7 100644 --- a/cmd/tar-split/checksize.go +++ b/cmd/tar-split/checksize.go @@ -10,7 +10,7 @@ import ( "os" "github.com/Sirupsen/logrus" - "github.com/codegangsta/cli" + "github.com/urfave/cli" "github.com/vbatts/tar-split/tar/asm" "github.com/vbatts/tar-split/tar/storage" ) diff --git a/cmd/tar-split/disasm.go b/cmd/tar-split/disasm.go index 7f8a056..5472894 100644 --- a/cmd/tar-split/disasm.go +++ b/cmd/tar-split/disasm.go @@ -7,7 +7,7 @@ import ( "os" "github.com/Sirupsen/logrus" - "github.com/codegangsta/cli" + "github.com/urfave/cli" "github.com/vbatts/tar-split/tar/asm" "github.com/vbatts/tar-split/tar/storage" ) diff --git a/cmd/tar-split/main.go b/cmd/tar-split/main.go index c584352..8b4035f 100644 --- a/cmd/tar-split/main.go +++ b/cmd/tar-split/main.go @@ -4,7 +4,7 @@ import ( "os" "github.com/Sirupsen/logrus" - "github.com/codegangsta/cli" + "github.com/urfave/cli" "github.com/vbatts/tar-split/version" ) From 7560005f21d918cc59698527424fc6759877152f Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Mon, 13 Mar 2017 18:28:54 -0400 Subject: [PATCH 12/14] README: adding a golang report card Signed-off-by: Vincent Batts --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4c544d8..c2e7f48 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # tar-split [![Build Status](https://travis-ci.org/vbatts/tar-split.svg?branch=master)](https://travis-ci.org/vbatts/tar-split) +[![Go Report Card](https://goreportcard.com/badge/github.com/vbatts/tar-split)](https://goreportcard.com/report/github.com/vbatts/tar-split) Pristinely disassembling a tar archive, and stashing needed raw bytes and offsets to reassemble a validating original archive. From 245403c324d6ea47167227ee70aaef1c4c87ff43 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Mon, 13 Mar 2017 18:33:27 -0400 Subject: [PATCH 13/14] travis: test more go versions Thanks to @tianon, for pointing to https://github.com/travis-ci/travis-build/blob/5e3ef60b0d43b8ef56fb6fa77ba3269fbf945fa6/lib/travis/build/config.rb#L54-L70 Signed-off-by: Vincent Batts --- .travis.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2d9a842..dcce57a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,11 @@ language: go go: - tip - - 1.6.2 - - 1.5.4 + - 1.x + - 1.8.x + - 1.7.x + - 1.6.x + - 1.5.x # let us have pretty, fast Docker-based Travis workers! sudo: false From c6dd42815acecceec10987978f48488c37ead18f Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 14 Mar 2017 11:04:10 -0400 Subject: [PATCH 14/14] archive/tar: monotonic clock adjustment commit 0e3355903d2ebcf5ee9e76096f51ac9a116a9dbb upstream Signed-off-by: Vincent Batts --- archive/tar/tar_test.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/archive/tar/tar_test.go b/archive/tar/tar_test.go index d63c072..9ef319a 100644 --- a/archive/tar/tar_test.go +++ b/archive/tar/tar_test.go @@ -94,13 +94,12 @@ func TestRoundTrip(t *testing.T) { var b bytes.Buffer tw := NewWriter(&b) hdr := &Header{ - Name: "file.txt", - Uid: 1 << 21, // too big for 8 octal digits - Size: int64(len(data)), - ModTime: time.Now(), + Name: "file.txt", + Uid: 1 << 21, // too big for 8 octal digits + Size: int64(len(data)), + // https://github.com/golang/go/commit/0e3355903d2ebcf5ee9e76096f51ac9a116a9dbb#diff-d7bf2a98d7b57b6ff754ca406f1b7581R105 + ModTime: time.Now().AddDate(0, 0, 0).Round(1 * time.Second), } - // tar only supports second precision. - hdr.ModTime = hdr.ModTime.Add(-time.Duration(hdr.ModTime.Nanosecond()) * time.Nanosecond) if err := tw.WriteHeader(hdr); err != nil { t.Fatalf("tw.WriteHeader: %v", err) }