1
0
Fork 0
forked from mirrors/tar-split

Compare commits

..

20 commits

Author SHA1 Message Date
b9127a1393 Merge pull request #38 from vbatts/travis
travis: test more go versions
2017-03-14 11:24:38 -04:00
c6dd42815a
archive/tar: monotonic clock adjustment
commit 0e3355903d2ebcf5ee9e76096f51ac9a116a9dbb upstream

Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
2017-03-14 11:04:10 -04:00
245403c324
travis: test more go versions
Thanks to @tianon, for pointing to
5e3ef60b0d/lib/travis/build/config.rb (L54-L70)

Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
2017-03-14 08:38:13 -04:00
7560005f21
README: adding a golang report card
Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
2017-03-13 18:28:54 -04:00
bd4c5d64c3
main: switch import paths to urfave
Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
2016-09-27 02:54:18 +00:00
d3f1b54304
version: bump to v0.10.1
Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
2016-09-26 19:53:52 -04:00
f28028292a Merge branch 'master' of github.com:vbatts/tar-split 2016-09-26 19:52:55 -04:00
416fa5dcfe Merge pull request #36 from dmcgowan/fix-extra-nil-accounting
archive/tar: fix writing too many raw bytes
2016-09-26 18:31:47 -04:00
Derek McGowan
6b59e6942e archive/tar: fix writing too many raw bytes
When an EOF is read, only the part of the header buffer which
was read should be accounted for.

Signed-off-by: Derek McGowan <derek@mcgstyle.net>
2016-09-26 14:01:48 -07:00
7410961e75 tar/asm: failing test for lack of EOF nils
Reported-by: Derek McGowan <derek@mcgstyle.net>
Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
2016-09-26 13:39:03 -07:00
eb3808673d
version: bump to v0.10.0
Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
2016-09-23 11:01:58 -04:00
ae8540dc47 Merge pull request #34 from dmcgowan/fix-panic-issue-33
Fix panic in Next
2016-09-23 09:41:12 -04:00
Derek McGowan
e527e70d25 Fix panic in Next
readHeader should never return nil with a tr.err also nil.
To correct this, ensure tr.err never gets reset to nil followed
by a nil return.
2016-09-22 17:38:18 -07:00
6810cedb21 benchmark: add a comparison of 'archive/tar'
Since this project has forked logic of upstream 'archive/tar', this does
a brief comparison including the RawBytes usage.

```bash
$ go test -run="XXX" -bench=.
testing: warning: no tests to run
BenchmarkUpstreamTar-4                      2000            700809 ns/op
BenchmarkOurTarNoAccounting-4               2000            692055 ns/op
BenchmarkOurTarYesAccounting-4              2000            723184 ns/op
PASS
ok      vb/tar-split    4.461s
```

From this, the difference is negligible.

Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
2016-07-26 09:50:08 -04:00
28bc4c32f9 Merge pull request #32 from vbatts/fix-travis
travis: update golang versions
2016-06-26 15:00:37 -04:00
beaeceb06f travis: update golang versions
This is not saying that tar-split no longer works on go1.3 or go1.4, but
rather that the headache of `go vet` having a version dependent ability
to install it, makes it a headache in travis.

Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
2016-06-26 14:56:04 -04:00
54e3a92a60 Merge branch 'master' of github.com:vbatts/tar-split 2016-06-26 14:43:38 -04:00
354fd6cf34 cmd: add a disasm --no-stdout flag
Since sometimes you just need to > /dev/null

Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
2016-06-26 10:15:12 -04:00
226f7c7490 README: update archive/tar version reference
Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
2016-03-30 16:38:51 -04:00
e2a62d6b0d README.md: fix thumbnail
Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
2016-02-29 11:40:38 -05:00
13 changed files with 125 additions and 25 deletions

View file

@ -1,16 +1,17 @@
language: go
go:
- tip
- 1.5.1
- 1.4.3
- 1.3.3
- 1.x
- 1.8.x
- 1.7.x
- 1.6.x
- 1.5.x
# let us have pretty, fast Docker-based Travis workers!
sudo: false
install:
- go get -d ./...
- go get golang.org/x/tools/cmd/vet
script:
- go test -v ./...

View file

@ -1,6 +1,7 @@
# tar-split
[![Build Status](https://travis-ci.org/vbatts/tar-split.svg?branch=master)](https://travis-ci.org/vbatts/tar-split)
[![Go Report Card](https://goreportcard.com/badge/github.com/vbatts/tar-split)](https://goreportcard.com/report/github.com/vbatts/tar-split)
Pristinely disassembling a tar archive, and stashing needed raw bytes and offsets to reassemble a validating original archive.
@ -39,7 +40,7 @@ This demonstrates the `tar-split` command and how to assemble a tar archive from
This demonstrates the tar-split integration for docker-1.8. Providing consistent tar archives for the image layer content.
![docker tar-split demo](https://www.youtube.com/upload_thumbnail?v=tV_Dia8E8xw&t=2&ts=1445028436275)
![docker tar-split demo](https://i.ytimg.com/vi_webp/vh5wyjIOBtc/default.webp)
[youtube vide of docker layer checksums](https://youtu.be/tV_Dia8E8xw)
## Caveat
@ -66,7 +67,7 @@ Do not break the API of stdlib `archive/tar` in our fork (ideally find an upstre
## Std Version
The version of golang stdlib `archive/tar` is from go1.4.1, and their master branch around [a9dddb53f](https://github.com/golang/go/tree/a9dddb53f).
The version of golang stdlib `archive/tar` is from go1.6
It is minimally extended to expose the raw bytes of the TAR, rather than just the marshalled headers and file stream.

View file

@ -608,11 +608,12 @@ func (tr *Reader) readHeader() *Header {
header := tr.hdrBuff[:]
copy(header, zeroBlock)
if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil {
if n, err := io.ReadFull(tr.r, header); err != nil {
tr.err = err
// because it could read some of the block, but reach EOF first
if tr.err == io.EOF && tr.RawAccounting {
if _, tr.err = tr.rawBytes.Write(header); tr.err != nil {
return nil
if _, err := tr.rawBytes.Write(header[:n]); err != nil {
tr.err = err
}
}
return nil // io.EOF is okay here
@ -625,11 +626,12 @@ func (tr *Reader) readHeader() *Header {
// Two blocks of zero bytes marks the end of the archive.
if bytes.Equal(header, zeroBlock[0:blockSize]) {
if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil {
if n, err := io.ReadFull(tr.r, header); err != nil {
tr.err = err
// because it could read some of the block, but reach EOF first
if tr.err == io.EOF && tr.RawAccounting {
if _, tr.err = tr.rawBytes.Write(header); tr.err != nil {
return nil
if _, err := tr.rawBytes.Write(header[:n]); err != nil {
tr.err = err
}
}
return nil // io.EOF is okay here

View file

@ -94,13 +94,12 @@ func TestRoundTrip(t *testing.T) {
var b bytes.Buffer
tw := NewWriter(&b)
hdr := &Header{
Name: "file.txt",
Uid: 1 << 21, // too big for 8 octal digits
Size: int64(len(data)),
ModTime: time.Now(),
Name: "file.txt",
Uid: 1 << 21, // too big for 8 octal digits
Size: int64(len(data)),
// https://github.com/golang/go/commit/0e3355903d2ebcf5ee9e76096f51ac9a116a9dbb#diff-d7bf2a98d7b57b6ff754ca406f1b7581R105
ModTime: time.Now().AddDate(0, 0, 0).Round(1 * time.Second),
}
// tar only supports second precision.
hdr.ModTime = hdr.ModTime.Add(-time.Duration(hdr.ModTime.Nanosecond()) * time.Nanosecond)
if err := tw.WriteHeader(hdr); err != nil {
t.Fatalf("tw.WriteHeader: %v", err)
}

View file

@ -6,7 +6,7 @@ import (
"os"
"github.com/Sirupsen/logrus"
"github.com/codegangsta/cli"
"github.com/urfave/cli"
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
)

View file

@ -10,7 +10,7 @@ import (
"os"
"github.com/Sirupsen/logrus"
"github.com/codegangsta/cli"
"github.com/urfave/cli"
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
)

View file

@ -3,10 +3,11 @@ package main
import (
"compress/gzip"
"io"
"io/ioutil"
"os"
"github.com/Sirupsen/logrus"
"github.com/codegangsta/cli"
"github.com/urfave/cli"
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
)
@ -48,7 +49,13 @@ func CommandDisasm(c *cli.Context) {
if err != nil {
logrus.Fatal(err)
}
i, err := io.Copy(os.Stdout, its)
var out io.Writer
if c.Bool("no-stdout") {
out = ioutil.Discard
} else {
out = os.Stdout
}
i, err := io.Copy(out, its)
if err != nil {
logrus.Fatal(err)
}

View file

@ -4,7 +4,7 @@ import (
"os"
"github.com/Sirupsen/logrus"
"github.com/codegangsta/cli"
"github.com/urfave/cli"
"github.com/vbatts/tar-split/version"
)
@ -42,6 +42,10 @@ func main() {
Value: "tar-data.json.gz",
Usage: "output of disassembled tar stream",
},
cli.BoolFlag{
Name: "no-stdout",
Usage: "do not throughput the stream to STDOUT",
},
},
},
{

View file

@ -139,6 +139,8 @@ var testCases = []struct {
{"./testdata/longlink.tar.gz", "d9f6babe107b7247953dff6b5b5ae31a3a880add", 20480},
{"./testdata/fatlonglink.tar.gz", "8537f03f89aeef537382f8b0bb065d93e03b0be8", 26234880},
{"./testdata/iso-8859.tar.gz", "ddafa51cb03c74ec117ab366ee2240d13bba1ec3", 10240},
{"./testdata/extranils.tar.gz", "e187b4b3e739deaccc257342f4940f34403dc588", 10648},
{"./testdata/notenoughnils.tar.gz", "72f93f41efd95290baa5c174c234f5d4c22ce601", 512},
}
func TestTarStream(t *testing.T) {

BIN
tar/asm/testdata/extranils.tar.gz vendored Normal file

Binary file not shown.

BIN
tar/asm/testdata/notenoughnils.tar.gz vendored Normal file

Binary file not shown.

84
tar_benchmark_test.go Normal file
View file

@ -0,0 +1,84 @@
package tartest
import (
"io"
"io/ioutil"
"os"
"testing"
upTar "archive/tar"
ourTar "github.com/vbatts/tar-split/archive/tar"
)
var testfile = "./archive/tar/testdata/sparse-formats.tar"
func BenchmarkUpstreamTar(b *testing.B) {
for n := 0; n < b.N; n++ {
fh, err := os.Open(testfile)
if err != nil {
b.Fatal(err)
}
tr := upTar.NewReader(fh)
for {
_, err := tr.Next()
if err != nil {
if err == io.EOF {
break
}
fh.Close()
b.Fatal(err)
}
io.Copy(ioutil.Discard, tr)
}
fh.Close()
}
}
func BenchmarkOurTarNoAccounting(b *testing.B) {
for n := 0; n < b.N; n++ {
fh, err := os.Open(testfile)
if err != nil {
b.Fatal(err)
}
tr := ourTar.NewReader(fh)
tr.RawAccounting = false // this is default, but explicit here
for {
_, err := tr.Next()
if err != nil {
if err == io.EOF {
break
}
fh.Close()
b.Fatal(err)
}
io.Copy(ioutil.Discard, tr)
}
fh.Close()
}
}
func BenchmarkOurTarYesAccounting(b *testing.B) {
for n := 0; n < b.N; n++ {
fh, err := os.Open(testfile)
if err != nil {
b.Fatal(err)
}
tr := ourTar.NewReader(fh)
tr.RawAccounting = true // This enables mechanics for collecting raw bytes
for {
_ = tr.RawBytes()
_, err := tr.Next()
_ = tr.RawBytes()
if err != nil {
if err == io.EOF {
break
}
fh.Close()
b.Fatal(err)
}
io.Copy(ioutil.Discard, tr)
_ = tr.RawBytes()
}
fh.Close()
}
}

View file

@ -1,7 +1,7 @@
package version
// AUTO-GENEREATED. DO NOT EDIT
// 2016-02-15 09:43:15.919197079 -0500 EST
// 2016-09-26 19:53:30.825879 -0400 EDT
// VERSION is the generated version from /home/vbatts/src/vb/tar-split/version
var VERSION = "v0.9.13-19-862ccd05b"
var VERSION = "v0.10.1-4-gf280282"