vbatts/registry
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Remove tarsum support for digest package

Browse source 
tarsum is not actually used by the registry. Remove support for it.

Convert numerous uses in unit tests to SHA256.

Update docs to remove mentions of tarsums (which were often inaccurate).

Remove tarsum dependency.

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>
This commit is contained in:
Aaron Lehmann 2015-12-15 17:18:13 -08:00
parent 200cbe8b8e
commit 4c850e7165
47 changed files with 90 additions and 2358 deletions
Download patch file Download diff file Expand all files Collapse all files

37
digest/digest.go
View file

@ -4,17 +4,11 @@ import (
"fmt"
"hash"
"io"
"io/ioutil"
"regexp"
"strings"
"github.com/docker/docker/pkg/tarsum"
)
const (
// DigestTarSumV1EmptyTar is the digest for the empty tar file.
DigestTarSumV1EmptyTar = "tarsum.v1+sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
// DigestSha256EmptyTar is the canonical sha256 digest of empty data
DigestSha256EmptyTar = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
)
@ -28,11 +22,6 @@ const (
//
// sha256:7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc
//
// More important for this code base, this type is compatible with tarsum
// digests. For example, the following would be a valid Digest:
//
// tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b
//
// This allows to abstract the digest behind this type and work only in those
// terms.
type Digest string
@ -78,25 +67,6 @@ func FromReader(rd io.Reader) (Digest, error) {
return Canonical.FromReader(rd)
}
// FromTarArchive produces a tarsum digest from reader rd.
func FromTarArchive(rd io.Reader) (Digest, error) {
ts, err := tarsum.NewTarSum(rd, true, tarsum.Version1)
if err != nil {
return "", err
}
if _, err := io.Copy(ioutil.Discard, ts); err != nil {
return "", err
}
d, err := ParseDigest(ts.Sum(nil))
if err != nil {
return "", err
}
return d, nil
}
// FromBytes digests the input and returns a Digest.
func FromBytes(p []byte) Digest {
digester := Canonical.New()
@ -117,13 +87,6 @@ func FromBytes(p []byte) Digest {
// error if not.
func (d Digest) Validate() error {
s := string(d)
// Common case will be tarsum
_, err := ParseTarSum(s)
if err == nil {
return nil
}
// Continue on for general parser
if !DigestRegexpAnchored.MatchString(s) {
return ErrDigestInvalidFormat

39
digest/digest_test.go
View file

@ -1,8 +1,6 @@
package digest
import (
"bytes"
"io"
"testing"
)
@ -13,21 +11,6 @@ func TestParseDigest(t *testing.T) {
algorithm Algorithm
hex string
}{
{
input: "tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b",
algorithm: "tarsum+sha256",
hex: "e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b",
},
{
input: "tarsum.dev+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b",
algorithm: "tarsum.dev+sha256",
hex: "e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b",
},
{
input: "tarsum.v1+sha256:220a60ecd4a3c32c282622a625a54db9ba0ff55b5ba9c29c7064a2bc358b6a3e",
algorithm: "tarsum.v1+sha256",
hex: "220a60ecd4a3c32c282622a625a54db9ba0ff55b5ba9c29c7064a2bc358b6a3e",
},
{
input: "sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b",
algorithm: "sha256",
@ -97,25 +80,3 @@ func TestParseDigest(t *testing.T) {
}
}
}
// A few test cases used to fix behavior we expect in storage backend.
func TestFromTarArchiveZeroLength(t *testing.T) {
checkTarsumDigest(t, "zero-length archive", bytes.NewReader([]byte{}), DigestTarSumV1EmptyTar)
}
func TestFromTarArchiveEmptyTar(t *testing.T) {
// String of 1024 zeros is a valid, empty tar file.
checkTarsumDigest(t, "1024 zero bytes", bytes.NewReader(bytes.Repeat([]byte("\x00"), 1024)), DigestTarSumV1EmptyTar)
}
func checkTarsumDigest(t *testing.T, msg string, rd io.Reader, expected Digest) {
dgst, err := FromTarArchive(rd)
if err != nil {
t.Fatalf("unexpected error digesting %s: %v", msg, err)
}
if dgst != expected {
t.Fatalf("unexpected digest for %s: %q != %q", msg, dgst, expected)
}
}

7
digest/digester.go
View file

@ -13,10 +13,9 @@ type Algorithm string
// supported digest types
const (
SHA256 Algorithm = "sha256" // sha256 with hex encoding
SHA384 Algorithm = "sha384" // sha384 with hex encoding
SHA512 Algorithm = "sha512" // sha512 with hex encoding
TarsumV1SHA256 Algorithm = "tarsum+v1+sha256" // supported tarsum version, verification only
SHA256 Algorithm = "sha256" // sha256 with hex encoding
SHA384 Algorithm = "sha384" // sha384 with hex encoding
SHA512 Algorithm = "sha512" // sha512 with hex encoding
// Canonical is the primary digest algorithm used with the distribution
// project. Other digests may be used but this one is the primary storage

14
digest/doc.go
View file

@ -1,7 +1,7 @@
// Package digest provides a generalized type to opaquely represent message
// digests and their operations within the registry. The Digest type is
// designed to serve as a flexible identifier in a content-addressable system.
// More importantly, it provides tools and wrappers to work with tarsums and
// More importantly, it provides tools and wrappers to work with
// hash.Hash-based digests with little effort.
//
// Basics
@ -16,17 +16,7 @@
// sha256:7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc
//
// In this case, the string "sha256" is the algorithm and the hex bytes are
// the "digest". A tarsum example will be more illustrative of the use case
// involved in the registry:
//
// tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b
//
// For this, we consider the algorithm to be "tarsum+sha256". Prudent
// applications will favor the ParseDigest function to verify the format over
// using simple type casts. However, a normal string can be cast as a digest
// with a simple type conversion:
//
// Digest("tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b")
// the "digest".
//
// Because the Digest type is simply a string, once a valid Digest is
// obtained, comparisons are cheap, quick and simple to express with the

70
digest/tarsum.go
View file

@ -1,70 +0,0 @@
package digest
import (
"fmt"
"regexp"
)
// TarsumRegexp defines a regular expression to match tarsum identifiers.
var TarsumRegexp = regexp.MustCompile("tarsum(?:.[a-z0-9]+)?\\+[a-zA-Z0-9]+:[A-Fa-f0-9]+")
// TarsumRegexpCapturing defines a regular expression to match tarsum identifiers with
// capture groups corresponding to each component.
var TarsumRegexpCapturing = regexp.MustCompile("(tarsum)(.([a-z0-9]+))?\\+([a-zA-Z0-9]+):([A-Fa-f0-9]+)")
// TarSumInfo contains information about a parsed tarsum.
type TarSumInfo struct {
// Version contains the version of the tarsum.
Version string
// Algorithm contains the algorithm for the final digest
Algorithm string
// Digest contains the hex-encoded digest.
Digest string
}
// InvalidTarSumError provides informations about a TarSum that cannot be parsed
// by ParseTarSum.
type InvalidTarSumError string
func (e InvalidTarSumError) Error() string {
return fmt.Sprintf("invalid tarsum: %q", string(e))
}
// ParseTarSum parses a tarsum string into its components of interest. For
// example, this method may receive the tarsum in the following format:
//
// tarsum.v1+sha256:220a60ecd4a3c32c282622a625a54db9ba0ff55b5ba9c29c7064a2bc358b6a3e
//
// The function will return the following:
//
// TarSumInfo{
// Version: "v1",
// Algorithm: "sha256",
// Digest: "220a60ecd4a3c32c282622a625a54db9ba0ff55b5ba9c29c7064a2bc358b6a3e",
// }
//
func ParseTarSum(tarSum string) (tsi TarSumInfo, err error) {
components := TarsumRegexpCapturing.FindStringSubmatch(tarSum)
if len(components) != 1+TarsumRegexpCapturing.NumSubexp() {
return TarSumInfo{}, InvalidTarSumError(tarSum)
}
return TarSumInfo{
Version: components[3],
Algorithm: components[4],
Digest: components[5],
}, nil
}
// String returns the valid, string representation of the tarsum info.
func (tsi TarSumInfo) String() string {
if tsi.Version == "" {
return fmt.Sprintf("tarsum+%s:%s", tsi.Algorithm, tsi.Digest)
}
return fmt.Sprintf("tarsum.%s+%s:%s", tsi.Version, tsi.Algorithm, tsi.Digest)
}

79
digest/tarsum_test.go
View file

@ -1,79 +0,0 @@
package digest
import (
"reflect"
"testing"
)
func TestParseTarSumComponents(t *testing.T) {
for _, testcase := range []struct {
input string
expected TarSumInfo
err error
}{
{
input: "tarsum.v1+sha256:220a60ecd4a3c32c282622a625a54db9ba0ff55b5ba9c29c7064a2bc358b6a3e",
expected: TarSumInfo{
Version: "v1",
Algorithm: "sha256",
Digest: "220a60ecd4a3c32c282622a625a54db9ba0ff55b5ba9c29c7064a2bc358b6a3e",
},
},
{
input: "",
err: InvalidTarSumError(""),
},
{
input: "purejunk",
err: InvalidTarSumError("purejunk"),
},
{
input: "tarsum.v23+test:12341234123412341effefefe",
expected: TarSumInfo{
Version: "v23",
Algorithm: "test",
Digest: "12341234123412341effefefe",
},
},
// The following test cases are ported from docker core
{
// Version 0 tarsum
input: "tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b",
expected: TarSumInfo{
Algorithm: "sha256",
Digest: "e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b",
},
},
{
// Dev version tarsum
input: "tarsum.dev+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b",
expected: TarSumInfo{
Version: "dev",
Algorithm: "sha256",
Digest: "e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b",
},
},
} {
tsi, err := ParseTarSum(testcase.input)
if err != nil {
if testcase.err != nil && err == testcase.err {
continue // passes
}
t.Fatalf("unexpected error parsing tarsum: %v", err)
}
if testcase.err != nil {
t.Fatalf("expected error not encountered on %q: %v", testcase.input, testcase.err)
}
if !reflect.DeepEqual(tsi, testcase.expected) {
t.Fatalf("expected tarsum info: %v != %v", tsi, testcase.expected)
}
if testcase.input != tsi.String() {
t.Fatalf("input should equal output: %q != %q", tsi.String(), testcase.input)
}
}
}

86
digest/verifiers.go
View file

@ -3,9 +3,6 @@ package digest
import (
"hash"
"io"
"io/ioutil"
"github.com/docker/docker/pkg/tarsum"
)
// Verifier presents a general verification interface to be used with message
@ -27,70 +24,10 @@ func NewDigestVerifier(d Digest) (Verifier, error) {
return nil, err
}
alg := d.Algorithm()
switch alg {
case "sha256", "sha384", "sha512":
return hashVerifier{
hash: alg.Hash(),
digest: d,
}, nil
default:
// Assume we have a tarsum.
version, err := tarsum.GetVersionFromTarsum(string(d))
if err != nil {
return nil, err
}
pr, pw := io.Pipe()
// TODO(stevvooe): We may actually want to ban the earlier versions of
// tarsum. That decision may not be the place of the verifier.
ts, err := tarsum.NewTarSum(pr, true, version)
if err != nil {
return nil, err
}
// TODO(sday): Ick! A goroutine per digest verification? We'll have to
// get the tarsum library to export an io.Writer variant.
go func() {
if _, err := io.Copy(ioutil.Discard, ts); err != nil {
pr.CloseWithError(err)
} else {
pr.Close()
}
}()
return &tarsumVerifier{
digest: d,
ts: ts,
pr: pr,
pw: pw,
}, nil
}
}
// NewLengthVerifier returns a verifier that returns true when the number of
// read bytes equals the expected parameter.
func NewLengthVerifier(expected int64) Verifier {
return &lengthVerifier{
expected: expected,
}
}
type lengthVerifier struct {
expected int64 // expected bytes read
len int64 // bytes read
}
func (lv *lengthVerifier) Write(p []byte) (n int, err error) {
n = len(p)
lv.len += int64(n)
return n, err
}
func (lv *lengthVerifier) Verified() bool {
return lv.expected == lv.len
return hashVerifier{
hash: d.Algorithm().Hash(),
digest: d,
}, nil
}
type hashVerifier struct {
@ -105,18 +42,3 @@ func (hv hashVerifier) Write(p []byte) (n int, err error) {
func (hv hashVerifier) Verified() bool {
return hv.digest == NewDigest(hv.digest.Algorithm(), hv.hash)
}
type tarsumVerifier struct {
digest Digest
ts tarsum.TarSum
pr *io.PipeReader
pw *io.PipeWriter
}
func (tv *tarsumVerifier) Write(p []byte) (n int, err error) {
return tv.pw.Write(p)
}
func (tv *tarsumVerifier) Verified() bool {
return tv.digest == Digest(tv.ts.Sum(nil))
}

114
digest/verifiers_test.go
View file

@ -3,13 +3,8 @@ package digest
import (
"bytes"
"crypto/rand"
"encoding/base64"
"io"
"os"
"strings"
"testing"
"github.com/docker/distribution/testutil"
)
func TestDigestVerifier(t *testing.T) {
@ -27,43 +22,6 @@ func TestDigestVerifier(t *testing.T) {
if !verifier.Verified() {
t.Fatalf("bytes not verified")
}
tf, tarSum, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating tarfile: %v", err)
}
digest, err = FromTarArchive(tf)
if err != nil {
t.Fatalf("error digesting tarsum: %v", err)
}
if digest.String() != tarSum {
t.Fatalf("unexpected digest: %q != %q", digest.String(), tarSum)
}
expectedSize, _ := tf.Seek(0, os.SEEK_END) // Get tar file size
tf.Seek(0, os.SEEK_SET) // seek back
// This is the most relevant example for the registry application. It's
// effectively a read through pipeline, where the final sink is the digest
// verifier.
verifier, err = NewDigestVerifier(digest)
if err != nil {
t.Fatalf("unexpected error getting digest verifier: %s", err)
}
lengthVerifier := NewLengthVerifier(expectedSize)
rd := io.TeeReader(tf, lengthVerifier)
io.Copy(verifier, rd)
if !lengthVerifier.Verified() {
t.Fatalf("verifier detected incorrect length")
}
if !verifier.Verified() {
t.Fatalf("bytes not verified")
}
}
// TestVerifierUnsupportedDigest ensures that unsupported digest validation is
@ -81,79 +39,11 @@ func TestVerifierUnsupportedDigest(t *testing.T) {
}
}
// TestJunkNoDeadlock ensures that junk input into a digest verifier properly
// returns errors from the tarsum library. Specifically, we pass in a file
// with a "bad header" and should see the error from the io.Copy to verifier.
// This has been seen with gzipped tarfiles, mishandled by the tarsum package,
// but also on junk input, such as html.
func TestJunkNoDeadlock(t *testing.T) {
expected := Digest("tarsum.dev+sha256:62e15750aae345f6303469a94892e66365cc5e3abdf8d7cb8b329f8fb912e473")
junk := bytes.Repeat([]byte{'a'}, 1024)
verifier, err := NewDigestVerifier(expected)
if err != nil {
t.Fatalf("unexpected error creating verifier: %v", err)
}
rd := bytes.NewReader(junk)
if _, err := io.Copy(verifier, rd); err == nil {
t.Fatalf("unexpected error verifying input data: %v", err)
}
}
// TestBadTarNoDeadlock runs a tar with a "bad" tar header through digest
// verifier, ensuring that the verifier returns an error properly.
func TestBadTarNoDeadlock(t *testing.T) {
// TODO(stevvooe): This test is exposing a bug in tarsum where if we pass
// a gzipped tar file into tarsum, the library returns an error. This
// should actually work. When the tarsum package is fixed, this test will
// fail and we can remove this test or invert it.
// This tarfile was causing deadlocks in verifiers due mishandled copy error.
// This is a gzipped tar, which we typically don't see but should handle.
//
// From https://registry-1.docker.io/v2/library/ubuntu/blobs/tarsum.dev+sha256:62e15750aae345f6303469a94892e66365cc5e3abdf8d7cb8b329f8fb912e473
const badTar = `
H4sIAAAJbogA/0otSdZnoDEwMDAxMDc1BdJggE6D2YZGJobGBmbGRsZAdYYGBkZGDAqmtHYYCJQW
lyQWAZ1CqTnonhsiAAAAAP//AsV/YkEJTdMAGfFvZmA2Gv/0AAAAAAD//4LFf3F+aVFyarFeTmZx
CbXtAOVnMxMTXPFvbGpmjhb/xobmwPinSyCO8PgHAAAA///EVU9v2z4MvedTEMihl9a5/26/YTkU
yNKiTTDsKMt0rE0WDYmK628/ym7+bFmH2DksQACbIB/5+J7kObwiQsXc/LdYVGibLObRccw01Qv5
19EZ7hbbZudVgWtiDFCSh4paYII4xOVxNgeHLXrYow+GXAAqgSuEQhzlTR5ZgtlsVmB+aKe8rswe
zzsOjwtoPGoTEGplHHhMCJqxSNUPwesbEGbzOXxR34VCHndQmjfhUKhEq/FURI0FqJKFR5q9NE5Z
qbaoBGoglAB+5TSK0sOh3c3UPkRKE25dEg8dDzzIWmqN2wG3BNY4qRL1VFFAoJJb5SXHU90n34nk
SUS8S0AeGwqGyXdZel1nn7KLGhPO0kDeluvN48ty9Q2269ft8/PTy2b5GfKuh9/2LBIWo6oz+N8G
uodmWLETg0mW4lMP4XYYCL4+rlawftpIO40SA+W6Yci9wRZE1MNOjmyGdhBQRy9OHpqOdOGh/wT7
nZdOkHZ650uIK+WrVZdkgErJfnNEJysLnI5FSAj4xuiCQNpOIoNWmhyLByVHxEpLf3dkr+k9KMsV
xV0FhiVB21hgD3V5XwSqRdOmsUYr7oNtZXTVzyTHc2/kqokBy2ihRMVRTN+78goP5Ur/aMhz+KOJ
3h2UsK43kdwDo0Q9jfD7ie2RRur7MdpIrx1Z3X4j/Q1qCswN9r/EGCvXiUy0fI4xeSknnH/92T/+
fgIAAP//GkWjYBSMXAAIAAD//2zZtzAAEgAA`
expected := Digest("tarsum.dev+sha256:62e15750aae345f6303469a94892e66365cc5e3abdf8d7cb8b329f8fb912e473")
verifier, err := NewDigestVerifier(expected)
if err != nil {
t.Fatalf("unexpected error creating verifier: %v", err)
}
rd := base64.NewDecoder(base64.StdEncoding, strings.NewReader(badTar))
if _, err := io.Copy(verifier, rd); err == nil {
t.Fatalf("unexpected error verifying input data: %v", err)
}
if verifier.Verified() {
// For now, we expect an error, since tarsum library cannot handle
// compressed tars (!!!).
t.Fatalf("no error received after invalid tar")
}
}
// TODO(stevvooe): Add benchmarks to measure bytes/second throughput for
// DigestVerifier. We should be tarsum/gzip limited for common cases but we
// want to verify this.
// DigestVerifier.
//
// The relevant benchmarks for comparison can be run with the following
// The relevant benchmark for comparison can be run with the following
// commands:
//
// go test -bench . crypto/sha1
// go test -bench . github.com/docker/docker/pkg/tarsum
//