From 7835d261d84840ff79a2cbe53a23727e5332d071 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Thu, 20 Aug 2015 12:39:50 -0700 Subject: [PATCH] Add generic content digest tool Previously a useful gist, this changeset polishes the original tarsum tool into a utility that can be used to calculate content digests. Any algorithm from the digest package is supported with additional support from tarsum. This tool is very useful for quickly checking backend digests and verifying correctness. Signed-off-by: Stephen J Day --- Makefile | 6 ++- cmd/digest/main.go | 129 +++++++++++++++++++++++++++++++++++++++++++++ digest/digest.go | 11 ++-- digest/digester.go | 28 ++++++++++ 4 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 cmd/digest/main.go diff --git a/Makefile b/Makefile index b9dbf66a..8b2d8fde 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,10 @@ ${PREFIX}/bin/registry: version/version.go $(shell find . -type f -name '*.go') @echo "+ $@" @go build -tags "${DOCKER_BUILDTAGS}" -o $@ ${GO_LDFLAGS} ${GO_GCFLAGS} ./cmd/registry +${PREFIX}/bin/digest: version/version.go $(shell find . -type f -name '*.go') + @echo "+ $@" + @go build -tags "${DOCKER_BUILDTAGS}" -o $@ ${GO_LDFLAGS} ${GO_GCFLAGS} ./cmd/digest + ${PREFIX}/bin/registry-api-descriptor-template: version/version.go $(shell find . -type f -name '*.go') @echo "+ $@" @go build -o $@ ${GO_LDFLAGS} ${GO_GCFLAGS} ./cmd/registry-api-descriptor-template @@ -62,7 +66,7 @@ test-full: @echo "+ $@" @go test ./... -binaries: ${PREFIX}/bin/registry ${PREFIX}/bin/registry-api-descriptor-template +binaries: ${PREFIX}/bin/registry ${PREFIX}/bin/digest ${PREFIX}/bin/registry-api-descriptor-template @echo "+ $@" clean: diff --git a/cmd/digest/main.go b/cmd/digest/main.go new file mode 100644 index 00000000..f4870d92 --- /dev/null +++ b/cmd/digest/main.go @@ -0,0 +1,129 @@ +package main + +import ( + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "strings" + + "github.com/docker/distribution/digest" + "github.com/docker/distribution/version" + "github.com/docker/docker/pkg/tarsum" +) + +var ( + algorithm = digest.Canonical + showVersion bool +) + +type job struct { + name string + reader io.Reader +} + +func init() { + flag.Var(&algorithm, "a", "select the digest algorithm (shorthand)") + flag.Var(&algorithm, "algorithm", "select the digest algorithm") + flag.BoolVar(&showVersion, "version", false, "show the version and exit") + + log.SetFlags(0) + log.SetPrefix(os.Args[0] + ": ") +} + +func usage() { + fmt.Fprintf(os.Stderr, "usage: %s [files...]\n", os.Args[0]) + fmt.Fprintf(os.Stderr, ` +Calculate the digest of one or more input files, emitting the result +to standard out. If no files are provided, the digest of stdin will +be calculated. + +`) + flag.PrintDefaults() +} + +func unsupported() { + log.Fatalf("unsupported digest algorithm: %v", algorithm) +} + +func main() { + var jobs []job + + flag.Usage = usage + flag.Parse() + if showVersion { + version.PrintVersion() + return + } + + var fail bool // if we fail on one item, foul the exit code + if flag.NArg() > 0 { + for _, path := range flag.Args() { + fp, err := os.Open(path) + + if err != nil { + log.Printf("%s: %v", path, err) + fail = true + continue + } + defer fp.Close() + + jobs = append(jobs, job{name: path, reader: fp}) + } + } else { + // just read stdin + jobs = append(jobs, job{name: "-", reader: os.Stdin}) + } + + digestFn := algorithm.FromReader + + if !algorithm.Available() { + // we cannot digest if is not available. An exception is made for + // tarsum. + if !strings.HasPrefix(algorithm.String(), "tarsum") { + unsupported() + } + + var version tarsum.Version + if algorithm == "tarsum" { + // small hack: if we just have tarsum, use latest + version = tarsum.Version1 + } else { + var err error + version, err = tarsum.GetVersionFromTarsum(algorithm.String()) + if err != nil { + unsupported() + } + } + + digestFn = func(rd io.Reader) (digest.Digest, error) { + ts, err := tarsum.NewTarSum(rd, true, version) + if err != nil { + return "", err + } + + if _, err := io.Copy(ioutil.Discard, ts); err != nil { + return "", err + } + + return digest.Digest(ts.Sum(nil)), nil + } + } + + for _, job := range jobs { + dgst, err := digestFn(job.reader) + if err != nil { + log.Printf("%s: %v", job.name, err) + fail = true + continue + } + + fmt.Printf("%v\t%s\n", dgst, job.name) + } + + if fail { + os.Exit(1) + } +} diff --git a/digest/digest.go b/digest/digest.go index 68991685..a0221216 100644 --- a/digest/digest.go +++ b/digest/digest.go @@ -70,15 +70,10 @@ func ParseDigest(s string) (Digest, error) { return d, d.Validate() } -// FromReader returns the most valid digest for the underlying content. +// FromReader returns the most valid digest for the underlying content using +// the canonical digest algorithm. func FromReader(rd io.Reader) (Digest, error) { - digester := Canonical.New() - - if _, err := io.Copy(digester.Hash(), rd); err != nil { - return "", err - } - - return digester.Digest(), nil + return Canonical.FromReader(rd) } // FromTarArchive produces a tarsum digest from reader rd. diff --git a/digest/digester.go b/digest/digester.go index 556dd93a..4f03e189 100644 --- a/digest/digester.go +++ b/digest/digester.go @@ -3,6 +3,7 @@ package digest import ( "crypto" "hash" + "io" ) // Algorithm identifies and implementation of a digester by an identifier. @@ -49,6 +50,22 @@ func (a Algorithm) Available() bool { return h.Available() } +func (a Algorithm) String() string { + return string(a) +} + +// Set implemented to allow use of Algorithm as a command line flag. +func (a *Algorithm) Set(value string) error { + if value == "" { + *a = Canonical + } else { + // just do a type conversion, support is queried with Available. + *a = Algorithm(value) + } + + return nil +} + // New returns a new digester for the specified algorithm. If the algorithm // does not have a digester implementation, nil will be returned. This can be // checked by calling Available before calling New. @@ -69,6 +86,17 @@ func (a Algorithm) Hash() hash.Hash { return algorithms[a].New() } +// FromReader returns the digest of the reader using the algorithm. +func (a Algorithm) FromReader(rd io.Reader) (Digest, error) { + digester := a.New() + + if _, err := io.Copy(digester.Hash(), rd); err != nil { + return "", err + } + + return digester.Digest(), nil +} + // TODO(stevvooe): Allow resolution of verifiers using the digest type and // this registration system.