dist: expand functionality of the dist tool

With this change, we add the following commands to the dist tool:

- `ingest`: verify and accept content into storage
- `active`: display active ingest processes
- `list`: list content in storage
- `path`: provide a path to a blob by digest
- `delete`: remove a piece of content from storage

We demonstrate the utility with the following shell pipeline:

```
$ ./dist fetch docker.io/library/redis latest mediatype:application/vnd.docker.distribution.manifest.v2+json | \
    jq -r '.layers[] | "./dist fetch docker.io/library/redis "+.digest + "| ./dist ingest --expected-digest "+.digest+" --expected-size "+(.size | tostring) +" docker.io/library/redis@"+.digest' | xargs -I{} -P10 -n1 sh -c "{}"
```

The above fetches a manifest, pipes it to jq, which assembles a shell
pipeline to ingest each layer into the content store. Because the
transactions are keyed by their digest, concurrent downloads and
downloads of repeated content are ignored. Each process is then executed
parallel using xargs.

Put shortly, this is a parallel layer download.

In a separate shell session, could monitor the active downloads with the
following:

```
$ watch -n0.2 ./dist active
```

For now, the content is downloaded into `.content` in the current
working directory. To watch the contents of this directory, you can use
the following:

```
$ watch -n0.2 tree .content
```

This will help to understand what is going on internally.

To get access to the layers, you can use the path command:

```
$./dist path sha256:010c454d55e53059beaba4044116ea4636f8dd8181e975d893931c7e7204fffa
sha256:010c454d55e53059beaba4044116ea4636f8dd8181e975d893931c7e7204fffa /home/sjd/go/src/github.com/docker/containerd/.content/blobs/sha256/010c454d55e53059beaba4044116ea4636f8dd8181e975d893931c7e7204fffa
```

When you are done, you can clear out the content with the classic xargs
pipeline:

```
$ ./dist list -q | xargs ./dist delete
```

Note that this is mostly a POC. Things like failed downloads and
abandoned download cleanup aren't quite handled. We'll probably make
adjustments around how content store transactions are handled to address
this.

From here, we'll build out full image pull and create tooling to get
runtime bundles from the fetched content.

Signed-off-by: Stephen J Day <stephen.day@docker.com>

cmd/dist/active.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"text/tabwriter"
+	"time"
+
+	"github.com/docker/containerd/content"
+	units "github.com/docker/go-units"
+	"github.com/urfave/cli"
+)
+
+var activeCommand = cli.Command{
+	Name:        "active",
+	Usage:       "display active transfers.",
+	ArgsUsage:   "[flags] [<key>, ...]",
+	Description: `Display the ongoing transfers.`,
+	Flags: []cli.Flag{
+		cli.DurationFlag{
+			Name:   "timeout, t",
+			Usage:  "total timeout for fetch",
+			EnvVar: "CONTAINERD_FETCH_TIMEOUT",
+		},
+		cli.StringFlag{
+			Name:  "root",
+			Usage: "path to content store root",
+			Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content
+		},
+	},
+	Action: func(context *cli.Context) error {
+		var (
+			// ctx  = contextpkg.Background()
+			root = context.String("root")
+		)
+
+		if !filepath.IsAbs(root) {
+			var err error
+			root, err = filepath.Abs(root)
+			if err != nil {
+				return err
+			}
+		}
+
+		cs, err := content.Open(root)
+		if err != nil {
+			return err
+		}
+
+		active, err := cs.Active()
+		if err != nil {
+			return err
+		}
+
+		tw := tabwriter.NewWriter(os.Stdout, 1, 8, 1, '\t', 0)
+		fmt.Fprintf(tw, "REF\tSIZE\tAGE\n")
+		for _, active := range active {
+			fmt.Fprintf(tw, "%s\t%s\t%s\n",
+				active.Ref,
+				units.HumanSize(float64(active.Size)),
+				units.HumanDuration(time.Since(active.ModTime)))
+		}
+		tw.Flush()
+
+		return nil
+	},
+}

cmd/dist/delete.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	contextpkg "context"
+	"fmt"
+	"path/filepath"
+
+	"github.com/docker/containerd/content"
+	"github.com/docker/containerd/log"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/urfave/cli"
+)
+
+var deleteCommand = cli.Command{
+	Name:      "delete",
+	Aliases:   []string{"del"},
+	Usage:     "permanently delete one or more blobs.",
+	ArgsUsage: "[flags] [<digest>, ...]",
+	Description: `Delete one or more blobs permanently. Successfully deleted
+	blobs are printed to stdout.`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "root",
+			Usage: "path to content store root",
+			Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content
+		},
+	},
+	Action: func(context *cli.Context) error {
+		var (
+			ctx       = contextpkg.Background()
+			root      = context.String("root")
+			args      = []string(context.Args())
+			exitError error
+		)
+
+		if !filepath.IsAbs(root) {
+			var err error
+			root, err = filepath.Abs(root)
+			if err != nil {
+				return err
+			}
+		}
+
+		cs, err := content.Open(root)
+		if err != nil {
+			return err
+		}
+
+		for _, arg := range args {
+			dgst, err := digest.Parse(arg)
+			if err != nil {
+				if exitError == nil {
+					exitError = err
+				}
+				log.G(ctx).WithError(err).Errorf("could not delete %v", dgst)
+				continue
+			}
+
+			if err := cs.Delete(dgst); err != nil {
+				if exitError == nil {
+					exitError = err
+				}
+				log.G(ctx).WithError(err).Errorf("could not delete %v", dgst)
+				continue
+			}
+
+			fmt.Println(dgst)
+		}
+
+		return exitError
+	},
+}

cmd/dist/ingest.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	contextpkg "context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/docker/containerd/content"
+	"github.com/opencontainers/go-digest"
+	"github.com/urfave/cli"
+)
+
+var ingestCommand = cli.Command{
+	Name:        "ingest",
+	Usage:       "accept content into the store",
+	ArgsUsage:   "[flags] <key>",
+	Description: `Ingest objects into the local content store.`,
+	Flags: []cli.Flag{
+		cli.DurationFlag{
+			Name:   "timeout",
+			Usage:  "total timeout for fetch",
+			EnvVar: "CONTAINERD_FETCH_TIMEOUT",
+		},
+		cli.StringFlag{
+			Name:   "path, p",
+			Usage:  "path to content store",
+			Value:  ".content", // TODO(stevvooe): for now, just use the PWD/.content
+			EnvVar: "CONTAINERD_DIST_CONTENT_STORE",
+		},
+		cli.Int64Flag{
+			Name:  "expected-size",
+			Usage: "validate against provided size",
+		},
+		cli.StringFlag{
+			Name:  "expected-digest",
+			Usage: "verify content against expected digest",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		var (
+			ctx            = contextpkg.Background()
+			timeout        = context.Duration("timeout")
+			root           = context.String("path")
+			ref            = context.Args().First()
+			expectedSize   = context.Int64("expected-size")
+			expectedDigest = digest.Digest(context.String("expected-digest"))
+		)
+
+		if timeout > 0 {
+			var cancel func()
+			ctx, cancel = contextpkg.WithTimeout(ctx, timeout)
+			defer cancel()
+		}
+
+		if err := expectedDigest.Validate(); expectedDigest != "" && err != nil {
+			return err
+		}
+
+		if !filepath.IsAbs(root) {
+			var err error
+			root, err = filepath.Abs(root)
+			if err != nil {
+				return err
+			}
+		}
+
+		cs, err := content.Open(root)
+		if err != nil {
+			return err
+		}
+
+		if expectedDigest != "" {
+			if ok, err := cs.Exists(expectedDigest); err != nil {
+				return err
+			} else if ok {
+				fmt.Fprintf(os.Stderr, "content with digest %v already exists\n", expectedDigest)
+				return nil
+			}
+		}
+
+		if ref == "" {
+			if expectedDigest == "" {
+				return fmt.Errorf("must specify a transaction reference or expected digest")
+			}
+
+			ref = strings.Replace(expectedDigest.String(), ":", "-", -1)
+		}
+
+		// TODO(stevvooe): Allow ingest to be reentrant. Currently, we expect
+		// all data to be written in a single invocation. Allow multiple writes
+		// to the same transaction key followed by a commit.
+		return content.WriteBlob(cs, os.Stdin, ref, expectedSize, expectedDigest)
+	},
+}

cmd/dist/list.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	contextpkg "context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"text/tabwriter"
+	"time"
+
+	"github.com/docker/containerd/content"
+	"github.com/docker/containerd/log"
+	units "github.com/docker/go-units"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/urfave/cli"
+)
+
+var listCommand = cli.Command{
+	Name:        "list",
+	Aliases:     []string{"ls"},
+	Usage:       "list all blobs in the store.",
+	ArgsUsage:   "[flags] [<prefix>, ...]",
+	Description: `List blobs in the content store.`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "root",
+			Usage: "path to content store root",
+			Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content
+		},
+		cli.BoolFlag{
+			Name:  "quiet, q",
+			Usage: "print only the blob digest",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		var (
+			ctx   = contextpkg.Background()
+			root  = context.String("root")
+			quiet = context.Bool("quiet")
+			args  = []string(context.Args())
+		)
+
+		if !filepath.IsAbs(root) {
+			var err error
+			root, err = filepath.Abs(root)
+			if err != nil {
+				return err
+			}
+		}
+
+		cs, err := content.Open(root)
+		if err != nil {
+			return err
+		}
+
+		if len(args) > 0 {
+			// TODO(stevvooe): Implement selection of a few blobs. Not sure
+			// what kind of efficiency gains we can actually get here.
+			log.G(ctx).Warnf("args ignored; need to implement matchers")
+		}
+
+		var walkFn content.WalkFunc
+		if quiet {
+			walkFn = func(path string, fi os.FileInfo, dgst digest.Digest) error {
+				fmt.Println(dgst)
+				return nil
+			}
+		} else {
+			tw := tabwriter.NewWriter(os.Stdout, 1, 8, 1, '\t', 0)
+			defer tw.Flush()
+
+			fmt.Fprintf(tw, "DIGEST\tSIZE\tAGE\n")
+			walkFn = func(path string, fi os.FileInfo, dgst digest.Digest) error {
+				fmt.Fprintf(tw, "%s\t%s\t%s\n",
+					dgst,
+					units.HumanSize(float64(fi.Size())),
+					units.HumanDuration(time.Since(fi.ModTime())))
+				return nil
+			}
+
+		}
+
+		return cs.Walk(walkFn)
+	},
+}

cmd/dist/main.go

@@ -30,6 +30,11 @@ distribution tool
 	}
 	app.Commands = []cli.Command{
 		fetchCommand,
+		ingestCommand,
+		activeCommand,
+		pathCommand,
+		deleteCommand,
+		listCommand,
 	}
 	app.Before = func(context *cli.Context) error {
 		if context.GlobalBool("debug") {

cmd/dist/path.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	contextpkg "context"
+	"fmt"
+	"path/filepath"
+
+	"github.com/docker/containerd/content"
+	"github.com/docker/containerd/log"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/urfave/cli"
+)
+
+var pathCommand = cli.Command{
+	Name:      "path",
+	Usage:     "print the path to one or more blobs",
+	ArgsUsage: "[flags] [<digest>, ...]",
+	Description: `Display the paths to one or more blobs.
+	
+Output paths can be used to directly access blobs on disk.`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:   "root",
+			Usage:  "path to content store root",
+			Value:  ".content", // TODO(stevvooe): for now, just use the PWD/.content
+			EnvVar: "CONTAINERD_DIST_CONTENT_STORE",
+		},
+		cli.BoolFlag{
+			Name:  "quiet, q",
+			Usage: "elide digests in output",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		var (
+			ctx       = contextpkg.Background()
+			root      = context.String("root")
+			args      = []string(context.Args())
+			quiet     = context.Bool("quiet")
+			exitError error
+		)
+
+		if !filepath.IsAbs(root) {
+			var err error
+			root, err = filepath.Abs(root)
+			if err != nil {
+				return err
+			}
+		}
+
+		cs, err := content.Open(root)
+		if err != nil {
+			return err
+		}
+
+		// TODO(stevvooe): Take the set of paths from stdin.
+
+		if len(args) < 1 {
+			return fmt.Errorf("please specify a blob digest")
+		}
+
+		for _, arg := range args {
+			dgst, err := digest.Parse(arg)
+			if err != nil {
+				log.G(ctx).WithError(err).Errorf("parsing %q as digest failed", arg)
+				if exitError == nil {
+					exitError = err
+				}
+				continue
+			}
+
+			p, err := cs.GetPath(dgst)
+			if err != nil {
+				log.G(ctx).WithError(err).Errorf("getting path for %q failed", dgst)
+				if exitError == nil {
+					exitError = err
+				}
+				continue
+			}
+
+			if !quiet {
+				fmt.Println(dgst, p)
+			} else {
+				fmt.Println(p)
+			}
+		}
+
+		return exitError
+	},
+}