Merge pull request #472 from stevvooe/expanding-dist-tool

dist: expand functionality of the dist tool

cmd/dist/active.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"text/tabwriter"
+	"time"
+
+	"github.com/docker/containerd/content"
+	units "github.com/docker/go-units"
+	"github.com/urfave/cli"
+)
+
+var activeCommand = cli.Command{
+	Name:        "active",
+	Usage:       "display active transfers.",
+	ArgsUsage:   "[flags] [<key>, ...]",
+	Description: `Display the ongoing transfers.`,
+	Flags: []cli.Flag{
+		cli.DurationFlag{
+			Name:   "timeout, t",
+			Usage:  "total timeout for fetch",
+			EnvVar: "CONTAINERD_FETCH_TIMEOUT",
+		},
+		cli.StringFlag{
+			Name:  "root",
+			Usage: "path to content store root",
+			Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content
+		},
+	},
+	Action: func(context *cli.Context) error {
+		var (
+			// ctx  = contextpkg.Background()
+			root = context.String("root")
+		)
+
+		if !filepath.IsAbs(root) {
+			var err error
+			root, err = filepath.Abs(root)
+			if err != nil {
+				return err
+			}
+		}
+
+		cs, err := content.Open(root)
+		if err != nil {
+			return err
+		}
+
+		active, err := cs.Active()
+		if err != nil {
+			return err
+		}
+
+		tw := tabwriter.NewWriter(os.Stdout, 1, 8, 1, '\t', 0)
+		fmt.Fprintf(tw, "REF\tSIZE\tAGE\n")
+		for _, active := range active {
+			fmt.Fprintf(tw, "%s\t%s\t%s\n",
+				active.Ref,
+				units.HumanSize(float64(active.Size)),
+				units.HumanDuration(time.Since(active.ModTime)))
+		}
+		tw.Flush()
+
+		return nil
+	},
+}

cmd/dist/delete.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	contextpkg "context"
+	"fmt"
+	"path/filepath"
+
+	"github.com/docker/containerd/content"
+	"github.com/docker/containerd/log"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/urfave/cli"
+)
+
+var deleteCommand = cli.Command{
+	Name:      "delete",
+	Aliases:   []string{"del"},
+	Usage:     "permanently delete one or more blobs.",
+	ArgsUsage: "[flags] [<digest>, ...]",
+	Description: `Delete one or more blobs permanently. Successfully deleted
+	blobs are printed to stdout.`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "root",
+			Usage: "path to content store root",
+			Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content
+		},
+	},
+	Action: func(context *cli.Context) error {
+		var (
+			ctx       = contextpkg.Background()
+			root      = context.String("root")
+			args      = []string(context.Args())
+			exitError error
+		)
+
+		if !filepath.IsAbs(root) {
+			var err error
+			root, err = filepath.Abs(root)
+			if err != nil {
+				return err
+			}
+		}
+
+		cs, err := content.Open(root)
+		if err != nil {
+			return err
+		}
+
+		for _, arg := range args {
+			dgst, err := digest.Parse(arg)
+			if err != nil {
+				if exitError == nil {
+					exitError = err
+				}
+				log.G(ctx).WithError(err).Errorf("could not delete %v", dgst)
+				continue
+			}
+
+			if err := cs.Delete(dgst); err != nil {
+				if exitError == nil {
+					exitError = err
+				}
+				log.G(ctx).WithError(err).Errorf("could not delete %v", dgst)
+				continue
+			}
+
+			fmt.Println(dgst)
+		}
+
+		return exitError
+	},
+}

cmd/dist/ingest.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	contextpkg "context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/docker/containerd/content"
+	"github.com/opencontainers/go-digest"
+	"github.com/urfave/cli"
+)
+
+var ingestCommand = cli.Command{
+	Name:        "ingest",
+	Usage:       "accept content into the store",
+	ArgsUsage:   "[flags] <key>",
+	Description: `Ingest objects into the local content store.`,
+	Flags: []cli.Flag{
+		cli.DurationFlag{
+			Name:   "timeout",
+			Usage:  "total timeout for fetch",
+			EnvVar: "CONTAINERD_FETCH_TIMEOUT",
+		},
+		cli.StringFlag{
+			Name:   "path, p",
+			Usage:  "path to content store",
+			Value:  ".content", // TODO(stevvooe): for now, just use the PWD/.content
+			EnvVar: "CONTAINERD_DIST_CONTENT_STORE",
+		},
+		cli.Int64Flag{
+			Name:  "expected-size",
+			Usage: "validate against provided size",
+		},
+		cli.StringFlag{
+			Name:  "expected-digest",
+			Usage: "verify content against expected digest",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		var (
+			ctx            = contextpkg.Background()
+			timeout        = context.Duration("timeout")
+			root           = context.String("path")
+			ref            = context.Args().First()
+			expectedSize   = context.Int64("expected-size")
+			expectedDigest = digest.Digest(context.String("expected-digest"))
+		)
+
+		if timeout > 0 {
+			var cancel func()
+			ctx, cancel = contextpkg.WithTimeout(ctx, timeout)
+			defer cancel()
+		}
+
+		if err := expectedDigest.Validate(); expectedDigest != "" && err != nil {
+			return err
+		}
+
+		if !filepath.IsAbs(root) {
+			var err error
+			root, err = filepath.Abs(root)
+			if err != nil {
+				return err
+			}
+		}
+
+		cs, err := content.Open(root)
+		if err != nil {
+			return err
+		}
+
+		if expectedDigest != "" {
+			if ok, err := cs.Exists(expectedDigest); err != nil {
+				return err
+			} else if ok {
+				fmt.Fprintf(os.Stderr, "content with digest %v already exists\n", expectedDigest)
+				return nil
+			}
+		}
+
+		if ref == "" {
+			if expectedDigest == "" {
+				return fmt.Errorf("must specify a transaction reference or expected digest")
+			}
+
+			ref = strings.Replace(expectedDigest.String(), ":", "-", -1)
+		}
+
+		// TODO(stevvooe): Allow ingest to be reentrant. Currently, we expect
+		// all data to be written in a single invocation. Allow multiple writes
+		// to the same transaction key followed by a commit.
+		return content.WriteBlob(cs, os.Stdin, ref, expectedSize, expectedDigest)
+	},
+}

cmd/dist/list.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	contextpkg "context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"text/tabwriter"
+	"time"
+
+	"github.com/docker/containerd/content"
+	"github.com/docker/containerd/log"
+	units "github.com/docker/go-units"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/urfave/cli"
+)
+
+var listCommand = cli.Command{
+	Name:        "list",
+	Aliases:     []string{"ls"},
+	Usage:       "list all blobs in the store.",
+	ArgsUsage:   "[flags] [<prefix>, ...]",
+	Description: `List blobs in the content store.`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "root",
+			Usage: "path to content store root",
+			Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content
+		},
+		cli.BoolFlag{
+			Name:  "quiet, q",
+			Usage: "print only the blob digest",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		var (
+			ctx   = contextpkg.Background()
+			root  = context.String("root")
+			quiet = context.Bool("quiet")
+			args  = []string(context.Args())
+		)
+
+		if !filepath.IsAbs(root) {
+			var err error
+			root, err = filepath.Abs(root)
+			if err != nil {
+				return err
+			}
+		}
+
+		cs, err := content.Open(root)
+		if err != nil {
+			return err
+		}
+
+		if len(args) > 0 {
+			// TODO(stevvooe): Implement selection of a few blobs. Not sure
+			// what kind of efficiency gains we can actually get here.
+			log.G(ctx).Warnf("args ignored; need to implement matchers")
+		}
+
+		var walkFn content.WalkFunc
+		if quiet {
+			walkFn = func(path string, fi os.FileInfo, dgst digest.Digest) error {
+				fmt.Println(dgst)
+				return nil
+			}
+		} else {
+			tw := tabwriter.NewWriter(os.Stdout, 1, 8, 1, '\t', 0)
+			defer tw.Flush()
+
+			fmt.Fprintf(tw, "DIGEST\tSIZE\tAGE\n")
+			walkFn = func(path string, fi os.FileInfo, dgst digest.Digest) error {
+				fmt.Fprintf(tw, "%s\t%s\t%s\n",
+					dgst,
+					units.HumanSize(float64(fi.Size())),
+					units.HumanDuration(time.Since(fi.ModTime())))
+				return nil
+			}
+
+		}
+
+		return cs.Walk(walkFn)
+	},
+}

cmd/dist/main.go

@@ -30,6 +30,11 @@ distribution tool
 	}
 	app.Commands = []cli.Command{
 		fetchCommand,
+		ingestCommand,
+		activeCommand,
+		pathCommand,
+		deleteCommand,
+		listCommand,
 	}
 	app.Before = func(context *cli.Context) error {
 		if context.GlobalBool("debug") {

cmd/dist/path.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	contextpkg "context"
+	"fmt"
+	"path/filepath"
+
+	"github.com/docker/containerd/content"
+	"github.com/docker/containerd/log"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/urfave/cli"
+)
+
+var pathCommand = cli.Command{
+	Name:      "path",
+	Usage:     "print the path to one or more blobs",
+	ArgsUsage: "[flags] [<digest>, ...]",
+	Description: `Display the paths to one or more blobs.
+	
+Output paths can be used to directly access blobs on disk.`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:   "root",
+			Usage:  "path to content store root",
+			Value:  ".content", // TODO(stevvooe): for now, just use the PWD/.content
+			EnvVar: "CONTAINERD_DIST_CONTENT_STORE",
+		},
+		cli.BoolFlag{
+			Name:  "quiet, q",
+			Usage: "elide digests in output",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		var (
+			ctx       = contextpkg.Background()
+			root      = context.String("root")
+			args      = []string(context.Args())
+			quiet     = context.Bool("quiet")
+			exitError error
+		)
+
+		if !filepath.IsAbs(root) {
+			var err error
+			root, err = filepath.Abs(root)
+			if err != nil {
+				return err
+			}
+		}
+
+		cs, err := content.Open(root)
+		if err != nil {
+			return err
+		}
+
+		// TODO(stevvooe): Take the set of paths from stdin.
+
+		if len(args) < 1 {
+			return fmt.Errorf("please specify a blob digest")
+		}
+
+		for _, arg := range args {
+			dgst, err := digest.Parse(arg)
+			if err != nil {
+				log.G(ctx).WithError(err).Errorf("parsing %q as digest failed", arg)
+				if exitError == nil {
+					exitError = err
+				}
+				continue
+			}
+
+			p, err := cs.GetPath(dgst)
+			if err != nil {
+				log.G(ctx).WithError(err).Errorf("getting path for %q failed", dgst)
+				if exitError == nil {
+					exitError = err
+				}
+				continue
+			}
+
+			if !quiet {
+				fmt.Println(dgst, p)
+			} else {
+				fmt.Println(p)
+			}
+		}
+
+		return exitError
+	},
+}