content: refactor content store for API

After iterating on the GRPC API, the changes required for the actual
implementation are now included in the content store. The begin change
is the move to a single, atomic `Ingester.Writer` method for locking
content ingestion on a key. From this, comes several new interface
definitions.

The main benefit here is the clarification between `Status` and `Info`
that came out of the GPRC API. `Status` tells the status of a write,
whereas `Info` is for querying metadata about various blobs.

Signed-off-by: Stephen J Day <stephen.day@docker.com>
This commit is contained in:
Stephen J Day 2017-02-17 00:07:02 -08:00
parent baaf7543dc
commit 621164bc84
No known key found for this signature in database
GPG key ID: 67B3DED84EDC823F
14 changed files with 573 additions and 611 deletions

23
cmd/dist/active.go vendored
View file

@ -3,11 +3,9 @@ package main
import (
"fmt"
"os"
"path/filepath"
"text/tabwriter"
"time"
"github.com/docker/containerd/content"
units "github.com/docker/go-units"
"github.com/urfave/cli"
)
@ -26,24 +24,11 @@ var activeCommand = cli.Command{
cli.StringFlag{
Name: "root",
Usage: "path to content store root",
Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content
Value: "/tmp/content", // TODO(stevvooe): for now, just use the PWD/.content
},
},
Action: func(context *cli.Context) error {
var (
// ctx = contextpkg.Background()
root = context.String("root")
)
if !filepath.IsAbs(root) {
var err error
root, err = filepath.Abs(root)
if err != nil {
return err
}
}
cs, err := content.Open(root)
cs, err := resolveContentStore(context)
if err != nil {
return err
}
@ -58,8 +43,8 @@ var activeCommand = cli.Command{
for _, active := range active {
fmt.Fprintf(tw, "%s\t%s\t%s\n",
active.Ref,
units.HumanSize(float64(active.Size)),
units.HumanDuration(time.Since(active.ModTime)))
units.HumanSize(float64(active.Offset)),
units.HumanDuration(time.Since(active.StartedAt)))
}
tw.Flush()

34
cmd/dist/common.go vendored Normal file
View file

@ -0,0 +1,34 @@
package main
import (
"net"
"path/filepath"
"time"
"github.com/docker/containerd/content"
"github.com/urfave/cli"
"google.golang.org/grpc"
)
func resolveContentStore(context *cli.Context) (*content.Store, error) {
root := context.GlobalString("root")
if !filepath.IsAbs(root) {
var err error
root, err = filepath.Abs(root)
if err != nil {
return nil, err
}
}
return content.NewStore(root)
}
func connectGRPC(context *cli.Context) (*grpc.ClientConn, error) {
socket := context.GlobalString("socket")
return grpc.Dial(socket,
grpc.WithBlock(),
grpc.WithInsecure(),
grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
return net.DialTimeout("unix", socket, timeout)
}),
)
}

21
cmd/dist/delete.go vendored
View file

@ -3,9 +3,7 @@ package main
import (
contextpkg "context"
"fmt"
"path/filepath"
"github.com/docker/containerd/content"
"github.com/docker/containerd/log"
digest "github.com/opencontainers/go-digest"
"github.com/urfave/cli"
@ -18,30 +16,15 @@ var deleteCommand = cli.Command{
ArgsUsage: "[flags] [<digest>, ...]",
Description: `Delete one or more blobs permanently. Successfully deleted
blobs are printed to stdout.`,
Flags: []cli.Flag{
cli.StringFlag{
Name: "root",
Usage: "path to content store root",
Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content
},
},
Flags: []cli.Flag{},
Action: func(context *cli.Context) error {
var (
ctx = contextpkg.Background()
root = context.String("root")
args = []string(context.Args())
exitError error
)
if !filepath.IsAbs(root) {
var err error
root, err = filepath.Abs(root)
if err != nil {
return err
}
}
cs, err := content.Open(root)
cs, err := resolveContentStore(context)
if err != nil {
return err
}

39
cmd/dist/get.go vendored Normal file
View file

@ -0,0 +1,39 @@
package main
import (
"io"
"os"
digest "github.com/opencontainers/go-digest"
"github.com/urfave/cli"
)
var getCommand = cli.Command{
Name: "get",
Usage: "get the data for an object",
ArgsUsage: "[flags] [<digest>, ...]",
Description: `Display the paths to one or more blobs.
Output paths can be used to directly access blobs on disk.`,
Flags: []cli.Flag{},
Action: func(context *cli.Context) error {
cs, err := resolveContentStore(context)
if err != nil {
return err
}
dgst, err := digest.Parse(context.Args().First())
if err != nil {
return err
}
rc, err := cs.Open(dgst)
if err != nil {
return err
}
defer rc.Close()
_, err = io.Copy(os.Stdout, rc)
return err
},
}

52
cmd/dist/ingest.go vendored
View file

@ -4,8 +4,6 @@ import (
contextpkg "context"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/docker/containerd/content"
"github.com/opencontainers/go-digest"
@ -18,17 +16,6 @@ var ingestCommand = cli.Command{
ArgsUsage: "[flags] <key>",
Description: `Ingest objects into the local content store.`,
Flags: []cli.Flag{
cli.DurationFlag{
Name: "timeout",
Usage: "total timeout for fetch",
EnvVar: "CONTAINERD_FETCH_TIMEOUT",
},
cli.StringFlag{
Name: "path, p",
Usage: "path to content store",
Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content
EnvVar: "CONTAINERD_DIST_CONTENT_STORE",
},
cli.Int64Flag{
Name: "expected-size",
Usage: "validate against provided size",
@ -40,57 +27,32 @@ var ingestCommand = cli.Command{
},
Action: func(context *cli.Context) error {
var (
ctx = contextpkg.Background()
timeout = context.Duration("timeout")
root = context.String("path")
ctx = background
cancel func()
ref = context.Args().First()
expectedSize = context.Int64("expected-size")
expectedDigest = digest.Digest(context.String("expected-digest"))
)
if timeout > 0 {
var cancel func()
ctx, cancel = contextpkg.WithTimeout(ctx, timeout)
defer cancel()
}
ctx, cancel = contextpkg.WithCancel(ctx)
defer cancel()
if err := expectedDigest.Validate(); expectedDigest != "" && err != nil {
return err
}
if !filepath.IsAbs(root) {
var err error
root, err = filepath.Abs(root)
if err != nil {
return err
}
}
cs, err := content.Open(root)
cs, err := resolveContentStore(context)
if err != nil {
return err
}
if expectedDigest != "" {
if ok, err := cs.Exists(expectedDigest); err != nil {
return err
} else if ok {
fmt.Fprintf(os.Stderr, "content with digest %v already exists\n", expectedDigest)
return nil
}
}
if ref == "" {
if expectedDigest == "" {
return fmt.Errorf("must specify a transaction reference or expected digest")
}
ref = strings.Replace(expectedDigest.String(), ":", "-", -1)
return fmt.Errorf("must specify a transaction reference")
}
// TODO(stevvooe): Allow ingest to be reentrant. Currently, we expect
// all data to be written in a single invocation. Allow multiple writes
// to the same transaction key followed by a commit.
return content.WriteBlob(cs, os.Stdin, ref, expectedSize, expectedDigest)
return content.WriteBlob(ctx, cs, os.Stdin, ref, expectedSize, expectedDigest)
},
}

17
cmd/dist/list.go vendored
View file

@ -4,7 +4,6 @@ import (
contextpkg "context"
"fmt"
"os"
"path/filepath"
"text/tabwriter"
"time"
@ -22,11 +21,6 @@ var listCommand = cli.Command{
ArgsUsage: "[flags] [<prefix>, ...]",
Description: `List blobs in the content store.`,
Flags: []cli.Flag{
cli.StringFlag{
Name: "root",
Usage: "path to content store root",
Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content
},
cli.BoolFlag{
Name: "quiet, q",
Usage: "print only the blob digest",
@ -35,20 +29,11 @@ var listCommand = cli.Command{
Action: func(context *cli.Context) error {
var (
ctx = contextpkg.Background()
root = context.String("root")
quiet = context.Bool("quiet")
args = []string(context.Args())
)
if !filepath.IsAbs(root) {
var err error
root, err = filepath.Abs(root)
if err != nil {
return err
}
}
cs, err := content.Open(root)
cs, err := resolveContentStore(context)
if err != nil {
return err
}

27
cmd/dist/main.go vendored
View file

@ -1,6 +1,7 @@
package main
import (
contextpkg "context"
"fmt"
"os"
@ -9,6 +10,10 @@ import (
"github.com/urfave/cli"
)
var (
background = contextpkg.Background()
)
func main() {
app := cli.NewApp()
app.Name = "dist"
@ -27,20 +32,38 @@ distribution tool
Name: "debug",
Usage: "enable debug output in logs",
},
cli.DurationFlag{
Name: "timeout",
Usage: "total timeout for fetch",
EnvVar: "CONTAINERD_FETCH_TIMEOUT",
},
cli.StringFlag{
Name: "root",
Usage: "path to content store root",
Value: "/tmp/content", // TODO(stevvooe): for now, just use the PWD/.content
},
}
app.Commands = []cli.Command{
fetchCommand,
ingestCommand,
activeCommand,
pathCommand,
getCommand,
deleteCommand,
listCommand,
applyCommand,
}
app.Before = func(context *cli.Context) error {
if context.GlobalBool("debug") {
var (
debug = context.GlobalBool("debug")
timeout = context.GlobalDuration("timeout")
)
if debug {
logrus.SetLevel(logrus.DebugLevel)
}
if timeout > 0 {
background, _ = contextpkg.WithTimeout(background, timeout)
}
return nil
}
if err := app.Run(os.Args); err != nil {

89
cmd/dist/path.go vendored
View file

@ -1,89 +0,0 @@
package main
import (
contextpkg "context"
"fmt"
"path/filepath"
"github.com/docker/containerd/content"
"github.com/docker/containerd/log"
digest "github.com/opencontainers/go-digest"
"github.com/urfave/cli"
)
var pathCommand = cli.Command{
Name: "path",
Usage: "print the path to one or more blobs",
ArgsUsage: "[flags] [<digest>, ...]",
Description: `Display the paths to one or more blobs.
Output paths can be used to directly access blobs on disk.`,
Flags: []cli.Flag{
cli.StringFlag{
Name: "root",
Usage: "path to content store root",
Value: ".content", // TODO(stevvooe): for now, just use the PWD/.content
EnvVar: "CONTAINERD_DIST_CONTENT_STORE",
},
cli.BoolFlag{
Name: "quiet, q",
Usage: "elide digests in output",
},
},
Action: func(context *cli.Context) error {
var (
ctx = contextpkg.Background()
root = context.String("root")
args = []string(context.Args())
quiet = context.Bool("quiet")
exitError error
)
if !filepath.IsAbs(root) {
var err error
root, err = filepath.Abs(root)
if err != nil {
return err
}
}
cs, err := content.Open(root)
if err != nil {
return err
}
// TODO(stevvooe): Take the set of paths from stdin.
if len(args) < 1 {
return fmt.Errorf("please specify a blob digest")
}
for _, arg := range args {
dgst, err := digest.Parse(arg)
if err != nil {
log.G(ctx).WithError(err).Errorf("parsing %q as digest failed", arg)
if exitError == nil {
exitError = err
}
continue
}
p, err := cs.GetPath(dgst)
if err != nil {
log.G(ctx).WithError(err).Errorf("getting path for %q failed", dgst)
if exitError == nil {
exitError = err
}
continue
}
if !quiet {
fmt.Println(dgst, p)
} else {
fmt.Println(p)
}
}
return exitError
},
}